Add debug information to hunt down port collisions.

We've been seeing a lot of sporadic test failures with "Cannot assign
requested address" lately. Add some debug information to help us find
the cause:

- When server startup fails, print "netstat -tnlap" output to the test
  log. If the failure was caused by "Cannot assign requested address",
  this will hopefully tell us which process was occupying the port.
- In pageserver and safekeeper startup, print its PID. This way, we can
  correlate the PID from netstat output with the test that launched it.
- In safekeeper startup, print the HTTP port it's using to the log, in
  addition to the libpq port. The pageserver was already doing it.

.dockerignore

@@ -14,6 +14,7 @@
 !pgxn/
 !proxy/
 !safekeeper/
+!storage_broker/
 !vendor/postgres-v14/
 !vendor/postgres-v15/
 !workspace_hack/

.github/actions/allure-report/action.yml

@@ -32,8 +32,8 @@ runs:
           exit 2
         fi
 
-    - name: Calculate key
-      id: calculate-key
+    - name: Calculate variables
+      id: calculate-vars
       shell: bash -euxo pipefail {0}
       run: |
         # TODO: for manually triggered workflows (via workflow_dispatch) we need to have a separate key
@@ -41,14 +41,22 @@ runs:
         pr_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH" || true)
         if [ "${pr_number}" != "null" ]; then
           key=pr-${pr_number}
-        elif [ "${GITHUB_REF}" = "refs/heads/main" ]; then
+        elif [ "${GITHUB_REF_NAME}" = "main" ]; then
           # Shortcut for a special branch
           key=main
+        elif [ "${GITHUB_REF_NAME}" = "release" ]; then
+          # Shortcut for a special branch
+          key=release
         else
-          key=branch-$(echo ${GITHUB_REF#refs/heads/} | tr -c "[:alnum:]._-" "-")
+          key=branch-$(printf "${GITHUB_REF_NAME}" | tr -c "[:alnum:]._-" "-")
         fi
         echo "KEY=${key}" >> $GITHUB_OUTPUT
 
+        # Sanitize test selection to remove `/` and any other special characters
+        # Use printf instead of echo to avoid having `\n` at the end of the string
+        test_selection=$(printf "${{ inputs.test_selection }}" | tr -c "[:alnum:]._-" "-" )
+        echo "TEST_SELECTION=${test_selection}" >> $GITHUB_OUTPUT
+
     - uses: actions/setup-java@v3
       if: ${{ inputs.action == 'generate' }}
       with:
@@ -74,10 +82,11 @@ runs:
     - name: Upload Allure results
       if: ${{ inputs.action == 'store' }}
       env:
-        REPORT_PREFIX: reports/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
-        RAW_PREFIX: reports-raw/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
+        REPORT_PREFIX: reports/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
+        RAW_PREFIX: reports-raw/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
         TEST_OUTPUT: /tmp/test_output
         BUCKET: neon-github-public-dev
+        TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }}
       shell: bash -euxo pipefail {0}
       run: |
         # Add metadata
@@ -98,7 +107,7 @@ runs:
           BUILD_TYPE=${{ inputs.build_type }}
         EOF
 
-        ARCHIVE="${GITHUB_RUN_ID}-${{ inputs.test_selection }}-${GITHUB_RUN_ATTEMPT}-$(date +%s).tar.zst"
+        ARCHIVE="${GITHUB_RUN_ID}-${TEST_SELECTION}-${GITHUB_RUN_ATTEMPT}-$(date +%s).tar.zst"
         ZSTD_NBTHREADS=0
 
         tar -C ${TEST_OUTPUT}/allure/results -cf ${ARCHIVE} --zstd .
@@ -109,8 +118,9 @@ runs:
       if: ${{ inputs.action == 'generate' }}
       shell: bash -euxo pipefail {0}
       env:
-        LOCK_FILE: reports/${{ steps.calculate-key.outputs.KEY }}/lock.txt
+        LOCK_FILE: reports/${{ steps.calculate-vars.outputs.KEY }}/lock.txt
         BUCKET: neon-github-public-dev
+        TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }}
       run: |
         LOCK_TIMEOUT=300 # seconds
 
@@ -123,12 +133,12 @@ runs:
             fi
             sleep 1
           done
-          echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ inputs.test_selection }}" > lock.txt
+          echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" > lock.txt
           aws s3 mv --only-show-errors lock.txt "s3://${BUCKET}/${LOCK_FILE}"
 
           # A double-check that exactly WE have acquired the lock
           aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt
-          if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ inputs.test_selection }}" ]; then
+          if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" ]; then
             break
           fi
         done
@@ -137,8 +147,8 @@ runs:
       if: ${{ inputs.action == 'generate' }}
       id: generate-report
       env:
-        REPORT_PREFIX: reports/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
-        RAW_PREFIX: reports-raw/${{ steps.calculate-key.outputs.KEY }}/${{ inputs.build_type }}
+        REPORT_PREFIX: reports/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
+        RAW_PREFIX: reports-raw/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }}
         TEST_OUTPUT: /tmp/test_output
         BUCKET: neon-github-public-dev
       shell: bash -euxo pipefail {0}
@@ -192,12 +202,13 @@ runs:
       if: ${{ inputs.action == 'generate' && always() }}
       shell: bash -euxo pipefail {0}
       env:
-        LOCK_FILE: reports/${{ steps.calculate-key.outputs.KEY }}/lock.txt
+        LOCK_FILE: reports/${{ steps.calculate-vars.outputs.KEY }}/lock.txt
         BUCKET: neon-github-public-dev
+        TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }}
       run: |
         aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt || exit 0
 
-        if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ inputs.test_selection }}" ]; then
+        if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" ]; then
           aws s3 rm "s3://${BUCKET}/${LOCK_FILE}"
         fi
 

.github/actions/neon-branch-create/action.yml

@@ -0,0 +1,154 @@
+name: 'Create Branch'
+description: 'Create Branch using API'
+
+inputs:
+  api_key:
+    desctiption: 'Neon API key'
+    required: true
+  environment:
+    desctiption: 'dev (aka captest) or staging'
+    required: true
+  project_id:
+    desctiption: 'ID of the Project to create Branch in'
+    required: true
+outputs:
+  dsn:
+    description: 'Created Branch DSN (for main database)'
+    value: ${{ steps.change-password.outputs.dsn }}
+  branch_id:
+    description: 'Created Branch ID'
+    value: ${{ steps.create-branch.outputs.branch_id }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Parse Input
+      id: parse-input
+      shell: bash -euxo pipefail {0}
+      run: |
+        case "${ENVIRONMENT}" in
+          dev)
+            API_HOST=console.dev.neon.tech
+            ;;
+          staging)
+            API_HOST=console.stage.neon.tech
+            ;;
+          *)
+            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
+            exit 1
+            ;;
+        esac
+
+        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
+      env:
+        ENVIRONMENT: ${{ inputs.environment }}
+
+    - name: Create New Branch
+      id: create-branch
+      shell: bash -euxo pipefail {0}
+      run: |
+        for i in $(seq 1 10); do
+         branch=$(curl \
+            "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches" \
+            --header "Accept: application/json" \
+            --header "Content-Type: application/json" \
+            --header "Authorization: Bearer ${API_KEY}" \
+            --data "{
+              \"branch\": {
+                \"name\": \"Created by actions/neon-branch-create; GITHUB_RUN_ID=${GITHUB_RUN_ID} at $(date +%s)\"
+              }
+            }")
+
+          if [ -z "${branch}" ]; then
+            sleep 1
+            continue
+          fi
+
+          branch_id=$(echo $branch | jq --raw-output '.branch.id')
+          if [ "${branch_id}" == "null" ]; then
+            sleep 1
+            continue
+          fi
+
+          break
+        done
+
+        if [ -z "${branch_id}" ] || [ "${branch_id}" == "null" ]; then
+          echo 2>&1 "Failed to create branch after 10 attempts, the latest response was: ${branch}"
+          exit 1
+        fi
+
+        branch_id=$(echo $branch | jq --raw-output '.branch.id')
+        echo "branch_id=${branch_id}" >> $GITHUB_OUTPUT
+
+        host=$(echo $branch | jq --raw-output '.endpoints[0].host')
+        echo "host=${host}" >> $GITHUB_OUTPUT
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}
+        PROJECT_ID: ${{ inputs.project_id }}
+
+    - name: Get Role name
+      id: role-name
+      shell: bash -euxo pipefail {0}
+      run: |
+        roles=$(curl \
+          "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches/${BRANCH_ID}/roles" \
+          --fail \
+          --header "Accept: application/json" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer ${API_KEY}"
+          )
+
+        role_name=$(echo $roles | jq --raw-output '.roles[] | select(.protected == false) | .name')
+        echo "role_name=${role_name}" >> $GITHUB_OUTPUT
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}
+        PROJECT_ID: ${{ inputs.project_id }}
+        BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}
+
+    - name: Change Password
+      id: change-password
+      # A shell without `set -x` to not to expose password/dsn in logs
+      shell: bash -euo pipefail {0}
+      run: |
+        for i in $(seq 1 10); do
+          reset_password=$(curl \
+            "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches/${BRANCH_ID}/roles/${ROLE_NAME}/reset_password" \
+            --request POST \
+            --header "Accept: application/json" \
+            --header "Content-Type: application/json" \
+            --header "Authorization: Bearer ${API_KEY}"
+            )
+
+          if [ -z "${reset_password}" ]; then
+            sleep 1
+            continue
+          fi
+
+          password=$(echo $reset_password | jq --raw-output '.role.password')
+          if [ "${password}" == "null" ]; then
+            sleep 1
+            continue
+          fi
+
+          echo "::add-mask::${password}"
+          break
+        done
+
+        if [ -z "${password}" ] || [ "${password}" == "null" ]; then
+          echo 2>&1 "Failed to reset password after 10 attempts, the latest response was: ${reset_password}"
+          exit 1
+        fi
+
+        dsn="postgres://${ROLE_NAME}:${password}@${HOST}/neondb"
+        echo "::add-mask::${dsn}"
+        echo "dsn=${dsn}" >> $GITHUB_OUTPUT
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}
+        PROJECT_ID: ${{ inputs.project_id }}
+        BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}
+        ROLE_NAME: ${{ steps.role-name.outputs.role_name }}
+        HOST: ${{ steps.create-branch.outputs.host }}

.github/actions/neon-branch-delete/action.yml

@@ -0,0 +1,79 @@
+name: 'Delete Branch'
+description: 'Delete Branch using API'
+
+inputs:
+  api_key:
+    desctiption: 'Neon API key'
+    required: true
+  environment:
+    desctiption: 'dev (aka captest) or staging'
+    required: true
+  project_id:
+    desctiption: 'ID of the Project which should be deleted'
+    required: true
+  branch_id:
+    desctiption: 'ID of the branch to delete'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Parse Input
+      id: parse-input
+      shell: bash -euxo pipefail {0}
+      run: |
+        case "${ENVIRONMENT}" in
+          dev)
+            API_HOST=console.dev.neon.tech
+            ;;
+          staging)
+            API_HOST=console.stage.neon.tech
+            ;;
+          *)
+            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
+            exit 1
+            ;;
+        esac
+
+        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
+      env:
+        ENVIRONMENT: ${{ inputs.environment }}
+
+    - name: Delete Branch
+      # Do not try to delete a branch if .github/actions/neon-project-create
+      # or .github/actions/neon-branch-create failed before
+      if: ${{ inputs.project_id != '' && inputs.branch_id != '' }}
+      shell: bash -euxo pipefail {0}
+      run: |
+        for i in $(seq 1 10); do
+          deleted_branch=$(curl \
+            "https://${API_HOST}/api/v2/projects/${PROJECT_ID}/branches/${BRANCH_ID}" \
+            --request DELETE \
+            --header "Accept: application/json" \
+            --header "Content-Type: application/json" \
+            --header "Authorization: Bearer ${API_KEY}"
+            )
+
+          if [ -z "${deleted_branch}" ]; then
+            sleep 1
+            continue
+          fi
+
+          branch_id=$(echo $deleted_branch | jq --raw-output '.branch.id')
+          if [ "${branch_id}" == "null" ]; then
+            sleep 1
+            continue
+          fi
+
+          break
+        done
+
+        if [ -z "${branch_id}" ] || [ "${branch_id}" == "null" ]; then
+          echo 2>&1 "Failed to delete branch after 10 attempts, the latest response was: ${deleted_branch}"
+          exit 1
+        fi
+      env:
+        API_KEY: ${{ inputs.api_key }}
+        PROJECT_ID: ${{ inputs.project_id }}
+        BRANCH_ID: ${{ inputs.branch_id }}
+        API_HOST: ${{ steps.parse-input.outputs.api_host }}

.github/actions/neon-project-create/action.yml

@@ -6,7 +6,7 @@ inputs:
     desctiption: 'Neon API key'
     required: true
   environment:
-    desctiption: 'dev (aka captest) or stage'
+    desctiption: 'dev (aka captest) or staging'
     required: true
   region_id:
     desctiption: 'Region ID, if not set the project will be created in the default region'
@@ -29,11 +29,11 @@ runs:
         case "${ENVIRONMENT}" in
           dev)
             API_HOST=console.dev.neon.tech
-            REGION_ID=${REGION_ID:-eu-west-1}
+            REGION_ID=${REGION_ID:-aws-eu-west-1}
             ;;
           staging)
             API_HOST=console.stage.neon.tech
-            REGION_ID=${REGION_ID:-us-east-1}
+            REGION_ID=${REGION_ID:-aws-us-east-2}
             ;;
           *)
             echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
@@ -53,7 +53,7 @@ runs:
       shell: bash -euo pipefail {0}
       run: |
         project=$(curl \
-          "https://${API_HOST}/api/v1/projects" \
+          "https://${API_HOST}/api/v2/projects" \
           --fail \
           --header "Accept: application/json" \
           --header "Content-Type: application/json" \
@@ -61,7 +61,6 @@ runs:
           --data "{
             \"project\": {
               \"name\": \"Created by actions/neon-project-create; GITHUB_RUN_ID=${GITHUB_RUN_ID}\",
-              \"platform_id\": \"aws\",
               \"region_id\": \"${REGION_ID}\",
               \"settings\": { }
             }
@@ -70,11 +69,11 @@ runs:
         # Mask password
         echo "::add-mask::$(echo $project | jq --raw-output '.roles[] | select(.name != "web_access") | .password')"
 
-        dsn=$(echo $project | jq --raw-output '.roles[] | select(.name != "web_access") | .dsn')/main
+        dsn=$(echo $project | jq --raw-output '.connection_uris[0].connection_uri')
         echo "::add-mask::${dsn}"
         echo "dsn=${dsn}" >> $GITHUB_OUTPUT
 
-        project_id=$(echo $project | jq --raw-output '.id')
+        project_id=$(echo $project | jq --raw-output '.project.id')
         echo "project_id=${project_id}" >> $GITHUB_OUTPUT
       env:
         API_KEY: ${{ inputs.api_key }}

.github/actions/neon-project-delete/action.yml

@@ -6,7 +6,7 @@ inputs:
     desctiption: 'Neon API key'
     required: true
   environment:
-    desctiption: 'dev (aka captest) or stage'
+    desctiption: 'dev (aka captest) or staging'
     required: true
   project_id:
     desctiption: 'ID of the Project to delete'
@@ -37,17 +37,17 @@ runs:
         ENVIRONMENT: ${{ inputs.environment }}
 
     - name: Delete Neon Project
+      # Do not try to delete a project if .github/actions/neon-project-create failed before
+      if: ${{ inputs.project_id != '' }}
       shell: bash -euxo pipefail {0}
       run: |
-        # Allow PROJECT_ID to be empty/null for cases when .github/actions/neon-project-create failed
-        if [ -n "${PROJECT_ID}" ]; then
-          curl -X "POST" \
-            "https://${API_HOST}/api/v1/projects/${PROJECT_ID}/delete" \
-            --fail \
-            --header "Accept: application/json" \
-            --header "Content-Type: application/json" \
-            --header "Authorization: Bearer ${API_KEY}"
-        fi
+        curl \
+          "https://${API_HOST}/api/v2/projects/${PROJECT_ID}" \
+          --fail \
+          --request DELETE \
+          --header "Accept: application/json" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer ${API_KEY}"
       env:
         API_KEY: ${{ inputs.api_key }}
         PROJECT_ID: ${{ inputs.project_id }}

.github/actions/run-python-test-set/action.yml

@@ -55,6 +55,22 @@ runs:
         name: neon-${{ runner.os }}-${{ inputs.build_type }}-artifact
         path: /tmp/neon
 
+    - name: Download Neon binaries for the previous release
+      if: inputs.build_type != 'remote'
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-${{ inputs.build_type }}-artifact
+        path: /tmp/neon-previous
+        prefix: latest
+
+    - name: Download compatibility snapshot for Postgres 14
+      if: inputs.build_type != 'remote'
+      uses: ./.github/actions/download
+      with:
+        name: compatibility-snapshot-${{ inputs.build_type }}-pg14
+        path: /tmp/compatibility_snapshot_pg14
+        prefix: latest
+
     - name: Checkout
       if: inputs.needs_postgres_source == 'true'
       uses: actions/checkout@v3
@@ -73,23 +89,18 @@ runs:
       shell: bash -euxo pipefail {0}
       run: ./scripts/pysync
 
-    - name: Download compatibility snapshot for Postgres 14
-      if: inputs.build_type != 'remote'
-      uses: ./.github/actions/download
-      with:
-        name: compatibility-snapshot-${{ inputs.build_type }}-pg14
-        path: /tmp/compatibility_snapshot_pg14
-        prefix: latest
-
     - name: Run pytest
       env:
         NEON_BIN: /tmp/neon/bin
+        COMPATIBILITY_NEON_BIN: /tmp/neon-previous/bin
+        COMPATIBILITY_POSTGRES_DISTRIB_DIR: /tmp/neon-previous/pg_install
         TEST_OUTPUT: /tmp/test_output
         BUILD_TYPE: ${{ inputs.build_type }}
         AWS_ACCESS_KEY_ID: ${{ inputs.real_s3_access_key_id }}
         AWS_SECRET_ACCESS_KEY: ${{ inputs.real_s3_secret_access_key }}
         COMPATIBILITY_SNAPSHOT_DIR: /tmp/compatibility_snapshot_pg14
-        ALLOW_BREAKING_CHANGES: contains(github.event.pull_request.labels.*.name, 'breaking changes')
+        ALLOW_BACKWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'backward compatibility breakage')
+        ALLOW_FORWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'forward compatibility breakage')
       shell: bash -euxo pipefail {0}
       run: |
         # PLATFORM will be embedded in the perf test report
@@ -112,7 +123,12 @@ runs:
           exit 1
         fi
         if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then
+          # -n4 uses four processes to run tests via pytest-xdist
           EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
+
+          # --dist=loadgroup points tests marked with @pytest.mark.xdist_group
+          # to the same worker to make @pytest.mark.order work with xdist
+          EXTRA_PARAMS="--dist=loadgroup $EXTRA_PARAMS"
         fi
 
         if [[ "${{ inputs.run_with_real_s3 }}" == "true" ]]; then
@@ -147,9 +163,9 @@ runs:
         # --verbose prints name of each test (helpful when there are
         # multiple tests in one file)
         # -rA prints summary in the end
-        # -n4 uses four processes to run tests via pytest-xdist
         # -s is not used to prevent pytest from capturing output, because tests are running
         # in parallel and logs are mixed between different tests
+        #
         mkdir -p $TEST_OUTPUT/allure/results
         "${cov_prefix[@]}" ./scripts/pytest \
           --junitxml=$TEST_OUTPUT/junit.xml \
@@ -169,12 +185,12 @@ runs:
       uses: ./.github/actions/upload
       with:
         name: compatibility-snapshot-${{ inputs.build_type }}-pg14-${{ github.run_id }}
-        # The path includes a test name (test_prepare_snapshot) and directory that the test creates (compatibility_snapshot_pg14), keep the path in sync with the test
-        path: /tmp/test_output/test_prepare_snapshot/compatibility_snapshot_pg14/
+        # The path includes a test name (test_create_snapshot) and directory that the test creates (compatibility_snapshot_pg14), keep the path in sync with the test
+        path: /tmp/test_output/test_create_snapshot/compatibility_snapshot_pg14/
         prefix: latest
 
     - name: Create Allure report
-      if: always()
+      if: success() || failure()
       uses: ./.github/actions/allure-report
       with:
         action: store

.github/ansible/.gitignore

@@ -1,5 +1,3 @@
-zenith_install.tar.gz
-.zenith_current_version
 neon_install.tar.gz
 .neon_current_version
 

.github/ansible/get_binaries.sh

@@ -25,6 +25,7 @@ mkdir neon_install/bin/
 docker cp ${ID}:/usr/local/bin/pageserver neon_install/bin/
 docker cp ${ID}:/usr/local/bin/pageserver_binutils neon_install/bin/
 docker cp ${ID}:/usr/local/bin/safekeeper neon_install/bin/
+docker cp ${ID}:/usr/local/bin/storage_broker neon_install/bin/
 docker cp ${ID}:/usr/local/bin/proxy neon_install/bin/
 docker cp ${ID}:/usr/local/v14/bin/ neon_install/v14/bin/
 docker cp ${ID}:/usr/local/v15/bin/ neon_install/v15/bin/

.github/ansible/production.hosts.yaml

@@ -22,6 +22,10 @@ storage:
           console_region_id: aws-us-west-2
         zenith-1-ps-3:
           console_region_id: aws-us-west-2
+        zenith-1-ps-4:
+          console_region_id: aws-us-west-2
+        zenith-1-ps-5:
+          console_region_id: aws-us-west-2
 
     safekeepers:
       hosts:

.github/ansible/staging.eu-west-1.hosts.yaml

@@ -0,0 +1,33 @@
+storage:
+  vars:
+    bucket_name: neon-dev-storage-eu-west-1
+    bucket_region: eu-west-1
+    console_mgmt_base_url: http://console-staging.local
+    etcd_endpoints: etcd-0.eu-west-1.aws.neon.build:2379
+    pageserver_config_stub:
+      pg_distrib_dir: /usr/local
+      remote_storage:
+        bucket_name: "{{ bucket_name }}"
+        bucket_region: "{{ bucket_region }}"
+        prefix_in_bucket: "pageserver/v1"
+    safekeeper_s3_prefix: safekeeper/v1/wal
+    hostname_suffix: ""
+    remote_user: ssm-user
+    ansible_aws_ssm_region: eu-west-1
+    ansible_aws_ssm_bucket_name: neon-dev-storage-eu-west-1
+    console_region_id: aws-eu-west-1
+
+  children:
+    pageservers:
+      hosts:
+        pageserver-0.eu-west-1.aws.neon.build:
+          ansible_host: i-01d496c5041c7f34c
+
+    safekeepers:
+      hosts:
+        safekeeper-0.eu-west-1.aws.neon.build:
+          ansible_host: i-05226ef85722831bf
+        safekeeper-1.eu-west-1.aws.neon.build:
+          ansible_host: i-06969ee1bf2958bfc
+        safekeeper-2.eu-west-1.aws.neon.build:
+          ansible_host: i-087892e9625984a0b

.github/ansible/staging.hosts.yaml

@@ -3,7 +3,7 @@ storage:
     bucket_name: zenith-staging-storage-us-east-1
     bucket_region: us-east-1
     console_mgmt_base_url: http://console-staging.local
-    etcd_endpoints: zenith-us-stage-etcd.local:2379
+    etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379
     pageserver_config_stub:
       pg_distrib_dir: /usr/local
       remote_storage:

.github/ansible/staging.us-east-2.hosts.yaml

@@ -22,6 +22,8 @@ storage:
       hosts:
         pageserver-0.us-east-2.aws.neon.build:
           ansible_host: i-0c3e70929edb5d691
+        pageserver-1.us-east-2.aws.neon.build:
+          ansible_host: i-0565a8b4008aa3f40
 
     safekeepers:
       hosts:

.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml

@@ -0,0 +1,31 @@
+# Helm chart values for neon-proxy-scram.
+# This is a YAML-formatted file.
+
+image:
+  repository: neondatabase/neon
+
+settings:
+  authBackend: "console"
+  authEndpoint: "http://console-staging.local/management/api/v2"
+  domain: "*.eu-west-1.aws.neon.build"
+
+# -- Additional labels for neon-proxy pods
+podLabels:
+  zenith_service: proxy-scram
+  zenith_env: dev
+  zenith_region: eu-west-1
+  zenith_region_slug: eu-west-1
+
+exposedService:
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+    external-dns.alpha.kubernetes.io/hostname: eu-west-1.aws.neon.build
+
+#metrics:
+#  enabled: true
+#  serviceMonitor:
+#    enabled: true
+#    selector:
+#      release: kube-prometheus-stack

.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml

@@ -0,0 +1,39 @@
+# Helm chart values for neon-proxy-link.
+# This is a YAML-formatted file.
+
+image:
+  repository: neondatabase/neon
+
+settings:
+  authBackend: "link"
+  authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/"
+  uri: "https://console.stage.neon.tech/psql_session/"
+
+# -- Additional labels for neon-proxy-link pods
+podLabels:
+  zenith_service: proxy
+  zenith_env: dev
+  zenith_region: us-east-2
+  zenith_region_slug: us-east-2
+
+service:
+  type: LoadBalancer
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internal
+    external-dns.alpha.kubernetes.io/hostname: neon-proxy-link-mgmt.beta.us-east-2.aws.neon.build
+
+exposedService:
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+    external-dns.alpha.kubernetes.io/hostname: neon-proxy-link.beta.us-east-2.aws.neon.build
+
+#metrics:
+#  enabled: true
+#  serviceMonitor:
+#    enabled: true
+#    selector:
+#      release: kube-prometheus-stack

.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml

@@ -0,0 +1,31 @@
+# Helm chart values for neon-proxy-scram.
+# This is a YAML-formatted file.
+
+image:
+  repository: neondatabase/neon
+
+settings:
+  authBackend: "console"
+  authEndpoint: "http://console-staging.local/management/api/v2"
+  domain: "*.cloud.stage.neon.tech"
+
+# -- Additional labels for neon-proxy pods
+podLabels:
+  zenith_service: proxy-scram-legacy
+  zenith_env: dev
+  zenith_region: us-east-2
+  zenith_region_slug: us-east-2
+
+exposedService:
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: external
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+    external-dns.alpha.kubernetes.io/hostname: neon-proxy-scram-legacy.beta.us-east-2.aws.neon.build
+
+#metrics:
+#  enabled: true
+#  serviceMonitor:
+#    enabled: true
+#    selector:
+#      release: kube-prometheus-stack

.github/workflows/benchmarking.yml

@@ -110,8 +110,14 @@ jobs:
         rm -rf perf-report-staging
         mkdir -p perf-report-staging
         # Set --sparse-ordering option of pytest-order plugin to ensure tests are running in order of appears in the file,
-        # it's important for test_perf_pgbench.py::test_pgbench_remote_* tests
-        ./scripts/pytest test_runner/performance/ -v -m "remote_cluster" --sparse-ordering --out-dir perf-report-staging --timeout 5400
+        # it's important for test_perf_pgbench.py::test_pgbench_remote_* tests.
+        # Do not run tests from test_runner/performance/test_perf_olap.py because they require a prepared DB. We run them separately in `clickbench-compare` job.
+        ./scripts/pytest test_runner/performance/ -v \
+          -m "remote_cluster" \
+          --sparse-ordering \
+          --out-dir perf-report-staging \
+          --timeout 5400 \
+          --ignore test_runner/performance/test_perf_olap.py
 
     - name: Submit result
       env:
@@ -144,7 +150,9 @@ jobs:
         # neon-captest-new: Run pgbench in a freshly created project
         # neon-captest-reuse: Same, but reusing existing project
         # neon-captest-prefetch: Same, with prefetching enabled (new project)
-        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch ]
+        # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
+        # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
+        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch, rds-postgres ]
         db_size: [ 10gb ]
         include:
           - platform: neon-captest-new
@@ -164,7 +172,7 @@ jobs:
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
       PLATFORM: ${{ matrix.platform }}
 
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
       options: --init
@@ -205,10 +213,13 @@ jobs:
             CONNSTR=${{ steps.create-neon-project.outputs.dsn }}
             ;;
           rds-aurora)
-            CONNSTR=${{ secrets.BENCHMARK_RDS_CONNSTR }}
+            CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CONNSTR }}
+            ;;
+          rds-postgres)
+            CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CONNSTR }}
             ;;
           *)
-            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'neon-captest-new', 'neon-captest-prefetch' or 'rds-aurora'"
+            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'neon-captest-new', 'neon-captest-prefetch', 'rds-aurora', or 'rds-postgres'"
             exit 1
             ;;
         esac
@@ -220,8 +231,8 @@ jobs:
     - name: Set database options
       if: matrix.platform == 'neon-captest-prefetch'
       run: |
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET enable_seqscan_prefetch=on"
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET seqscan_prefetch_buffers=10"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE neondb SET enable_seqscan_prefetch=on"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE neondb SET seqscan_prefetch_buffers=10"
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
 
@@ -265,7 +276,7 @@ jobs:
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
 
     - name: Create Allure report
-      if: always()
+      if: success() || failure()
       uses: ./.github/actions/allure-report
       with:
         action: generate
@@ -287,3 +298,112 @@ jobs:
         slack-message: "Periodic perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
       env:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+  clickbench-compare:
+    # ClichBench DB for rds-aurora and rds-Postgres deployed to the same clusters
+    # we use for performance testing in pgbench-compare.
+    # Run this job only when pgbench-compare is finished to avoid the intersection.
+    # We might change it after https://github.com/neondatabase/neon/issues/2900.
+    #
+    # *_CLICKBENCH_CONNSTR: Genuine ClickBench DB with ~100M rows
+    # *_CLICKBENCH_10M_CONNSTR: DB with the first 10M rows of ClickBench DB
+    if: success() || failure()
+    needs: [ pgbench-compare ]
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # neon-captest-prefetch: We have pre-created projects with prefetch enabled
+        # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
+        # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
+        platform: [ neon-captest-prefetch, rds-postgres, rds-aurora ]
+
+    env:
+      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
+      DEFAULT_PG_VERSION: 14
+      TEST_OUTPUT: /tmp/test_output
+      BUILD_TYPE: remote
+      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
+      PLATFORM: ${{ matrix.platform }}
+
+    runs-on: [ self-hosted, dev, x64 ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
+      options: --init
+
+    timeout-minutes: 360 # 6h
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Download Neon artifact
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-release-artifact
+        path: /tmp/neon/
+        prefix: latest
+
+    - name: Add Postgres binaries to PATH
+      run: |
+        ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
+        echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH
+
+    - name: Set up Connection String
+      id: set-up-connstr
+      run: |
+        case "${PLATFORM}" in
+          neon-captest-prefetch)
+            CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_10M_CONNSTR }}
+            ;;
+          rds-aurora)
+            CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CLICKBENCH_10M_CONNSTR }}
+            ;;
+          rds-postgres)
+            CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CLICKBENCH_10M_CONNSTR }}
+            ;;
+          *)
+            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-prefetch', 'rds-aurora', or 'rds-postgres'"
+            exit 1
+            ;;
+        esac
+
+        echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
+
+        psql ${CONNSTR} -c "SELECT version();"
+
+    - name: Set database options
+      if: matrix.platform == 'neon-captest-prefetch'
+      run: |
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET enable_seqscan_prefetch=on"
+        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE main SET seqscan_prefetch_buffers=10"
+      env:
+        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+
+    - name: Benchmark clickbench
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance/test_perf_olap.py
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        extra_params: -m remote_cluster --timeout 21600 -k test_clickbench
+      env:
+        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
+        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
+        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+
+    - name: Create Allure report
+      if: success() || failure()
+      uses: ./.github/actions/allure-report
+      with:
+        action: generate
+        build_type: ${{ env.BUILD_TYPE }}
+
+    - name: Post to a Slack channel
+      if: ${{ github.event.schedule && failure() }}
+      uses: slackapi/slack-github-action@v1
+      with:
+        channel-id: "C033QLM5P7D" # dev-staging-stream
+        slack-message: "Periodic OLAP perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/build_and_test.yml

@@ -18,8 +18,8 @@ env:
 
 jobs:
   tag:
-    runs-on: dev
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
+    runs-on: [ self-hosted, dev, x64 ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
     outputs:
       build-tag: ${{steps.build-tag.outputs.tag}}
 
@@ -46,7 +46,7 @@ jobs:
         id: build-tag
 
   build-neon:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -236,7 +236,7 @@ jobs:
         uses: ./.github/actions/save-coverage-data
 
   regress-tests:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -268,34 +268,8 @@ jobs:
         if: matrix.build_type == 'debug'
         uses: ./.github/actions/save-coverage-data
 
-  upload-latest-artifacts:
-    runs-on: dev
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
-      options: --init
-    needs: [ regress-tests ]
-    if: github.ref_name == 'main'
-    steps:
-      - name: Copy Neon artifact to the latest directory
-        shell: bash -euxo pipefail {0}
-        env:
-          BUCKET: neon-github-public-dev
-          PREFIX: artifacts/${{ github.run_id }}
-        run: |
-          for build_type in debug release; do
-            FILENAME=neon-${{ runner.os }}-${build_type}-artifact.tar.zst
-
-            S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${PREFIX} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true)
-            if [ -z "${S3_KEY}" ]; then
-              echo 2>&1 "Neither s3://${BUCKET}/${PREFIX}/${FILENAME} nor its version from previous attempts exist"
-              exit 1
-            fi
-
-            time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/artifacts/latest/${FILENAME}
-          done
-
   benchmarks:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -326,12 +300,12 @@ jobs:
       # while coverage is currently collected for the debug ones
 
   merge-allure-report:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
     needs: [ regress-tests, benchmarks ]
-    if: always()
+    if: success() || failure()
     strategy:
       fail-fast: false
       matrix:
@@ -364,7 +338,7 @@ jobs:
           DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json
 
   coverage-report:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -441,15 +415,19 @@ jobs:
         shell: bash -euxo pipefail {0}
 
   trigger-e2e-tests:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
       options: --init
-    needs: [ build-neon ]
+    needs: [ push-docker-hub, tag ]
     steps:
       - name: Set PR's status to pending and request a remote CI test
         run: |
+          # For pull requests, GH Actions set "github.sha" variable to point at a fake merge commit
+          # but we need to use a real sha of a latest commit in the PR's branch for the e2e job,
+          # to place a job run status update later.
           COMMIT_SHA=${{ github.event.pull_request.head.sha }}
+          # For non-PR kinds of runs, the above will produce an empty variable, pick the original sha value for those
           COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}}
 
           REMOTE_REPO="${{ github.repository_owner }}/cloud"
@@ -475,12 +453,14 @@ jobs:
               \"inputs\": {
                 \"ci_job_name\": \"neon-cloud-e2e\",
                 \"commit_hash\": \"$COMMIT_SHA\",
-                \"remote_repo\": \"${{ github.repository }}\"
+                \"remote_repo\": \"${{ github.repository }}\",
+                \"storage_image_tag\": \"${{ needs.tag.outputs.build-tag }}\",
+                \"compute_image_tag\": \"${{ needs.tag.outputs.build-tag }}\"
               }
             }"
 
   neon-image:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     needs: [ tag ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
 
@@ -498,7 +478,7 @@ jobs:
         run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}}
 
   compute-tools-image:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     needs: [ tag ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
 
@@ -512,28 +492,8 @@ jobs:
       - name: Kaniko build compute tools
         run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}}
 
-  compute-node-image:
-    runs-on: dev
-    container: gcr.io/kaniko-project/executor:v1.9.0-debug
-    needs: [ tag ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1 # v3 won't work with kaniko
-        with:
-          submodules: true
-          fetch-depth: 0
-
-      - name: Configure ECR login
-        run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json
-
-        # compute-node uses postgres 14, which is default now
-        # cloud repo depends on this image name, thus duplicating it
-        # remove compute-node when cloud repo is updated
-      - name: Kaniko build compute node with extensions v14 (compatibility)
-        run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}}
-
   compute-node-image-v14:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
     needs: [ tag ]
     steps:
@@ -549,9 +509,8 @@ jobs:
       - name: Kaniko build compute node with extensions v14
         run: /kaniko/executor --skip-unused-stages  --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache  --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v14 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}}
 
-
   compute-node-image-v15:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container: gcr.io/kaniko-project/executor:v1.9.0-debug
     needs: [ tag ]
     steps:
@@ -567,18 +526,58 @@ jobs:
       - name: Kaniko build compute node with extensions v15
         run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-v15 --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}}
 
+  test-images:
+    needs: [ tag, neon-image, compute-node-image-v14, compute-node-image-v15, compute-tools-image ]
+    runs-on: [ self-hosted, dev, x64 ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
+      # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
+      # Regular pageserver version string looks like
+      #   Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: []
+      # Bad versions might loop like:
+      #   Neon page server git-env:local failpoints: true, features: ["testing"]
+      # Ensure that we don't have bad versions.
+      - name: Verify image versions
+        shell: bash # ensure no set -e for better error messages
+        run: |
+          pageserver_version=$(docker run --rm 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
+
+          echo "Pageserver version string: $pageserver_version"
+
+          if ! echo "$pageserver_version" | grep -qv 'git-env:local' ; then
+            echo "Pageserver version should not be the default Dockerfile one"
+            exit 1
+          fi
+
+          if ! echo "$pageserver_version" | grep -qv '"testing"' ; then
+            echo "Pageserver version should have no testing feature enabled"
+            exit 1
+          fi
+
+      - name: Verify docker-compose example
+        run: env REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh
+
+      - name: Print logs and clean up
+        if: always()
+        run: |
+          docker compose -f ./docker-compose/docker-compose.yml logs || 0
+          docker compose -f ./docker-compose/docker-compose.yml down
+
   promote-images:
-    runs-on: dev
-    needs: [ tag, neon-image, compute-node-image, compute-node-image-v14, compute-node-image-v15, compute-tools-image ]
+    runs-on: [ self-hosted, dev, x64 ]
+    needs: [ tag, test-images ]
     if: github.event_name != 'workflow_dispatch'
     container: amazon/aws-cli
     strategy:
       fail-fast: false
       matrix:
-        # compute-node uses postgres 14, which is default now
-        # cloud repo depends on this image name, thus duplicating it
-        # remove compute-node when cloud repo is updated
-        name: [ neon, compute-node, compute-node-v14, compute-node-v15, compute-tools ]
+        name: [ neon, compute-node-v14, compute-node-v15, compute-tools ]
 
     steps:
       - name: Promote image to latest
@@ -587,7 +586,7 @@ jobs:
           aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST"
 
   push-docker-hub:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     needs: [ promote-images, tag ]
     container: golang:1.19-bullseye
 
@@ -608,9 +607,6 @@ jobs:
       - name: Pull compute tools image from ECR
         run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} compute-tools
 
-      - name: Pull compute node image from ECR
-        run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} compute-node
-
       - name: Pull compute node v14 image from ECR
         run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} compute-node-v14
 
@@ -627,7 +623,6 @@ jobs:
         run: |
           crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/neon:latest
           crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest
-          crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node:latest
           crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest
           crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:latest
 
@@ -643,9 +638,6 @@ jobs:
       - name: Push compute tools image to Docker Hub
         run: crane push compute-tools neondatabase/compute-tools:${{needs.tag.outputs.build-tag}}
 
-      - name: Push compute node image to Docker Hub
-        run: crane push compute-node neondatabase/compute-node:${{needs.tag.outputs.build-tag}}
-
       - name: Push compute node v14 image to Docker Hub
         run: crane push compute-node-v14 neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}}
 
@@ -662,7 +654,6 @@ jobs:
         run: |
           crane tag neondatabase/neon:${{needs.tag.outputs.build-tag}} latest
           crane tag neondatabase/compute-tools:${{needs.tag.outputs.build-tag}} latest
-          crane tag neondatabase/compute-node:${{needs.tag.outputs.build-tag}} latest
           crane tag neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}} latest
           crane tag neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}} latest
 
@@ -745,7 +736,7 @@ jobs:
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-new:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
     # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
     # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
@@ -770,7 +761,6 @@ jobs:
         run: |
           export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
           cd "$(pwd)/.github/ansible"
-
           if [[ "$GITHUB_REF_NAME" == "main" ]]; then
             ./get_binaries.sh
           elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
@@ -779,6 +769,38 @@ jobs:
             echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
             exit 1
           fi
+          ansible-galaxy collection install sivel.toiletwater
+          ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
+          rm -f neon_install.tar.gz .neon_current_version
+
+  deploy-pr-test-new:
+    runs-on: [ self-hosted, dev, x64 ]
+    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
+    # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
+    # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
+    needs: [ push-docker-hub, tag, regress-tests ]
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'deploy-test-storage') && 
+      github.event_name != 'workflow_dispatch'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        target_region: [ eu-west-1 ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 0
+
+      - name: Redeploy
+        run: |
+          export DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          cd "$(pwd)/.github/ansible"
+
+          ./get_binaries.sh
 
           ansible-galaxy collection install sivel.toiletwater
           ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{secrets.NEON_STAGING_API_KEY}}
@@ -789,7 +811,7 @@ jobs:
     container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
     # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version.
     # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    needs: [ push-docker-hub, tag, regress-tests ]
     if: |
       (github.ref_name == 'release') &&
       github.event_name != 'workflow_dispatch'
@@ -825,7 +847,7 @@ jobs:
           rm -f neon_install.tar.gz .neon_current_version
 
   deploy-proxy:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:latest
     # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
     needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
@@ -867,10 +889,10 @@ jobs:
           helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
 
   deploy-proxy-new:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned
     # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    needs: [ push-docker-hub, tag, regress-tests ]
     if: |
       (github.ref_name == 'main') &&
       github.event_name != 'workflow_dispatch'
@@ -882,6 +904,12 @@ jobs:
         include:
           - target_region:  us-east-2
             target_cluster: dev-us-east-2-beta
+            deploy_link_proxy: true
+            deploy_legacy_scram_proxy: true
+          - target_region:  eu-west-1
+            target_cluster: dev-eu-west-1-zeta
+            deploy_link_proxy: false
+            deploy_legacy_scram_proxy: false
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -894,16 +922,28 @@ jobs:
           helm repo add neondatabase https://neondatabase.github.io/helm-charts
           aws --region ${{ matrix.target_region }} eks update-kubeconfig --name  ${{ matrix.target_cluster }}
 
-      - name: Re-deploy proxy
+      - name: Re-deploy scram proxy
         run: |
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
           helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
 
+      - name: Re-deploy link proxy
+        if: matrix.deploy_link_proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+
+      - name: Re-deploy legacy scram proxy
+        if: matrix.deploy_legacy_scram_proxy
+        run: |
+          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
+          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
+
   deploy-proxy-prod-new:
     runs-on: prod
     container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
     # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently.
-    needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ]
+    needs: [ push-docker-hub, tag, regress-tests ]
     if: |
       (github.ref_name == 'release') &&
       github.event_name != 'workflow_dispatch'
@@ -936,8 +976,8 @@ jobs:
           DOCKER_TAG=${{needs.tag.outputs.build-tag}}
           helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
 
-  promote-compatibility-test-snapshot:
-    runs-on: dev
+  promote-compatibility-data:
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
@@ -950,9 +990,24 @@ jobs:
           BUCKET: neon-github-public-dev
           PREFIX: artifacts/latest
         run: |
+          # Update compatibility snapshot for the release
           for build_type in debug release; do
             OLD_FILENAME=compatibility-snapshot-${build_type}-pg14-${GITHUB_RUN_ID}.tar.zst
             NEW_FILENAME=compatibility-snapshot-${build_type}-pg14.tar.zst
 
             time aws s3 mv --only-show-errors s3://${BUCKET}/${PREFIX}/${OLD_FILENAME} s3://${BUCKET}/${PREFIX}/${NEW_FILENAME}
           done
+
+          # Update Neon artifact for the release (reuse already uploaded artifact)
+          for build_type in debug release; do
+            OLD_PREFIX=artifacts/${GITHUB_RUN_ID}
+            FILENAME=neon-${{ runner.os }}-${build_type}-artifact.tar.zst
+
+            S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${OLD_PREFIX} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true)
+            if [ -z "${S3_KEY}" ]; then
+              echo 2>&1 "Neither s3://${BUCKET}/${OLD_PREFIX}/${FILENAME} nor its version from previous attempts exist"
+              exit 1
+            fi
+
+            time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/${PREFIX}/${FILENAME}
+          done

.github/workflows/codestyle.yml

@@ -48,11 +48,11 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt update
-          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev
+          sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler
 
       - name: Install macOS postgres dependencies
         if: matrix.os == 'macos-latest'
-        run: brew install flex bison openssl
+        run: brew install flex bison openssl protobuf
 
       - name: Set pg 14 revision for caching
         id: pg_v14_rev
@@ -115,7 +115,7 @@ jobs:
         run: cargo build --locked --all --all-targets
 
   check-rust-dependencies:
-    runs-on: dev
+    runs-on: [ self-hosted, dev, x64 ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "vendor/postgres-v14"]
 	path = vendor/postgres-v14
 	url = https://github.com/neondatabase/postgres.git
-	branch = main
+	branch = REL_14_STABLE_neon
 [submodule "vendor/postgres-v15"]
 	path = vendor/postgres-v15
 	url = https://github.com/neondatabase/postgres.git

CODEOWNERS

@@ -0,0 +1,11 @@
+/compute_tools/ @neondatabase/control-plane
+/control_plane/ @neondatabase/compute @neondatabase/storage
+/libs/pageserver_api/ @neondatabase/compute @neondatabase/storage
+/libs/postgres_ffi/ @neondatabase/compute 
+/libs/remote_storage/ @neondatabase/storage 
+/libs/safekeeper_api/ @neondatabase/safekeepers  
+/pageserver/ @neondatabase/compute @neondatabase/storage 
+/pgxn/ @neondatabase/compute
+/proxy/ @neondatabase/control-plane 
+/safekeeper/ @neondatabase/safekeepers
+/vendor/ @neondatabase/compute

Cargo.lock

@@ -457,11 +457,26 @@ checksum = "6bf8832993da70a4c6d13c581f4463c2bdda27b9bf1c5498dc4365543abe6d6f"
 dependencies = [
  "atty",
  "bitflags",
+ "clap_derive",
  "clap_lex 0.3.0",
+ "once_cell",
  "strsim",
  "termcolor",
 ]
 
+[[package]]
+name = "clap_derive"
+version = "4.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c42f169caba89a7d512b5418b09864543eeb4d497416c917d7137863bd2076ad"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "clap_lex"
 version = "0.2.4"
@@ -586,6 +601,7 @@ dependencies = [
  "once_cell",
  "pageserver_api",
  "postgres",
+ "postgres_connection",
  "regex",
  "reqwest",
  "safekeeper_api",
@@ -1005,11 +1021,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fb8664f6ea68aba5503d42dd1be786b0f1bd9b7972e7f40208c83ef74db91bf"
 dependencies = [
  "http",
- "prost",
+ "prost 0.10.4",
  "tokio",
  "tokio-stream",
- "tonic",
- "tonic-build",
+ "tonic 0.7.2",
+ "tonic-build 0.7.2",
  "tower",
  "tower-service",
 ]
@@ -2140,9 +2156,11 @@ dependencies = [
  "num-traits",
  "once_cell",
  "pageserver_api",
+ "pin-project-lite",
  "postgres",
  "postgres-protocol",
  "postgres-types",
+ "postgres_connection",
  "postgres_ffi",
  "pprof",
  "pq_proto",
@@ -2390,6 +2408,19 @@ dependencies = [
  "postgres-protocol",
 ]
 
+[[package]]
+name = "postgres_connection"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "once_cell",
+ "postgres",
+ "tokio-postgres",
+ "url",
+ "workspace_hack",
+]
+
 [[package]]
 name = "postgres_ffi"
 version = "0.1.0"
@@ -2464,6 +2495,30 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.19"
@@ -2515,7 +2570,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e"
 dependencies = [
  "bytes",
- "prost-derive",
+ "prost-derive 0.10.1",
+]
+
+[[package]]
+name = "prost"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0841812012b2d4a6145fae9a6af1534873c32aa67fff26bd09f8fa42c83f95a"
+dependencies = [
+ "bytes",
+ "prost-derive 0.11.2",
 ]
 
 [[package]]
@@ -2533,13 +2598,35 @@ dependencies = [
  "log",
  "multimap",
  "petgraph",
- "prost",
- "prost-types",
+ "prost 0.10.4",
+ "prost-types 0.10.1",
  "regex",
  "tempfile",
  "which",
 ]
 
+[[package]]
+name = "prost-build"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d8b442418ea0822409d9e7d047cbf1e7e9e1760b172bf9982cf29d517c93511"
+dependencies = [
+ "bytes",
+ "heck",
+ "itertools",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph",
+ "prettyplease",
+ "prost 0.11.2",
+ "prost-types 0.11.2",
+ "regex",
+ "syn",
+ "tempfile",
+ "which",
+]
+
 [[package]]
 name = "prost-derive"
 version = "0.10.1"
@@ -2553,6 +2640,19 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "prost-derive"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "prost-types"
 version = "0.10.1"
@@ -2560,7 +2660,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68"
 dependencies = [
  "bytes",
- "prost",
+ "prost 0.10.4",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a"
+dependencies = [
+ "bytes",
+ "prost 0.11.2",
 ]
 
 [[package]]
@@ -3422,6 +3532,32 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
+[[package]]
+name = "storage_broker"
+version = "0.1.0"
+dependencies = [
+ "async-stream",
+ "bytes",
+ "clap 4.0.15",
+ "futures",
+ "futures-core",
+ "futures-util",
+ "git-version",
+ "humantime",
+ "hyper",
+ "metrics",
+ "once_cell",
+ "parking_lot 0.12.1",
+ "prost 0.11.2",
+ "tokio",
+ "tokio-stream",
+ "tonic 0.8.2",
+ "tonic-build 0.8.2",
+ "tracing",
+ "utils",
+ "workspace_hack",
+]
+
 [[package]]
 name = "str_stack"
 version = "0.1.0"
@@ -3822,8 +3958,40 @@ dependencies = [
  "hyper-timeout",
  "percent-encoding",
  "pin-project",
- "prost",
- "prost-derive",
+ "prost 0.10.4",
+ "prost-derive 0.10.1",
+ "tokio",
+ "tokio-stream",
+ "tokio-util",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+ "tracing-futures",
+]
+
+[[package]]
+name = "tonic"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55b9af819e54b8f33d453655bef9b9acc171568fb49523078d0cc4e7484200ec"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum",
+ "base64",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-timeout",
+ "percent-encoding",
+ "pin-project",
+ "prost 0.11.2",
+ "prost-derive 0.11.2",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -3842,7 +4010,20 @@ checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1"
 dependencies = [
  "prettyplease",
  "proc-macro2",
- "prost-build",
+ "prost-build 0.10.4",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tonic-build"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c6fd7c2581e36d63388a9e04c350c21beb7a8b059580b2e93993c526899ddc"
+dependencies = [
+ "prettyplease",
+ "proc-macro2",
+ "prost-build 0.11.2",
  "quote",
  "syn",
 ]
@@ -4401,7 +4582,8 @@ dependencies = [
  "num-bigint",
  "num-integer",
  "num-traits",
- "prost",
+ "prost 0.10.4",
+ "prost 0.11.2",
  "rand",
  "regex",
  "regex-syntax",

Cargo.toml

@@ -16,6 +16,7 @@ members = [
     "pageserver",
     "proxy",
     "safekeeper",
+    "storage_broker",
     "workspace_hack",
     "libs/*",
 ]
@@ -25,6 +26,10 @@ members = [
 # Besides, debug info should not affect the performance.
 debug = true
 
+# disable debug symbols for all packages except this one to decrease binaries size
+[profile.release.package."*"]
+debug = false
+
 [profile.release-line-debug]
 inherits = "release"
 debug = 1 # true = 2 = all symbols, 1 = line only

Dockerfile

@@ -44,7 +44,7 @@ COPY . .
 # Show build caching stats to check if it was used in the end.
 # Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, losing the compilation stats.
 RUN set -e \
-&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin draw_timeline_dir --bin safekeeper --bin proxy --locked --release \
+&& mold -run cargo build --bin pageserver --bin pageserver_binutils --bin draw_timeline_dir --bin safekeeper --bin storage_broker --bin proxy --locked --release \
     && cachepot -s
 
 # Build final image
@@ -67,6 +67,7 @@ COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver_binutils /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/draw_timeline_dir   /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper          /usr/local/bin
+COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_broker         /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy               /usr/local/bin
 
 COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/

Dockerfile.compute-node-v14

@@ -200,9 +200,6 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb
 # libreadline8 for psql
 # libossp-uuid16 for extension ossp-uuid
 # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS
-#
-# Lastly, link compute_ctl into zenith_ctl while we're at it,
-# so that we don't need to put this in another layer.
 RUN apt update &&  \
     apt install --no-install-recommends -y \
         libreadline8 \
@@ -211,8 +208,7 @@ RUN apt update &&  \
         libgdal28 \
         libproj19 \
         libprotobuf-c1 && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 USER postgres
 ENTRYPOINT ["/usr/local/bin/compute_ctl"]

Dockerfile.compute-node-v15

@@ -200,9 +200,6 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb
 # libreadline8 for psql
 # libossp-uuid16 for extension ossp-uuid
 # libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS
-#
-# Lastly, link compute_ctl into zenith_ctl while we're at it,
-# so that we don't need to put this in another layer.
 RUN apt update &&  \
     apt install --no-install-recommends -y \
         libreadline8 \
@@ -211,8 +208,7 @@ RUN apt update &&  \
         libgdal28 \
         libproj19 \
         libprotobuf-c1 && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    ln /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 USER postgres
 ENTRYPOINT ["/usr/local/bin/compute_ctl"]

Dockerfile.compute-node.legacy

@@ -1,88 +0,0 @@
-#
-# Legacy version of the Dockerfile for the compute node.
-# Used by e2e CI. Building Dockerfile.compute-node will take
-# unreasonable ammount of time without v2 runners.
-#
-# TODO: remove once cloud repo CI is moved to v2 runners.
-#
-
-
-# Allow specifiyng different compute-tools tag and image repo, so we are
-# able to use different images
-ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
-ARG IMAGE=compute-tools
-ARG TAG=latest
-
-#
-# Image with pre-built tools
-#
-FROM $REPOSITORY/$IMAGE:$TAG AS compute-deps
-# Only to get ready compute_ctl binary as deppendency
-
-#
-# Image with Postgres build deps
-#
-FROM debian:bullseye-slim AS build-deps
-
-RUN apt-get update && apt-get -yq install automake libtool build-essential bison flex libreadline-dev zlib1g-dev libxml2-dev \
-                                          libcurl4-openssl-dev libossp-uuid-dev
-
-#
-# Image with built Postgres
-#
-FROM build-deps AS pg-build
-
-# Add user postgres
-RUN adduser postgres
-RUN mkdir /pg && chown postgres:postgres /pg
-
-# Copy source files
-# version 14 is default for now
-COPY ./vendor/postgres-v14 /pg/
-COPY ./pgxn /pg/
-
-# Build and install Postgres locally
-RUN mkdir /pg/compute_build && cd /pg/compute_build && \
-    ../configure CFLAGS='-O2 -g3' --prefix=$(pwd)/postgres_bin --enable-debug --with-uuid=ossp && \
-    # Install main binaries and contribs
-    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \
-    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
-    # Install headers
-    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install
-
-# Install neon contrib
-RUN make MAKELEVEL=0 PG_CONFIG=/pg/compute_build/postgres_bin/bin/pg_config -j $(getconf _NPROCESSORS_ONLN) -C /pg/neon install
-
-USER postgres
-WORKDIR /pg
-
-#
-# Final compute node image to be exported
-#
-FROM debian:bullseye-slim
-
-# libreadline-dev is required to run psql
-RUN apt-get update && apt-get -yq install libreadline-dev libossp-uuid-dev
-
-# Add user postgres
-RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
-    echo "postgres:test_console_pass" | chpasswd && \
-    mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \
-    chown -R postgres:postgres /var/db/postgres && \
-    chmod 0750 /var/db/postgres/compute
-
-# Copy ready Postgres binaries
-COPY --from=pg-build /pg/compute_build/postgres_bin /usr/local
-
-# Copy binaries from compute-tools
-COPY --from=compute-deps /usr/local/bin/compute_ctl /usr/local/bin/compute_ctl
-
-# XXX: temporary symlink for compatibility with old control-plane
-RUN ln -s /usr/local/bin/compute_ctl /usr/local/bin/zenith_ctl
-
-# Add postgres shared objects to the search path
-RUN echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig
-
-USER postgres
-
-ENTRYPOINT ["/usr/local/bin/compute_ctl"]

Makefile

@@ -20,18 +20,18 @@ else
 	$(error Bad build type '$(BUILD_TYPE)', see Makefile for options)
 endif
 
-# Seccomp BPF is only available for Linux
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Linux)
+	# Seccomp BPF is only available for Linux
 	PG_CONFIGURE_OPTS += --with-libseccomp
-endif
-
-# macOS with brew-installed openssl requires explicit paths
-# It can be configured with OPENSSL_PREFIX variable
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Darwin)
-    OPENSSL_PREFIX ?= $(shell brew --prefix openssl@3)
-    PG_CONFIGURE_OPTS += --with-includes=$(OPENSSL_PREFIX)/include --with-libraries=$(OPENSSL_PREFIX)/lib
+else ifeq ($(UNAME_S),Darwin)
+	# macOS with brew-installed openssl requires explicit paths
+	# It can be configured with OPENSSL_PREFIX variable
+	OPENSSL_PREFIX ?= $(shell brew --prefix openssl@3)
+	PG_CONFIGURE_OPTS += --with-includes=$(OPENSSL_PREFIX)/include --with-libraries=$(OPENSSL_PREFIX)/lib
+	# macOS already has bison and flex in the system, but they are old and result in postgres-v14 target failure
+	# brew formulae are keg-only and not symlinked into HOMEBREW_PREFIX, force their usage
+	EXTRA_PATH_OVERRIDES += $(shell brew --prefix bison)/bin/:$(shell brew --prefix flex)/bin/:
 endif
 
 # Use -C option so that when PostgreSQL "make install" installs the
@@ -73,7 +73,8 @@ $(POSTGRES_INSTALL_DIR)/build/v14/config.status:
 	+@echo "Configuring Postgres v14 build"
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/v14
 	(cd $(POSTGRES_INSTALL_DIR)/build/v14 && \
-	$(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure CFLAGS='$(PG_CFLAGS)' \
+	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure \
+		CFLAGS='$(PG_CFLAGS)' \
 		$(PG_CONFIGURE_OPTS) \
 		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v14 > configure.log)
 
@@ -81,7 +82,8 @@ $(POSTGRES_INSTALL_DIR)/build/v15/config.status:
 	+@echo "Configuring Postgres v15 build"
 	mkdir -p $(POSTGRES_INSTALL_DIR)/build/v15
 	(cd $(POSTGRES_INSTALL_DIR)/build/v15 && \
-	$(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure CFLAGS='$(PG_CFLAGS)' \
+	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure \
+		CFLAGS='$(PG_CFLAGS)' \
 		$(PG_CONFIGURE_OPTS) \
 		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v15 > configure.log)
 
@@ -111,6 +113,8 @@ postgres-v14: postgres-v14-configure \
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14 MAKELEVEL=0 install
 	+@echo "Compiling libpq v14"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/interfaces/libpq install
+	+@echo "Compiling pg_prewarm v14"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_prewarm install
 	+@echo "Compiling pg_buffercache v14"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_buffercache install
 	+@echo "Compiling pageinspect v14"
@@ -123,6 +127,8 @@ postgres-v15: postgres-v15-configure \
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15 MAKELEVEL=0 install
 	+@echo "Compiling libpq v15"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/interfaces/libpq install
+	+@echo "Compiling pg_prewarm v15"
+	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_prewarm install
 	+@echo "Compiling pg_buffercache v15"
 	$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_buffercache install
 	+@echo "Compiling pageinspect v15"

README.md

@@ -35,12 +35,12 @@ Pageserver consists of:
 * On Ubuntu or Debian, this set of packages should be sufficient to build the code:
 ```bash
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
-libssl-dev clang pkg-config libpq-dev etcd cmake postgresql-client
+libssl-dev clang pkg-config libpq-dev etcd cmake postgresql-client protobuf-compiler
 ```
 * On Fedora, these packages are needed:
 ```bash
 dnf install flex bison readline-devel zlib-devel openssl-devel \
-  libseccomp-devel perl clang cmake etcd postgresql postgresql-contrib
+  libseccomp-devel perl clang cmake etcd postgresql postgresql-contrib protobuf-compiler
 ```
 
 2. [Install Rust](https://www.rust-lang.org/tools/install)
@@ -53,7 +53,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 1. Install XCode and dependencies
 ```
 xcode-select --install
-brew install protobuf etcd openssl
+brew install protobuf etcd openssl flex bison
 ```
 
 2. [Install Rust](https://www.rust-lang.org/tools/install)
@@ -125,24 +125,23 @@ Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (r
 # Create repository in .neon with proper paths to binaries and data
 # Later that would be responsibility of a package install script
 > ./target/debug/neon_local init
-Starting pageserver at '127.0.0.1:64000' in '.neon'
-
-Pageserver started
-Successfully initialized timeline 7dd0907914ac399ff3be45fb252bfdb7
-Stopping pageserver gracefully...done!
+Starting pageserver at '127.0.0.1:64000' in '.neon'.
+pageserver started, pid: 2545906
+Successfully initialized timeline de200bd42b49cc1814412c7e592dd6e9
+Stopped pageserver 1 process with pid 2545906
 
 # start pageserver and safekeeper
 > ./target/debug/neon_local start
-Starting etcd broker using /usr/bin/etcd
-Starting pageserver at '127.0.0.1:64000' in '.neon'
-
-Pageserver started
-Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1'
-Safekeeper started
+Starting etcd broker using "/usr/bin/etcd"
+etcd started, pid: 2545996
+Starting pageserver at '127.0.0.1:64000' in '.neon'.
+pageserver started, pid: 2546005
+Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1'.
+safekeeper 1 started, pid: 2546041
 
 # start postgres compute node
 > ./target/debug/neon_local pg start main
-Starting new postgres main on timeline de200bd42b49cc1814412c7e592dd6e9 ...
+Starting new postgres (v14) main on timeline de200bd42b49cc1814412c7e592dd6e9 ...
 Extracting base backup to create postgres instance: path=.neon/pgdatadirs/tenants/9ef87a5bf0d92544f6fafeeb3239695c/main port=55432
 Starting postgres node at 'host=127.0.0.1 port=55432 user=cloud_admin dbname=postgres'
 

cli-v2-story.md

@@ -1,188 +0,0 @@
-Create a new Zenith repository in the current directory:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli init
-    The files belonging to this database system will be owned by user "heikki".
-    This user must also own the server process.
-    
-    The database cluster will be initialized with locale "en_GB.UTF-8".
-    The default database encoding has accordingly been set to "UTF8".
-    The default text search configuration will be set to "english".
-    
-    Data page checksums are disabled.
-    
-    creating directory tmp ... ok
-    creating subdirectories ... ok
-    selecting dynamic shared memory implementation ... posix
-    selecting default max_connections ... 100
-    selecting default shared_buffers ... 128MB
-    selecting default time zone ... Europe/Helsinki
-    creating configuration files ... ok
-    running bootstrap script ... ok
-    performing post-bootstrap initialization ... ok
-    syncing data to disk ... ok
-    
-    initdb: warning: enabling "trust" authentication for local connections
-    You can change this by editing pg_hba.conf or using the option -A, or
-    --auth-local and --auth-host, the next time you run initdb.
-    new zenith repository was created in .zenith
-
-Initially, there is only one branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch
-      main
-
-Start a local Postgres instance on the branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start main
-    Creating data directory from snapshot at 0/15FFB08...
-    waiting for server to start....2021-04-13 09:27:43.919 EEST [984664] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
-    2021-04-13 09:27:43.920 EEST [984664] LOG:  listening on IPv6 address "::1", port 5432
-    2021-04-13 09:27:43.920 EEST [984664] LOG:  listening on IPv4 address "127.0.0.1", port 5432
-    2021-04-13 09:27:43.927 EEST [984664] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5432"
-    2021-04-13 09:27:43.939 EEST [984665] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
-    2021-04-13 09:27:43.939 EEST [984665] LOG:  creating missing WAL directory "pg_wal/archive_status"
-    2021-04-13 09:27:44.189 EEST [984665] LOG:  database system was not properly shut down; automatic recovery in progress
-    2021-04-13 09:27:44.195 EEST [984665] LOG:  invalid record length at 0/15FFB80: wanted 24, got 0
-    2021-04-13 09:27:44.195 EEST [984665] LOG:  redo is not required
-    2021-04-13 09:27:44.225 EEST [984664] LOG:  database system is ready to accept connections
-     done
-    server started
-
-Run some commands against it:
-
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "create table foo (t text);" 
-    CREATE TABLE
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "insert into foo values ('inserted on the main branch');" 
-    INSERT 0 1
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-    (1 row)
-
-Create a new branch called 'experimental'. We create it from the
-current end of the 'main' branch, but you could specify a different
-LSN as the start point instead.
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch experimental main
-    branching at end of WAL: 0/161F478
-    
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli branch 
-      experimental
-      main
-
-Start another Postgres instance off the 'experimental' branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start experimental -- -o -p5433
-    Creating data directory from snapshot at 0/15FFB08...
-    waiting for server to start....2021-04-13 09:28:41.874 EEST [984766] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
-    2021-04-13 09:28:41.875 EEST [984766] LOG:  listening on IPv6 address "::1", port 5433
-    2021-04-13 09:28:41.875 EEST [984766] LOG:  listening on IPv4 address "127.0.0.1", port 5433
-    2021-04-13 09:28:41.883 EEST [984766] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5433"
-    2021-04-13 09:28:41.896 EEST [984767] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
-    2021-04-13 09:28:42.265 EEST [984767] LOG:  database system was not properly shut down; automatic recovery in progress
-    2021-04-13 09:28:42.269 EEST [984767] LOG:  redo starts at 0/15FFB80
-    2021-04-13 09:28:42.272 EEST [984767] LOG:  invalid record length at 0/161F4B0: wanted 24, got 0
-    2021-04-13 09:28:42.272 EEST [984767] LOG:  redo done at 0/161F478 system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.00 s
-    2021-04-13 09:28:42.321 EEST [984766] LOG:  database system is ready to accept connections
-     done
-    server started
-
-Insert some a row on the 'experimental' branch:
-
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-    (1 row)
-    
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "insert into foo values ('inserted on experimental')" 
-    INSERT 0 1
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-     inserted on experimental
-    (2 rows)
-    
-See that the other Postgres instance is still running on 'main' branch on port 5432:
-
-
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5432 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-    (1 row)
-
-
-
-
-Everything is stored in the .zenith directory:
-
-    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/
-    total 12
-    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:28 datadirs
-    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:27 refs
-    drwxr-xr-x 4 heikki heikki 4096 Apr 13 09:28 timelines
-
-The 'datadirs' directory contains the datadirs of the running instances:
-
-    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/datadirs/
-    total 8
-    drwx------ 18 heikki heikki 4096 Apr 13 09:27 3c0c634c1674079b2c6d4edf7c91523e
-    drwx------ 18 heikki heikki 4096 Apr 13 09:28 697e3c103d4b1763cd6e82e4ff361d76
-    ~/git-sandbox/zenith (cli-v2)$ ls -l .zenith/datadirs/3c0c634c1674079b2c6d4edf7c91523e/
-    total 124
-    drwxr-xr-x 5 heikki heikki  4096 Apr 13 09:27 base
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 global
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_commit_ts
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_dynshmem
-    -rw------- 1 heikki heikki  4760 Apr 13 09:27 pg_hba.conf
-    -rw------- 1 heikki heikki  1636 Apr 13 09:27 pg_ident.conf
-    drwxr-xr-x 4 heikki heikki  4096 Apr 13 09:32 pg_logical
-    drwxr-xr-x 4 heikki heikki  4096 Apr 13 09:27 pg_multixact
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_notify
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_replslot
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_serial
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_snapshots
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_stat
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:34 pg_stat_tmp
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_subtrans
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_tblspc
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_twophase
-    -rw------- 1 heikki heikki     3 Apr 13 09:27 PG_VERSION
-    lrwxrwxrwx 1 heikki heikki    52 Apr 13 09:27 pg_wal -> ../../timelines/3c0c634c1674079b2c6d4edf7c91523e/wal
-    drwxr-xr-x 2 heikki heikki  4096 Apr 13 09:27 pg_xact
-    -rw------- 1 heikki heikki    88 Apr 13 09:27 postgresql.auto.conf
-    -rw------- 1 heikki heikki 28688 Apr 13 09:27 postgresql.conf
-    -rw------- 1 heikki heikki    96 Apr 13 09:27 postmaster.opts
-    -rw------- 1 heikki heikki   149 Apr 13 09:27 postmaster.pid
-
-Note how 'pg_wal' is just a symlink to the 'timelines' directory. The
-datadir is ephemeral, you can delete it at any time, and it can be reconstructed
-from the snapshots and WAL stored in the 'timelines' directory. So if you push/pull
-the repository, the 'datadirs' are not included. (They are like git working trees)
-
-    ~/git-sandbox/zenith (cli-v2)$ killall -9 postgres
-    ~/git-sandbox/zenith (cli-v2)$ rm -rf .zenith/datadirs/*
-    ~/git-sandbox/zenith (cli-v2)$ ./target/debug/cli start experimental -- -o -p5433
-    Creating data directory from snapshot at 0/15FFB08...
-    waiting for server to start....2021-04-13 09:37:05.476 EEST [985340] LOG:  starting PostgreSQL 14devel on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
-    2021-04-13 09:37:05.477 EEST [985340] LOG:  listening on IPv6 address "::1", port 5433
-    2021-04-13 09:37:05.477 EEST [985340] LOG:  listening on IPv4 address "127.0.0.1", port 5433
-    2021-04-13 09:37:05.487 EEST [985340] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5433"
-    2021-04-13 09:37:05.498 EEST [985341] LOG:  database system was interrupted; last known up at 2021-04-13 09:27:33 EEST
-    2021-04-13 09:37:05.808 EEST [985341] LOG:  database system was not properly shut down; automatic recovery in progress
-    2021-04-13 09:37:05.813 EEST [985341] LOG:  redo starts at 0/15FFB80
-    2021-04-13 09:37:05.815 EEST [985341] LOG:  invalid record length at 0/161F770: wanted 24, got 0
-    2021-04-13 09:37:05.815 EEST [985341] LOG:  redo done at 0/161F738 system usage: CPU: user: 0.00 s, system: 0.00 s, elapsed: 0.00 s
-    2021-04-13 09:37:05.866 EEST [985340] LOG:  database system is ready to accept connections
-     done
-    server started
-    ~/git-sandbox/zenith (cli-v2)$ psql postgres -p5433 -c "select * from foo" 
-                  t              
-    -----------------------------
-     inserted on the main branch
-     inserted on experimental
-    (2 rows)
-

compute_tools/src/pg_helpers.rs

@@ -65,7 +65,7 @@ impl GenericOption {
             let name = match self.name.as_str() {
                 "safekeepers" => "neon.safekeepers",
                 "wal_acceptor_reconnect" => "neon.safekeeper_reconnect_timeout",
-                "wal_acceptor_connect_timeout" => "neon.safekeeper_connect_timeout",
+                "wal_acceptor_connection_timeout" => "neon.safekeeper_connection_timeout",
                 it => it,
             };
 

control_plane/Cargo.toml

@@ -23,6 +23,7 @@ url = "2.2.2"
 # Note: Do not directly depend on pageserver or safekeeper; use pageserver_api or safekeeper_api
 # instead, so that recompile times are better.
 pageserver_api = { path = "../libs/pageserver_api" }
+postgres_connection = { path = "../libs/postgres_connection" }
 safekeeper_api = { path = "../libs/safekeeper_api" }
 utils = { path = "../libs/utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }

control_plane/src/background_process.rs

@@ -26,8 +26,18 @@ use nix::unistd::Pid;
 
 use utils::lock_file;
 
-const RETRIES: u32 = 15;
-const RETRY_TIMEOUT_MILLIS: u64 = 500;
+// These constants control the loop used to poll for process start / stop.
+//
+// The loop waits for at most 10 seconds, polling every 100 ms.
+// Once a second, it prints a dot ("."), to give the user an indication that
+// it's waiting. If the process hasn't started/stopped after 5 seconds,
+// it prints a notice that it's taking long, but keeps waiting.
+//
+const RETRY_UNTIL_SECS: u64 = 10;
+const RETRIES: u64 = (RETRY_UNTIL_SECS * 1000) / RETRY_INTERVAL_MILLIS;
+const RETRY_INTERVAL_MILLIS: u64 = 100;
+const DOT_EVERY_RETRIES: u64 = 10;
+const NOTICE_AFTER_RETRIES: u64 = 50;
 
 /// Argument to `start_process`, to indicate whether it should create pidfile or if the process creates
 /// it itself.
@@ -39,11 +49,16 @@ pub enum InitialPidFile<'t> {
 }
 
 /// Start a background child process using the parameters given.
-pub fn start_process<F, S: AsRef<OsStr>>(
+pub fn start_process<
+    F,
+    S: AsRef<OsStr>,
+    EI: IntoIterator<Item = (String, String)>, // Not generic AsRef<OsStr>, otherwise empty `envs` prevents type inference
+>(
     process_name: &str,
     datadir: &Path,
     command: &Path,
     args: &[S],
+    envs: EI,
     initial_pid_file: InitialPidFile,
     process_status_check: F,
 ) -> anyhow::Result<Child>
@@ -69,6 +84,7 @@ where
         .stderr(same_file_for_stderr)
         .args(args);
     let filled_cmd = fill_aws_secrets_vars(fill_rust_env_vars(background_command));
+    filled_cmd.envs(envs);
 
     let mut spawned_process = filled_cmd.spawn().with_context(|| {
         format!("Could not spawn {process_name}, see console output and log files for details.")
@@ -107,16 +123,16 @@ where
                 return Ok(spawned_process);
             }
             Ok(false) => {
-                if retries < 5 {
+                if retries == NOTICE_AFTER_RETRIES {
+                    // The process is taking a long time to start up. Keep waiting, but
+                    // print a message
+                    print!("\n{process_name} has not started yet, continuing to wait");
+                }
+                if retries % DOT_EVERY_RETRIES == 0 {
                     print!(".");
                     io::stdout().flush().unwrap();
-                } else {
-                    if retries == 5 {
-                        println!() // put a line break after dots for second message
-                    }
-                    println!("{process_name} has not started yet, retrying ({retries})...");
                 }
-                thread::sleep(Duration::from_millis(RETRY_TIMEOUT_MILLIS));
+                thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS));
             }
             Err(e) => {
                 println!("{process_name} failed to start: {e:#}");
@@ -127,7 +143,8 @@ where
             }
         }
     }
-    anyhow::bail!("{process_name} could not start in {RETRIES} attempts");
+    println!();
+    anyhow::bail!("{process_name} did not start in {RETRY_UNTIL_SECS} seconds");
 }
 
 /// Stops the process, using the pid file given. Returns Ok also if the process is already not running.
@@ -158,7 +175,7 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any
     }
 
     // Wait until process is gone
-    for _ in 0..RETRIES {
+    for retries in 0..RETRIES {
         match process_has_stopped(pid) {
             Ok(true) => {
                 println!("\n{process_name} stopped");
@@ -170,9 +187,16 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any
                 return Ok(());
             }
             Ok(false) => {
-                print!(".");
-                io::stdout().flush().unwrap();
-                thread::sleep(Duration::from_secs(1))
+                if retries == NOTICE_AFTER_RETRIES {
+                    // The process is taking a long time to start up. Keep waiting, but
+                    // print a message
+                    print!("\n{process_name} has not stopped yet, continuing to wait");
+                }
+                if retries % DOT_EVERY_RETRIES == 0 {
+                    print!(".");
+                    io::stdout().flush().unwrap();
+                }
+                thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS));
             }
             Err(e) => {
                 println!("{process_name} with pid {pid} failed to stop: {e:#}");
@@ -180,24 +204,28 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any
             }
         }
     }
-
-    anyhow::bail!("{process_name} with pid {pid} failed to stop in {RETRIES} attempts");
+    println!();
+    anyhow::bail!("{process_name} with pid {pid} did not stop in {RETRY_UNTIL_SECS} seconds");
 }
 
 fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
-    let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", "1");
+    // If RUST_BACKTRACE is set, pass it through. But if it's not set, default
+    // to RUST_BACKTRACE=1.
+    let backtrace_setting = std::env::var_os("RUST_BACKTRACE");
+    let backtrace_setting = backtrace_setting
+        .as_deref()
+        .unwrap_or_else(|| OsStr::new("1"));
 
-    let var = "LLVM_PROFILE_FILE";
-    if let Some(val) = std::env::var_os(var) {
-        filled_cmd = filled_cmd.env(var, val);
+    let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", backtrace_setting);
+
+    // Pass through these environment variables to the command
+    for var in ["LLVM_PROFILE_FILE", "FAILPOINTS", "RUST_LOG"] {
+        if let Some(val) = std::env::var_os(var) {
+            filled_cmd = filled_cmd.env(var, val);
+        }
     }
 
-    const RUST_LOG_KEY: &str = "RUST_LOG";
-    if let Ok(rust_log_value) = std::env::var(RUST_LOG_KEY) {
-        filled_cmd.env(RUST_LOG_KEY, rust_log_value)
-    } else {
-        filled_cmd
-    }
+    filled_cmd
 }
 
 fn fill_aws_secrets_vars(mut cmd: &mut Command) -> &mut Command {

control_plane/src/compute.rs

@@ -322,6 +322,9 @@ impl PostgresNode {
         conf.append("shared_preload_libraries", "neon");
         conf.append_line("");
         conf.append("neon.pageserver_connstring", &pageserver_connstr);
+        if let AuthType::NeonJWT = auth_type {
+            conf.append("neon.safekeeper_token_env", "$ZENITH_AUTH_TOKEN");
+        }
         conf.append("neon.tenant_id", &self.tenant_id.to_string());
         conf.append("neon.timeline_id", &self.timeline_id.to_string());
         if let Some(lsn) = self.lsn {
@@ -343,7 +346,7 @@ impl PostgresNode {
         //   To be able to restore database in case of pageserver node crash, safekeeper should not
         //   remove WAL beyond this point. Too large lag can cause space exhaustion in safekeepers
         //   (if they are not able to upload WAL to S3).
-        conf.append("max_replication_write_lag", "500MB");
+        conf.append("max_replication_write_lag", "15MB");
         conf.append("max_replication_flush_lag", "10GB");
 
         if !self.env.safekeepers.is_empty() {

control_plane/src/connection.rs

@@ -1,57 +0,0 @@
-use url::Url;
-
-#[derive(Debug)]
-pub struct PgConnectionConfig {
-    url: Url,
-}
-
-impl PgConnectionConfig {
-    pub fn host(&self) -> &str {
-        self.url.host_str().expect("BUG: no host")
-    }
-
-    pub fn port(&self) -> u16 {
-        self.url.port().expect("BUG: no port")
-    }
-
-    /// Return a `<host>:<port>` string.
-    pub fn raw_address(&self) -> String {
-        format!("{}:{}", self.host(), self.port())
-    }
-
-    /// Connect using postgres protocol with TLS disabled.
-    pub fn connect_no_tls(&self) -> Result<postgres::Client, postgres::Error> {
-        postgres::Client::connect(self.url.as_str(), postgres::NoTls)
-    }
-}
-
-impl std::str::FromStr for PgConnectionConfig {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let mut url: Url = s.parse()?;
-
-        match url.scheme() {
-            "postgres" | "postgresql" => {}
-            other => anyhow::bail!("invalid scheme: {other}"),
-        }
-
-        // It's not a valid connection url if host is unavailable.
-        if url.host().is_none() {
-            anyhow::bail!(url::ParseError::EmptyHost);
-        }
-
-        // E.g. `postgres:bar`.
-        if url.cannot_be_a_base() {
-            anyhow::bail!("URL cannot be a base");
-        }
-
-        // Set the default PG port if it's missing.
-        if url.port().is_none() {
-            url.set_port(Some(5432))
-                .expect("BUG: couldn't set the default port");
-        }
-
-        Ok(Self { url })
-    }
-}

control_plane/src/etcd.rs

@@ -6,7 +6,7 @@ use crate::{background_process, local_env};
 
 pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
     let etcd_broker = &env.etcd_broker;
-    println!(
+    print!(
         "Starting etcd broker using {:?}",
         etcd_broker.etcd_binary_path
     );
@@ -39,6 +39,7 @@ pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> {
         &etcd_data_dir,
         &etcd_broker.etcd_binary_path,
         &args,
+        [],
         background_process::InitialPidFile::Create(&pid_file_path),
         || {
             for broker_endpoint in &etcd_broker.broker_endpoints {

control_plane/src/lib.rs

@@ -9,7 +9,6 @@
 
 mod background_process;
 pub mod compute;
-pub mod connection;
 pub mod etcd;
 pub mod local_env;
 pub mod pageserver;

control_plane/src/pageserver.rs

@@ -6,14 +6,15 @@ use std::path::{Path, PathBuf};
 use std::process::Child;
 use std::{io, result};
 
-use crate::connection::PgConnectionConfig;
 use anyhow::{bail, Context};
 use pageserver_api::models::{
     TenantConfigRequest, TenantCreateRequest, TenantInfo, TimelineCreateRequest, TimelineInfo,
 };
+use postgres_connection::{parse_host_port, PgConnectionConfig};
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
+use utils::auth::{Claims, Scope};
 use utils::{
     http::error::HttpErrorBody,
     id::{TenantId, TimelineId},
@@ -77,30 +78,24 @@ pub struct PageServerNode {
 
 impl PageServerNode {
     pub fn from_env(env: &LocalEnv) -> PageServerNode {
+        let (host, port) = parse_host_port(&env.pageserver.listen_pg_addr)
+            .expect("Unable to parse listen_pg_addr");
+        let port = port.unwrap_or(5432);
         let password = if env.pageserver.auth_type == AuthType::NeonJWT {
-            &env.pageserver.auth_token
+            Some(env.pageserver.auth_token.clone())
         } else {
-            ""
+            None
         };
 
         Self {
-            pg_connection_config: Self::pageserver_connection_config(
-                password,
-                &env.pageserver.listen_pg_addr,
-            ),
+            pg_connection_config: PgConnectionConfig::new_host_port(host, port)
+                .set_password(password),
             env: env.clone(),
             http_client: Client::new(),
             http_base_url: format!("http://{}/v1", env.pageserver.listen_http_addr),
         }
     }
 
-    /// Construct libpq connection string for connecting to the pageserver.
-    fn pageserver_connection_config(password: &str, listen_addr: &str) -> PgConnectionConfig {
-        format!("postgresql://no_user:{password}@{listen_addr}/no_db")
-            .parse()
-            .unwrap()
-    }
-
     pub fn initialize(
         &self,
         create_tenant: Option<TenantId>,
@@ -237,7 +232,7 @@ impl PageServerNode {
         datadir: &Path,
         update_config: bool,
     ) -> anyhow::Result<Child> {
-        println!(
+        print!(
             "Starting pageserver at '{}' in '{}'",
             self.pg_connection_config.raw_address(),
             datadir.display()
@@ -259,11 +254,21 @@ impl PageServerNode {
             args.extend(["-c", config_override]);
         }
 
+        let envs = if self.env.pageserver.auth_type != AuthType::Trust {
+            // Generate a token to connect from the pageserver to a safekeeper
+            let token = self
+                .env
+                .generate_auth_token(&Claims::new(None, Scope::SafekeeperData))?;
+            vec![("ZENITH_AUTH_TOKEN".to_owned(), token)]
+        } else {
+            vec![]
+        };
         background_process::start_process(
             "pageserver",
             datadir,
             &self.env.pageserver_bin(),
             &args,
+            envs,
             background_process::InitialPidFile::Expect(&self.pid_file()),
             || match self.check_status() {
                 Ok(()) => Ok(true),
@@ -362,6 +367,11 @@ impl PageServerNode {
                 .map(|x| x.parse::<NonZeroU64>())
                 .transpose()
                 .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
+            trace_read_requests: settings
+                .remove("trace_read_requests")
+                .map(|x| x.parse::<bool>())
+                .transpose()
+                .context("Failed to parse 'trace_read_requests' as bool")?,
         };
         if !settings.is_empty() {
             bail!("Unrecognized tenant settings: {settings:?}")
@@ -424,6 +434,11 @@ impl PageServerNode {
                     .map(|x| x.parse::<NonZeroU64>())
                     .transpose()
                     .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?,
+                trace_read_requests: settings
+                    .get("trace_read_requests")
+                    .map(|x| x.parse::<bool>())
+                    .transpose()
+                    .context("Failed to parse 'trace_read_requests' as bool")?,
             })
             .send()?
             .error_from_body()?;

control_plane/src/safekeeper.rs

@@ -5,12 +5,12 @@ use std::sync::Arc;
 use std::{io, result};
 
 use anyhow::Context;
+use postgres_connection::PgConnectionConfig;
 use reqwest::blocking::{Client, RequestBuilder, Response};
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
 use utils::{http::error::HttpErrorBody, id::NodeId};
 
-use crate::connection::PgConnectionConfig;
 use crate::pageserver::PageServerNode;
 use crate::{
     background_process,
@@ -86,10 +86,7 @@ impl SafekeeperNode {
 
     /// Construct libpq connection string for connecting to this safekeeper.
     fn safekeeper_connection_config(port: u16) -> PgConnectionConfig {
-        // TODO safekeeper authentication not implemented yet
-        format!("postgresql://no_user@127.0.0.1:{port}/no_db")
-            .parse()
-            .unwrap()
+        PgConnectionConfig::new_host_port(url::Host::parse("127.0.0.1").unwrap(), port)
     }
 
     pub fn datadir_path_by_id(env: &LocalEnv, sk_id: NodeId) -> PathBuf {
@@ -169,6 +166,7 @@ impl SafekeeperNode {
             &datadir,
             &self.env.safekeeper_bin(),
             &args,
+            [],
             background_process::InitialPidFile::Expect(&self.pid_file()),
             || match self.check_status() {
                 Ok(()) => Ok(true),

docker-compose/compute_wrapper/Dockerfile

@@ -0,0 +1,13 @@
+ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com
+ARG COMPUTE_IMAGE=compute-node-v14
+ARG TAG=latest
+
+FROM $REPOSITORY/${COMPUTE_IMAGE}:$TAG
+
+USER root
+RUN apt-get update &&       \
+    apt-get install -y curl \
+                       jq   \
+                       netcat
+
+USER postgres

docker-compose/docker-compose.yml

@@ -2,6 +2,7 @@ version: '3'
 
 services:
   etcd:
+    restart: always
     image: quay.io/coreos/etcd:v3.5.4
     ports:
       - 2379:2379
@@ -9,7 +10,7 @@ services:
     environment:
       # This signifficantly speeds up etcd and we anyway don't data persistency there.
       ETCD_UNSAFE_NO_FSYNC: "1"
-    command: 
+    command:
       - "etcd"
       - "--auto-compaction-mode=revision"
       - "--auto-compaction-retention=1"
@@ -24,6 +25,7 @@ services:
       - "--quota-backend-bytes=134217728" # 128 MB
 
   minio:
+    restart: always
     image: quay.io/minio/minio:RELEASE.2022-10-20T00-55-09Z
     ports:
       - 9000:9000
@@ -41,7 +43,7 @@ services:
     entrypoint:
       - "/bin/sh"
       - "-c"
-    command: 
+    command:
       - "until (/usr/bin/mc alias set minio http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD) do
              echo 'Waiting to start minio...' && sleep 1;
          done;
@@ -51,7 +53,8 @@ services:
       - minio
 
   pageserver:
-    image: neondatabase/neon:${TAG:-latest}
+    restart: always
+    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
     environment:
       - BROKER_ENDPOINT='http://etcd:2379'
       - AWS_ACCESS_KEY_ID=minio
@@ -77,7 +80,8 @@ services:
       - minio_create_buckets
 
   safekeeper1:
-    image: neondatabase/neon:${TAG:-latest}
+    restart: always
+    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454
       - SAFEKEEPER_ID=1
@@ -106,7 +110,8 @@ services:
       - minio_create_buckets
 
   safekeeper2:
-    image: neondatabase/neon:${TAG:-latest}
+    restart: always
+    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454
       - SAFEKEEPER_ID=2
@@ -135,7 +140,8 @@ services:
       - minio_create_buckets
 
   safekeeper3:
-    image: neondatabase/neon:${TAG:-latest}
+    restart: always
+    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454
       - SAFEKEEPER_ID=3
@@ -164,18 +170,21 @@ services:
       - minio_create_buckets
 
   compute:
+    restart: always
     build:
-      context: ./image/compute
+      context: ./compute_wrapper/
       args:
-        - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-14}:${TAG:-latest}
+        - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-14}
+        - TAG=${TAG:-latest}
         - http_proxy=$http_proxy
         - https_proxy=$https_proxy
     environment:
       - PG_VERSION=${PG_VERSION:-14}
       #- RUST_BACKTRACE=1
+    # Mount the test files directly, for faster editing cycle.
     volumes:
-      - ./compute/var/db/postgres/specs/:/var/db/postgres/specs/
-      - ./compute/shell/:/shell/
+      - ./compute_wrapper/var/db/postgres/specs/:/var/db/postgres/specs/
+      - ./compute_wrapper/shell/:/shell/
     ports:
       - 55433:55433 # pg protocol handler
       - 3080:3080 # http endpoints

docker-compose/docker_compose_test.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# A basic test to ensure Docker images are built correctly.
+# Build a wrapper around the compute, start all services and runs a simple SQL query.
+# Repeats the process for all currenly supported Postgres versions.
+
+# Implicitly accepts `REPOSITORY` and `TAG` env vars that are passed into the compose file
+# Their defaults point at DockerHub `neondatabase/neon:latest` image.`,
+# to verify custom image builds (e.g pre-published ones).
+
+# XXX: Current does not work on M1 macs due to x86_64 Docker images compiled only, and no seccomp support in M1 Docker emulation layer.
+
+set -eux -o pipefail
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml
+
+COMPUTE_CONTAINER_NAME=docker-compose-compute-1
+SQL="CREATE TABLE t(key int primary key, value text); insert into t values(1,1); select * from t;"
+PSQL_OPTION="-h localhost -U cloud_admin -p 55433 -c '$SQL' postgres"
+
+cleanup() {
+    echo "show container information"
+    docker ps
+    docker compose -f $COMPOSE_FILE logs
+    echo "stop containers..."
+    docker compose -f $COMPOSE_FILE down
+}
+
+echo "clean up containers if exists"
+cleanup
+
+for pg_version in 14 15; do
+    echo "start containers (pg_version=$pg_version)."
+    PG_VERSION=$pg_version docker compose -f $COMPOSE_FILE up --build -d
+
+    echo "wait until the compute is ready. timeout after 60s. "
+    cnt=0
+    while sleep 1; do
+        # check timeout
+        cnt=`expr $cnt + 1`
+        if [ $cnt -gt 60 ]; then
+            echo "timeout before the compute is ready."
+            cleanup
+            exit 1
+        fi
+
+        # check if the compute is ready
+        set +o pipefail
+        result=`docker compose -f $COMPOSE_FILE logs "compute_is_ready" | grep "accepting connections" | wc -l`
+        set -o pipefail
+        if [ $result -eq 1 ]; then
+            echo "OK. The compute is ready to connect."
+            echo "execute simple queries."
+            docker exec $COMPUTE_CONTAINER_NAME /bin/bash -c "psql $PSQL_OPTION"
+            cleanup
+            break
+        fi
+    done
+done

docker-compose/image/compute/Dockerfile

@@ -1,10 +0,0 @@
-ARG COMPUTE_IMAGE=compute-node-v14:latest
-FROM neondatabase/${COMPUTE_IMAGE}
-
-USER root
-RUN apt-get update &&       \
-    apt-get install -y curl \
-                       jq   \
-                       netcat
-
-USER postgres

docs/SUMMARY.md

@@ -37,7 +37,7 @@
 
 - [Source view](./sourcetree.md)
   - [docker.md](./docker.md) — Docker images and building pipeline.
-  - [Error handling and logging]()
+  - [Error handling and logging](./error-handling.md)
   - [Testing]()
     - [Unit testing]()
     - [Integration testing]()

docs/authentication.md

@@ -1,30 +1,154 @@
 ## Authentication
 
 ### Overview
+We use JWT tokens in communication between almost all components (compute, pageserver, safekeeper, CLI) regardless of the protocol used (HTTP/PostgreSQL).
+Etcd currently has no authentication.
+Authentication is optional and is disabled by default for easier debugging.
+It is used in some tests, though.
+Note that we do not cover authentication with `pg.neon.tech` here.
 
-Current state of authentication includes usage of JWT tokens in communication between compute and pageserver and between CLI and pageserver. JWT token is signed using RSA keys. CLI generates a key pair during call to `neon_local init`. Using following openssl commands:
+For HTTP connections we use the Bearer authentication scheme.
+For PostgreSQL connections we expect the token to be passed as a password.
+There is a caveat for `psql`: it silently truncates passwords to 100 symbols, so to correctly pass JWT via `psql` you have to either use `PGPASSWORD` environment variable, or store password in `psql`'s config file.
+
+Current token scopes are described in `utils::auth::Scope`.
+There are no expiration or rotation schemes.
+
+_TODO_: some scopes allow both access to server management API and to the data.
+These probably should be split into multiple scopes.
+
+Tokens should not occur in logs.
+They may sometimes occur in configuration files, although this is discouraged
+because configs may be parsed and dumped into logs.
+
+#### Tokens generation and validation
+JWT tokens are signed using a private key.
+Compute/pageserver/safekeeper use the private key's public counterpart to validate JWT tokens.
+These components should not have access to the private key and may only get tokens from their configuration or external clients. 
+
+The key pair is generated once for an installation of compute/pageserver/safekeeper, e.g. by `neon_local init`.
+There is currently no way to rotate the key without bringing down all components.
+
+### CLI
+CLI generates a key pair during call to `neon_local init` with the following commands:
 
 ```bash
-openssl genrsa -out private_key.pem 2048
-openssl rsa -in private_key.pem -pubout -outform PEM -out public_key.pem
+openssl genrsa -out auth_private_key.pem 2048
+openssl rsa -in auth_private_key.pem -pubout -outform PEM -out auth_public_key.pem
 ```
 
-CLI also generates signed token and saves it in the config for later access to pageserver. Now authentication is optional. Pageserver has two variables in config: `auth_validation_public_key_path` and `auth_type`, so when auth type present and set to `NeonJWT` pageserver will require authentication for connections. Actual JWT is passed in password field of connection string. There is a caveat for psql, it silently truncates passwords to 100 symbols, so to correctly pass JWT via psql you have to either use PGPASSWORD environment variable, or store password in psql config file.
+Configuration files for all components point to `public_key.pem` for JWT validation.
+However, authentication is disabled by default.
+There is no way to automatically enable it everywhere, you have to configure each component individually.
 
-Currently there is no authentication between compute and safekeepers, because this communication layer is under heavy refactoring. After this refactoring support for authentication will be added there too. Now safekeeper supports "hardcoded" token passed via environment variable to be able to use callmemaybe command in pageserver.
+CLI also generates signed token (full access to Pageserver) and saves it in
+the CLI's `config` file under `pageserver.auth_token`.
+Note that pageserver's config does not have any similar parameter.
+CLI is the only component which accesses that token.
+Technically it could generate it from the private key on each run,
+but it does not do that for some reason (_TODO_).
 
-Compute uses token passed via environment variable to communicate to pageserver and in the future to the safekeeper too.
+### Compute
+#### Overview
+Compute is a per-timeline PostgreSQL instance, so it should not have
+any access to data of other tenants.
+All tokens used by a compute are restricted to a specific tenant.
+There is no auth isolation from other timelines of the same tenant,
+but a non-rogue client never accesses another timeline even by an accident:
+timeline IDs are random and hard to guess.
 
-JWT authentication now supports two scopes: tenant and pageserverapi. Tenant scope is intended for use in tenant related api calls, e.g. create_branch. Compute launched for particular tenant also uses this scope. Scope pageserver api is intended to be used by console to manage pageserver. For now we have only one management operation - create tenant.
+#### Incoming connections
+All incoming connections are from PostgreSQL clients.
+Their authentication is just plain PostgreSQL authentication and out of scope for this document.
 
-Examples for token generation in python:
+There is no administrative API except those provided by PostgreSQL.
+
+#### Outgoing connections
+Compute connects to Pageserver for getting pages.
+The connection string is configured by the `neon.pageserver_connstring` PostgreSQL GUC, e.g. `postgresql://no_user:$ZENITH_AUTH_TOKEN@localhost:15028`.
+The environment variable inside the connection string is substituted with
+the JWT token.
+
+Compute connects to Safekeepers to write and commit data.
+The token is the same for all safekeepers.
+It's stored in an environment variable, whose name is configured
+by the `neon.safekeeper_token_env` PostgreSQL GUC.
+If the GUC is unset, no token is passed.
+
+Note that both tokens can be (and typically are) the same;
+the scope is the tenant and the token is usually passed through the
+`$ZENITH_AUTH_TOKEN` environment variable.
+
+### Pageserver
+#### Overview
+Pageserver keeps track of multiple tenants, each having multiple timelines.
+For each timeline, it connects to the corresponding Safekeeper.
+Information about "corresponding Safekeeper" is published by Safekeepers
+in the Etcd, but they do not publish access tokens, otherwise what is
+the point of authentication.
+
+Pageserver keeps a connection to some set of Safekeepers, which
+may or may not correspond to active Computes.
+Hence, we cannot obtain a per-timeline access token from a Compute.
+E.g. if the timeline's Compute terminates before all WAL is
+consumed by the Pageserver, the Pageserver continues consuming WAL.
+
+Pageserver replicas' authentication is the same as the main's.
+
+#### Incoming connections
+Pageserver listens for connections from computes.
+Each compute should present a token valid for the timeline's tenant.
+
+Pageserver also has HTTP API: some parts are per-tenant,
+some parts are server-wide, these are different scopes.
+
+The `auth_type` configuration variable in Pageserver's config may have
+either of three values:
+
+* `Trust` removes all authentication. The outdated `MD5` value does likewise
+* `NeonJWT` enables JWT validation.
+   Tokens are validated using the public key which lies in a PEM file
+   specified in the `auth_validation_public_key_path` config.
+
+#### Outgoing connections
+Pageserver makes a connection to a Safekeeper for each active timeline.
+As Pageserver may want to access any timeline it has on the disk,
+it is given a blanket JWT token to access any data on any Safekeeper.
+This token is passed through an environment variable called `ZENITH_AUTH_TOKEN`
+(non-configurable as of writing this text).
+
+A better way _may be_ to store JWT token for each timeline next to it,
+but may be not.
+
+### Safekeeper
+#### Overview
+Safekeeper keeps track of multiple tenants, each having multiple timelines.
+
+#### Incoming connections
+Safekeeper accepts connections from Compute/Pageserver, each
+connection corresponds to a specific timeline and requires
+a corresponding JWT token.
+
+Safekeeper also has HTTP API: some parts are per-tenant,
+some parts are server-wide, these are different scopes.
+
+The `auth-validation-public-key-path` command line options controls
+the authentication mode:
+
+* If the option is missing, there is no authentication or JWT token validation.
+* If the option is present, it should be a path to the public key PEM file used for JWT token validation.
+
+#### Outgoing connections
+No connections are initiated by a Safekeeper.
+
+### In the source code
+Tests do not use authentication by default.
+If you need it, you can enable it by configuring the test's environment:
 
 ```python
-# generate pageserverapi token
-management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algorithm="RS256")
-
-# generate tenant token
-tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256")
+neon_env_builder.auth_enabled = True
 ```
 
-Utility functions to work with jwts in rust are located in libs/utils/src/auth.rs
+You will have to generate tokens if you want to access components inside the test directly,
+use `AuthKeys.generate_*_token` methods for that.
+If you create a new scope, please create a new method to prevent mistypes in scope's name.

docs/error-handling.md

@@ -0,0 +1,198 @@
+# Error handling and logging
+
+## Logging errors
+
+The principle is that errors are logged when they are handled. If you
+just propagate an error to the caller in a function, you don't need to
+log it; the caller will. But if you consume an error in a function,
+you *must* log it (if it needs to be logged at all).
+
+For example:
+
+```rust
+fn read_motd_file() -> std::io::Result<String> {
+    let mut f = File::open("/etc/motd")?;
+    let mut result = String::new();
+    f.read_to_string(&mut result)?;
+    result
+}
+```
+
+Opening or reading the file could fail, but there is no need to log
+the error here. The function merely propagates the error to the
+caller, and it is up to the caller to log the error or propagate it
+further, if the failure is not expected. But if, for example, it is
+normal that the "/etc/motd" file doesn't exist, the caller can choose
+to silently ignore the error, or log it as an INFO or DEBUG level
+message:
+
+```rust
+fn get_message_of_the_day() -> String {
+    // Get the motd from /etc/motd, or return the default proverb
+    match read_motd_file() {
+        Ok(motd) => motd,
+        Err(err)  => {
+            // It's normal that /etc/motd doesn't exist, but if we fail to
+            // read it for some other reason, that's unexpected. The message
+            // of the day isn't very important though, so we just WARN and
+            // continue with the default in any case.
+            if err.kind() != std::io::ErrorKind::NotFound {
+                 tracing::warn!("could not read \"/etc/motd\": {err:?}");
+            }
+            "An old error is always more popular than a new truth. - German proverb"
+        }
+    }
+}
+```
+
+## Error types
+
+We use the `anyhow` crate widely. It contains many convenient macros
+like `bail!` and `ensure!` to construct and return errors, and to
+propagate many kinds of low-level errors, wrapped in `anyhow::Error`.
+
+A downside of `anyhow::Error` is that the caller cannot distinguish
+between different error cases. Most errors are propagated all the way
+to the mgmt API handler function, or the main loop that handles a
+connection with the compute node, and they are all handled the same
+way: the error is logged and returned to the client as an HTTP or
+libpq error.
+
+But in some cases, we need to distinguish between errors and handle
+them differently. For example, attaching a tenant to the pageserver
+could fail either because the tenant has already been attached, or
+because we could not load its metadata from cloud storage. The first
+case is more or less expected. The console sends the Attach request to
+the pageserver, and the pageserver completes the operation, but the
+network connection might be lost before the console receives the
+response. The console will retry the operation in that case, but the
+tenant has already been attached. It is important that the pagserver
+responds with the HTTP 403 Already Exists error in that case, rather
+than a generic HTTP 500 Internal Server Error.
+
+If you need to distinguish between different kinds of errors, create a
+new `Error` type. The `thiserror` crate is useful for that. But in
+most cases `anyhow::Error` is good enough.
+
+## Panics
+
+Depending on where a panic happens, it can cause the whole pageserver
+or safekeeper to restart, or just a single tenant. In either case,
+that is pretty bad and causes an outage. Avoid panics. Never use
+`unwrap()` or other calls that might panic, to verify inputs from the
+network or from disk.
+
+It is acceptable to use functions that might panic, like `unwrap()`, if
+it is obvious that it cannot panic. For example, if you have just
+checked that a variable is not None, it is OK to call `unwrap()` on it,
+but it is still preferable to use `expect("reason")` instead to explain
+why the function cannot fail.
+
+`assert!` and `panic!` are reserved for checking clear invariants and
+very obvious "can't happen" cases. When in doubt, use anyhow `ensure!`
+or `bail!` instead.
+
+## Error levels
+
+`tracing::Level` doesn't provide very clear guidelines on what the
+different levels mean, or when to use which level. Here is how we use
+them:
+
+### Error
+
+Examples:
+- could not open file "foobar"
+- invalid tenant id
+
+Errors are not expected to happen during normal operation. Incorrect
+inputs from client can cause ERRORs. For example, if a client tries to
+call a mgmt API that doesn't exist, or if a compute node sends passes
+an LSN that has already been garbage collected away.
+
+These should *not* happen during normal operations. "Normal
+operations" is not a very precise concept. But for example, disk
+errors are not expected to happen when the system is working, so those
+count as Errors. However, if a TCP connection to a compute node is
+lost, that is not considered an Error, because it doesn't affect the
+pageserver's or safekeeper's operation in any way, and happens fairly
+frequently when compute nodes are shut down, or are killed abruptly
+because of errors in the compute.
+
+**Errors are monitored, and always need human investigation to determine
+the cause.**
+
+Whether something should be logged at ERROR, WARNING or INFO level can
+depend on the callers and clients. For example, it might be unexpected
+and a sign of a serious issue if the console calls the
+"timeline_detail" mgmt API for a timeline that doesn't exist. ERROR
+would be appropriate in that case. But if the console routinely calls
+the API after deleting a timeline, to check if the deletion has
+completed, then it would be totally normal and an INFO or DEBUG level
+message would be more appropriate. If a message is logged as an ERROR,
+but it in fact happens frequently in production and never requires any
+action, it should probably be demoted to an INFO level message.
+
+### Warn
+
+Examples:
+- could not remove temporary file "foobar.temp"
+- unrecognized file "foobar" in timeline directory
+
+Warnings are similar to Errors, in that they should not happen
+when the system is operating normally. The difference between Error and
+Warning is that an Error means that the operation failed, whereas Warning
+means that something unexpected happened, but the operation continued anyway.
+For example, if deleting a file fails because the file already didn't exist,
+it should be logged as Warning.
+
+> **Note:** The python regression tests, under `test_regress`, check the
+> pageserver log after each test for any ERROR and WARN lines. If there are
+> any ERRORs or WARNs that have not been explicitly listed in the test as
+> allowed, the test is marked a failed. This is to catch unexpected errors
+> e.g. in background operations, that don't cause immediate misbehaviour in
+> the tested functionality.
+
+### Info
+
+Info level is used to log useful information when the system is
+operating normally. Info level is appropriate e.g. for logging state
+changes, background operations, and network connections.
+
+Examples:
+- "system is shutting down"
+- "tenant was created"
+- "retrying S3 upload"
+
+### Debug & Trace
+
+Debug and Trace level messages are not printed to the log in our normal
+production configuration, but could be enabled for a specific server or
+tenant, to aid debugging. (Although we don't actually have that
+capability as of this writing).
+
+## Context
+
+We use logging "spans" to hold context information about the current
+operation. Almost every operation happens on a particular tenant and
+timeline, so we enter a span with the "tenant_id" and "timeline_id"
+very early when processing an incoming API request, for example. All
+background operations should also run in a span containing at least
+those two fields, and any other parameters or information that might
+be useful when debugging an error that might happen when performing
+the operation.
+
+TODO: Spans are not captured in the Error when it is created, but when
+the error is logged. It would be more useful to capture them at Error
+creation. We should consider using `tracing_error::SpanTrace` to do
+that.
+
+## Error message style
+
+PostgreSQL has a style guide for writing error messages:
+
+https://www.postgresql.org/docs/current/error-style-guide.html
+
+Follow that guide when writing error messages in the PostgreSQL
+extension. We don't follow it strictly in the pageserver and
+safekeeper, but the advice in the PostgreSQL style guide is generally
+good, and you can't go wrong by following it.

docs/sourcetree.md

@@ -2,6 +2,11 @@
 
 Below you will find a brief overview of each subdir in the source tree in alphabetical order.
 
+`storage_broker`:
+
+Neon storage broker, providing messaging between safekeepers and pageservers.
+[storage_broker.md](./storage_broker.md)
+
 `/control_plane`:
 
 Local control plane.
@@ -83,6 +88,16 @@ A subject for future modularization.
 `/libs/metrics`:
 Helpers for exposing Prometheus metrics from the server.
 
+### Adding dependencies
+When you add a Cargo dependency, you should update hakari manifest by running commands below and committing the updated `Cargo.lock` and `workspace_hack/`. There may be no changes, that's fine.
+
+```bash
+cargo hakari generate
+cargo hakari manage-deps
+```
+
+If you don't have hakari installed (`error: no such subcommand: hakari`), install it by running `cargo install cargo-hakari`.
+
 ## Using Python
 Note that Debian/Ubuntu Python packages are stale, as it commonly happens,
 so manual installation of dependencies is not recommended.

docs/storage_broker.md

@@ -0,0 +1,27 @@
+# Storage broker
+
+Storage broker targets two issues:
+- Allowing safekeepers and pageservers learn which nodes also hold their
+  timelines, and timeline statuses there.
+- Avoiding O(n^2) connections between storage nodes while doing so.
+
+This is used
+- By pageservers to determine the most advanced and alive safekeeper to pull WAL from.
+- By safekeepers to synchronize on the timeline: advance
+  `remote_consistent_lsn`, `backup_lsn`, choose who offloads WAL to s3.
+
+Technically, it is a simple stateless pub-sub message broker based on tonic
+(grpc) making multiplexing easy. Since it is stateless, fault tolerance can be
+provided by k8s; there is no built in replication support, though it is not hard
+to add.
+
+Currently, the only message is `SafekeeperTimelineInfo`. Each safekeeper, for
+each active timeline, once in a while pushes timeline status to the broker.
+Other nodes subscribe and receive this info, using it per above.
+
+Broker serves /metrics on the same port as grpc service. 
+
+grpcurl can be used to check which values are currently being pushed:
+```
+grpcurl -proto broker/proto/broker.proto -d '{"all":{}}' -plaintext localhost:50051 storage_broker.BrokerService/SubscribeSafekeeperInfo
+```

libs/pageserver_api/src/models.rs

@@ -15,19 +15,37 @@ use bytes::{BufMut, Bytes, BytesMut};
 /// A state of a tenant in pageserver's memory.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub enum TenantState {
-    /// Tenant is fully operational, its background jobs might be running or not.
-    Active { background_jobs_running: bool },
-    /// A tenant is recognized by pageserver, but not yet ready to operate:
-    /// e.g. not present locally and being downloaded or being read into memory from the file system.
-    Paused,
-    /// A tenant is recognized by the pageserver, but no longer used for any operations, as failed to get activated.
+    // This tenant is being loaded from local disk
+    Loading,
+    // This tenant is being downloaded from cloud storage.
+    Attaching,
+    /// Tenant is fully operational
+    Active,
+    /// A tenant is recognized by pageserver, but it is being detached or the
+    /// system is being shut down.
+    Stopping,
+    /// A tenant is recognized by the pageserver, but can no longer be used for
+    /// any operations, because it failed to be activated.
     Broken,
 }
 
+impl TenantState {
+    pub fn has_in_progress_downloads(&self) -> bool {
+        match self {
+            Self::Loading => true,
+            Self::Attaching => true,
+            Self::Active => false,
+            Self::Stopping => false,
+            Self::Broken => false,
+        }
+    }
+}
+
 /// A state of a timeline in pageserver's memory.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub enum TimelineState {
-    /// Timeline is fully operational, its background jobs are running.
+    /// Timeline is fully operational. If the containing Tenant is Active, the timeline's
+    /// background jobs are running otherwise they will be launched when the tenant is activated.
     Active,
     /// A timeline is recognized by pageserver, but not yet ready to operate.
     /// The status indicates, that the timeline could eventually go back to Active automatically:
@@ -35,8 +53,9 @@ pub enum TimelineState {
     Suspended,
     /// A timeline is recognized by pageserver, but not yet ready to operate and not allowed to
     /// automatically become Active after certain events: only a management call can change this status.
-    Paused,
-    /// A timeline is recognized by the pageserver, but no longer used for any operations, as failed to get activated.
+    Stopping,
+    /// A timeline is recognized by the pageserver, but can no longer be used for
+    /// any operations, because it failed to be activated.
     Broken,
 }
 
@@ -73,6 +92,7 @@ pub struct TenantCreateRequest {
     pub walreceiver_connect_timeout: Option<String>,
     pub lagging_wal_timeout: Option<String>,
     pub max_lsn_wal_lag: Option<NonZeroU64>,
+    pub trace_read_requests: Option<bool>,
 }
 
 #[serde_as]
@@ -112,6 +132,7 @@ pub struct TenantConfigRequest {
     pub walreceiver_connect_timeout: Option<String>,
     pub lagging_wal_timeout: Option<String>,
     pub max_lsn_wal_lag: Option<NonZeroU64>,
+    pub trace_read_requests: Option<bool>,
 }
 
 impl TenantConfigRequest {
@@ -130,6 +151,7 @@ impl TenantConfigRequest {
             walreceiver_connect_timeout: None,
             lagging_wal_timeout: None,
             max_lsn_wal_lag: None,
+            trace_read_requests: None,
         }
     }
 }
@@ -165,6 +187,8 @@ pub struct TimelineInfo {
     pub latest_gc_cutoff_lsn: Lsn,
     #[serde_as(as = "DisplayFromStr")]
     pub disk_consistent_lsn: Lsn,
+    #[serde_as(as = "DisplayFromStr")]
+    pub remote_consistent_lsn: Lsn,
     pub current_logical_size: Option<u64>, // is None when timeline is Unloaded
     pub current_physical_size: Option<u64>, // is None when timeline is Unloaded
     pub current_logical_size_non_incremental: Option<u64>,
@@ -177,8 +201,6 @@ pub struct TimelineInfo {
     pub last_received_msg_ts: Option<u128>,
     pub pg_version: u32,
 
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    pub remote_consistent_lsn: Option<Lsn>,
     pub awaits_download: bool,
 
     pub state: TimelineState,

libs/postgres_connection/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "postgres_connection"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0"
+itertools = "0.10.3"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
+url = "2.2.2"
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }
+
+[dev-dependencies]
+once_cell = "1.13.0"

libs/postgres_connection/src/lib.rs

@@ -0,0 +1,253 @@
+use anyhow::{bail, Context};
+use itertools::Itertools;
+use std::borrow::Cow;
+use std::fmt;
+use url::Host;
+
+/// Parses a string of format either `host:port` or `host` into a corresponding pair.
+/// The `host` part should be a correct `url::Host`, while `port` (if present) should be
+/// a valid decimal u16 of digits only.
+pub fn parse_host_port<S: AsRef<str>>(host_port: S) -> Result<(Host, Option<u16>), anyhow::Error> {
+    let (host, port) = match host_port.as_ref().rsplit_once(':') {
+        Some((host, port)) => (
+            host,
+            // +80 is a valid u16, but not a valid port
+            if port.chars().all(|c| c.is_ascii_digit()) {
+                Some(port.parse::<u16>().context("Unable to parse port")?)
+            } else {
+                bail!("Port contains a non-ascii-digit")
+            },
+        ),
+        None => (host_port.as_ref(), None), // No colons, no port specified
+    };
+    let host = Host::parse(host).context("Unable to parse host")?;
+    Ok((host, port))
+}
+
+#[cfg(test)]
+mod tests_parse_host_port {
+    use crate::parse_host_port;
+    use url::Host;
+
+    #[test]
+    fn test_normal() {
+        let (host, port) = parse_host_port("hello:123").unwrap();
+        assert_eq!(host, Host::Domain("hello".to_owned()));
+        assert_eq!(port, Some(123));
+    }
+
+    #[test]
+    fn test_no_port() {
+        let (host, port) = parse_host_port("hello").unwrap();
+        assert_eq!(host, Host::Domain("hello".to_owned()));
+        assert_eq!(port, None);
+    }
+
+    #[test]
+    fn test_ipv6() {
+        let (host, port) = parse_host_port("[::1]:123").unwrap();
+        assert_eq!(host, Host::<String>::Ipv6(std::net::Ipv6Addr::LOCALHOST));
+        assert_eq!(port, Some(123));
+    }
+
+    #[test]
+    fn test_invalid_host() {
+        assert!(parse_host_port("hello world").is_err());
+    }
+
+    #[test]
+    fn test_invalid_port() {
+        assert!(parse_host_port("hello:+80").is_err());
+    }
+}
+
+#[derive(Clone)]
+pub struct PgConnectionConfig {
+    host: Host,
+    port: u16,
+    password: Option<String>,
+    options: Vec<String>,
+}
+
+/// A simplified PostgreSQL connection configuration. Supports only a subset of possible
+/// settings for simplicity. A password getter or `to_connection_string` methods are not
+/// added by design to avoid accidentally leaking password through logging, command line
+/// arguments to a child process, or likewise.
+impl PgConnectionConfig {
+    pub fn new_host_port(host: Host, port: u16) -> Self {
+        PgConnectionConfig {
+            host,
+            port,
+            password: None,
+            options: vec![],
+        }
+    }
+
+    pub fn host(&self) -> &Host {
+        &self.host
+    }
+
+    pub fn port(&self) -> u16 {
+        self.port
+    }
+
+    pub fn set_host(mut self, h: Host) -> Self {
+        self.host = h;
+        self
+    }
+
+    pub fn set_port(mut self, p: u16) -> Self {
+        self.port = p;
+        self
+    }
+
+    pub fn set_password(mut self, s: Option<String>) -> Self {
+        self.password = s;
+        self
+    }
+
+    pub fn extend_options<I: IntoIterator<Item = S>, S: Into<String>>(mut self, i: I) -> Self {
+        self.options.extend(i.into_iter().map(|s| s.into()));
+        self
+    }
+
+    /// Return a `<host>:<port>` string.
+    pub fn raw_address(&self) -> String {
+        format!("{}:{}", self.host(), self.port())
+    }
+
+    /// Build a client library-specific connection configuration.
+    /// Used for testing and when we need to add some obscure configuration
+    /// elements at the last moment.
+    pub fn to_tokio_postgres_config(&self) -> tokio_postgres::Config {
+        // Use `tokio_postgres::Config` instead of `postgres::Config` because
+        // the former supports more options to fiddle with later.
+        let mut config = tokio_postgres::Config::new();
+        config.host(&self.host().to_string()).port(self.port);
+        if let Some(password) = &self.password {
+            config.password(password);
+        }
+        if !self.options.is_empty() {
+            // These options are command-line options and should be escaped before being passed
+            // as an 'options' connection string parameter, see
+            // https://www.postgresql.org/docs/15/libpq-connect.html#LIBPQ-CONNECT-OPTIONS
+            //
+            // They will be space-separated, so each space inside an option should be escaped,
+            // and all backslashes should be escaped before that. Although we don't expect options
+            // with spaces at the moment, they're supported by PostgreSQL. Hence we support them
+            // in this typesafe interface.
+            //
+            // We use `Cow` to avoid allocations in the best case (no escaping). A fully imperative
+            // solution would require 1-2 allocations in the worst case as well, but it's harder to
+            // implement and this function is hardly a bottleneck. The function is only called around
+            // establishing a new connection.
+            #[allow(unstable_name_collisions)]
+            config.options(
+                &self
+                    .options
+                    .iter()
+                    .map(|s| {
+                        if s.contains(['\\', ' ']) {
+                            Cow::Owned(s.replace('\\', "\\\\").replace(' ', "\\ "))
+                        } else {
+                            Cow::Borrowed(s.as_str())
+                        }
+                    })
+                    .intersperse(Cow::Borrowed(" ")) // TODO: use impl from std once it's stabilized
+                    .collect::<String>(),
+            );
+        }
+        config
+    }
+
+    /// Connect using postgres protocol with TLS disabled.
+    pub fn connect_no_tls(&self) -> Result<postgres::Client, postgres::Error> {
+        postgres::Config::from(self.to_tokio_postgres_config()).connect(postgres::NoTls)
+    }
+}
+
+impl fmt::Debug for PgConnectionConfig {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // We want `password: Some(REDACTED-STRING)`, not `password: Some("REDACTED-STRING")`
+        // so even if the password is `REDACTED-STRING` (quite unlikely) there is no confusion.
+        // Hence `format_args!()`, it returns a "safe" string which is not escaped by `Debug`.
+        f.debug_struct("PgConnectionConfig")
+            .field("host", &self.host)
+            .field("port", &self.port)
+            .field(
+                "password",
+                &self
+                    .password
+                    .as_ref()
+                    .map(|_| format_args!("REDACTED-STRING")),
+            )
+            .finish()
+    }
+}
+
+#[cfg(test)]
+mod tests_pg_connection_config {
+    use crate::PgConnectionConfig;
+    use once_cell::sync::Lazy;
+    use url::Host;
+
+    static STUB_HOST: Lazy<Host> = Lazy::new(|| Host::Domain("stub.host.example".to_owned()));
+
+    #[test]
+    fn test_no_password() {
+        let cfg = PgConnectionConfig::new_host_port(STUB_HOST.clone(), 123);
+        assert_eq!(cfg.host(), &*STUB_HOST);
+        assert_eq!(cfg.port(), 123);
+        assert_eq!(cfg.raw_address(), "stub.host.example:123");
+        assert_eq!(
+            format!("{:?}", cfg),
+            "PgConnectionConfig { host: Domain(\"stub.host.example\"), port: 123, password: None }"
+        );
+    }
+
+    #[test]
+    fn test_ipv6() {
+        // May be a special case because hostname contains a colon.
+        let cfg = PgConnectionConfig::new_host_port(Host::parse("[::1]").unwrap(), 123);
+        assert_eq!(
+            cfg.host(),
+            &Host::<String>::Ipv6(std::net::Ipv6Addr::LOCALHOST)
+        );
+        assert_eq!(cfg.port(), 123);
+        assert_eq!(cfg.raw_address(), "[::1]:123");
+        assert_eq!(
+            format!("{:?}", cfg),
+            "PgConnectionConfig { host: Ipv6(::1), port: 123, password: None }"
+        );
+    }
+
+    #[test]
+    fn test_with_password() {
+        let cfg = PgConnectionConfig::new_host_port(STUB_HOST.clone(), 123)
+            .set_password(Some("password".to_owned()));
+        assert_eq!(cfg.host(), &*STUB_HOST);
+        assert_eq!(cfg.port(), 123);
+        assert_eq!(cfg.raw_address(), "stub.host.example:123");
+        assert_eq!(
+            format!("{:?}", cfg),
+            "PgConnectionConfig { host: Domain(\"stub.host.example\"), port: 123, password: Some(REDACTED-STRING) }"
+        );
+    }
+
+    #[test]
+    fn test_with_options() {
+        let cfg = PgConnectionConfig::new_host_port(STUB_HOST.clone(), 123).extend_options([
+            "hello",
+            "world",
+            "with space",
+            "and \\ backslashes",
+        ]);
+        assert_eq!(cfg.host(), &*STUB_HOST);
+        assert_eq!(cfg.port(), 123);
+        assert_eq!(cfg.raw_address(), "stub.host.example:123");
+        assert_eq!(
+            cfg.to_tokio_postgres_config().get_options(),
+            Some("hello world with\\ space and\\ \\\\\\ backslashes")
+        );
+    }
+}

libs/postgres_ffi/src/lib.rs

@@ -163,6 +163,27 @@ pub fn page_set_lsn(pg: &mut [u8], lsn: Lsn) {
     pg[4..8].copy_from_slice(&(lsn.0 as u32).to_le_bytes());
 }
 
+// This is port of function with the same name from freespace.c.
+// The only difference is that it does not have "level" parameter because XLogRecordPageWithFreeSpace
+// always call it with level=FSM_BOTTOM_LEVEL
+pub fn fsm_logical_to_physical(addr: BlockNumber) -> BlockNumber {
+    let mut leafno = addr;
+    const FSM_TREE_DEPTH: u32 = if pg_constants::SLOTS_PER_FSM_PAGE >= 1626 {
+        3
+    } else {
+        4
+    };
+
+    /* Count upper level nodes required to address the leaf page */
+    let mut pages: BlockNumber = 0;
+    for _l in 0..FSM_TREE_DEPTH {
+        pages += leafno + 1;
+        leafno /= pg_constants::SLOTS_PER_FSM_PAGE;
+    }
+    /* Turn the page count into 0-based block number */
+    pages - 1
+}
+
 pub mod waldecoder {
 
     use crate::{v14, v15};

libs/postgres_ffi/src/pg_constants.rs

@@ -197,6 +197,16 @@ pub const XLOG_CHECKPOINT_SHUTDOWN: u8 = 0x00;
 pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10;
 pub const XLP_LONG_HEADER: u16 = 0x0002;
 
+/* From fsm_internals.h */
+const FSM_NODES_PER_PAGE: usize = BLCKSZ as usize - SIZEOF_PAGE_HEADER_DATA - 4;
+const FSM_NON_LEAF_NODES_PER_PAGE: usize = BLCKSZ as usize / 2 - 1;
+const FSM_LEAF_NODES_PER_PAGE: usize = FSM_NODES_PER_PAGE - FSM_NON_LEAF_NODES_PER_PAGE;
+pub const SLOTS_PER_FSM_PAGE: u32 = FSM_LEAF_NODES_PER_PAGE as u32;
+
+/* From visibilitymap.c */
+pub const VM_HEAPBLOCKS_PER_PAGE: u32 =
+    (BLCKSZ as usize - SIZEOF_PAGE_HEADER_DATA) as u32 * (8 / 2); // MAPSIZE * (BITS_PER_BYTE / BITS_PER_HEAPBLOCK)
+
 // List of subdirectories inside pgdata.
 // Copied from src/bin/initdb/initdb.c
 pub const PGDATA_SUBDIRS: [&str; 22] = [

libs/tenant_size_model/src/lib.rs

@@ -33,8 +33,8 @@ pub struct Segment {
     /// Logical size before this state
     start_size: u64,
 
-    /// Logical size at this state
-    pub end_size: u64,
+    /// Logical size at this state. Can be None in the last Segment of a branch.
+    pub end_size: Option<u64>,
 
     /// Indices to [`Storage::segments`]
     ///
@@ -115,7 +115,7 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
             start_lsn: 0,
             end_lsn: 0,
             start_size: 0,
-            end_size: 0,
+            end_size: Some(0),
             children_after: Vec::new(),
         };
 
@@ -125,6 +125,39 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
         }
     }
 
+    /// Advances the branch with a new point, at given LSN.
+    pub fn insert_point<Q: ?Sized>(
+        &mut self,
+        branch: &Q,
+        op: Cow<'static, str>,
+        lsn: u64,
+        size: Option<u64>,
+    ) where
+        K: std::borrow::Borrow<Q>,
+        Q: std::hash::Hash + Eq,
+    {
+        let lastseg_id = *self.branches.get(branch).unwrap();
+        let newseg_id = self.segments.len();
+        let lastseg = &mut self.segments[lastseg_id];
+
+        assert!(lsn > lastseg.end_lsn);
+
+        let newseg = Segment {
+            op,
+            parent: Some(lastseg_id),
+            start_lsn: lastseg.end_lsn,
+            end_lsn: lsn,
+            start_size: lastseg.end_size.unwrap(),
+            end_size: size,
+            children_after: Vec::new(),
+            needed: false,
+        };
+        lastseg.children_after.push(newseg_id);
+
+        self.segments.push(newseg);
+        *self.branches.get_mut(branch).expect("read already") = newseg_id;
+    }
+
     /// Advances the branch with the named operation, by the relative LSN and logical size bytes.
     pub fn modify_branch<Q: ?Sized>(
         &mut self,
@@ -145,8 +178,8 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
             parent: Some(lastseg_id),
             start_lsn: lastseg.end_lsn,
             end_lsn: lastseg.end_lsn + lsn_bytes,
-            start_size: lastseg.end_size,
-            end_size: (lastseg.end_size as i64 + size_bytes) as u64,
+            start_size: lastseg.end_size.unwrap(),
+            end_size: Some((lastseg.end_size.unwrap() as i64 + size_bytes) as u64),
             children_after: Vec::new(),
             needed: false,
         };
@@ -321,7 +354,7 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
                 Some(SegmentSize {
                     seg_id,
                     method: SnapshotAfter,
-                    this_size: seg.end_size,
+                    this_size: seg.end_size.unwrap(),
                     children,
                 })
             } else {

libs/tenant_size_model/src/main.rs

@@ -174,7 +174,7 @@ fn graphviz_recurse(segments: &[Segment], node: &SegmentSize) {
     let seg_id = node.seg_id;
     let seg = segments.get(seg_id).unwrap();
     let lsn = seg.end_lsn;
-    let size = seg.end_size;
+    let size = seg.end_size.unwrap_or(0);
     let method = node.method;
 
     println!("  {{");
@@ -226,7 +226,7 @@ fn graphviz_recurse(segments: &[Segment], node: &SegmentSize) {
             print!(
                 " label=\"{} / {}\"",
                 next.end_lsn - seg.end_lsn,
-                (next.end_size as i128 - seg.end_size as i128)
+                (next.end_size.unwrap_or(0) as i128 - seg.end_size.unwrap_or(0) as i128)
             );
         } else {
             print!(" label=\"{}: {}\"", next.op, next.end_lsn - seg.end_lsn);

libs/utils/src/auth.rs

@@ -7,7 +7,7 @@ use serde;
 use std::fs;
 use std::path::Path;
 
-use anyhow::{bail, Result};
+use anyhow::Result;
 use jsonwebtoken::{
     decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation,
 };
@@ -21,8 +21,16 @@ const JWT_ALGORITHM: Algorithm = Algorithm::RS256;
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "lowercase")]
 pub enum Scope {
+    // Provides access to all data for a specific tenant (specified in `struct Claims` below)
+    // TODO: join these two?
     Tenant,
+    // Provides blanket access to all tenants on the pageserver plus pageserver-wide APIs.
+    // Should only be used e.g. for status check/tenant creation/list.
     PageServerApi,
+    // Provides blanket access to all data on the safekeeper plus safekeeper-wide APIs.
+    // Should only be used e.g. for status check.
+    // Currently also used for connection from any pageserver to any safekeeper.
+    SafekeeperData,
 }
 
 #[serde_as]
@@ -40,22 +48,6 @@ impl Claims {
     }
 }
 
-pub fn check_permission(claims: &Claims, tenant_id: Option<TenantId>) -> Result<()> {
-    match (&claims.scope, tenant_id) {
-        (Scope::Tenant, None) => {
-            bail!("Attempt to access management api with tenant scope. Permission denied")
-        }
-        (Scope::Tenant, Some(tenant_id)) => {
-            if claims.tenant_id.unwrap() != tenant_id {
-                bail!("Tenant id mismatch. Permission denied")
-            }
-            Ok(())
-        }
-        (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope
-        (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope
-    }
-}
-
 pub struct JwtAuth {
     decoding_key: DecodingKey,
     validation: Validation,

libs/utils/src/fs_ext.rs

@@ -0,0 +1,45 @@
+/// Extensions to `std::fs` types.
+use std::{fs, io, path::Path};
+
+pub trait PathExt {
+    /// Returns an error if `self` is not a directory.
+    fn is_empty_dir(&self) -> io::Result<bool>;
+}
+
+impl<P> PathExt for P
+where
+    P: AsRef<Path>,
+{
+    fn is_empty_dir(&self) -> io::Result<bool> {
+        Ok(fs::read_dir(self)?.into_iter().next().is_none())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::path::PathBuf;
+
+    #[test]
+    fn is_empty_dir() {
+        use super::PathExt;
+
+        let dir = tempfile::tempdir().unwrap();
+        let dir_path = dir.path();
+
+        // test positive case
+        assert!(
+            dir_path.is_empty_dir().expect("test failure"),
+            "new tempdir should be empty"
+        );
+
+        // invoke on a file to ensure it returns an error
+        let file_path: PathBuf = dir_path.join("testfile");
+        let f = std::fs::File::create(&file_path).unwrap();
+        drop(f);
+        assert!(file_path.is_empty_dir().is_err());
+
+        // do it again on a path, we know to be nonexistent
+        std::fs::remove_file(&file_path).unwrap();
+        assert!(file_path.is_empty_dir().is_err());
+    }
+}

libs/utils/src/http/endpoint.rs

@@ -1,6 +1,5 @@
-use crate::auth::{self, Claims, JwtAuth};
+use crate::auth::{Claims, JwtAuth};
 use crate::http::error;
-use crate::id::TenantId;
 use anyhow::anyhow;
 use hyper::header::AUTHORIZATION;
 use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server};
@@ -144,10 +143,14 @@ pub fn auth_middleware<B: hyper::body::HttpBody + Send + Sync + 'static>(
     })
 }
 
-pub fn check_permission(req: &Request<Body>, tenant_id: Option<TenantId>) -> Result<(), ApiError> {
+pub fn check_permission_with(
+    req: &Request<Body>,
+    check_permission: impl Fn(&Claims) -> Result<(), anyhow::Error>,
+) -> Result<(), ApiError> {
     match req.context::<Claims>() {
-        Some(claims) => Ok(auth::check_permission(&claims, tenant_id)
-            .map_err(|err| ApiError::Forbidden(err.to_string()))?),
+        Some(claims) => {
+            Ok(check_permission(&claims).map_err(|err| ApiError::Forbidden(err.to_string()))?)
+        }
         None => Ok(()), // claims is None because auth is disabled
     }
 }

libs/utils/src/id.rs

@@ -3,6 +3,13 @@ use std::{fmt, str::FromStr};
 use hex::FromHex;
 use rand::Rng;
 use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum IdError {
+    #[error("invalid id length {0}")]
+    SliceParseError(usize),
+}
 
 /// Neon ID is a 128-bit random ID.
 /// Used to represent various identifiers. Provides handy utility methods and impls.
@@ -22,6 +29,15 @@ impl Id {
         Id::from(arr)
     }
 
+    pub fn from_slice(src: &[u8]) -> Result<Id, IdError> {
+        if src.len() != 16 {
+            return Err(IdError::SliceParseError(src.len()));
+        }
+        let mut id_array = [0u8; 16];
+        id_array.copy_from_slice(src);
+        Ok(id_array.into())
+    }
+
     pub fn as_arr(&self) -> [u8; 16] {
         self.0
     }
@@ -100,6 +116,10 @@ macro_rules! id_newtype {
                 $t(Id::get_from_buf(buf))
             }
 
+            pub fn from_slice(src: &[u8]) -> Result<$t, IdError> {
+                Ok($t(Id::from_slice(src)?))
+            }
+
             pub fn as_arr(&self) -> [u8; 16] {
                 self.0.as_arr()
             }
@@ -204,6 +224,17 @@ pub struct TenantId(Id);
 
 id_newtype!(TenantId);
 
+/// Neon Connection Id identifies long-lived connections (for example a pagestream
+/// connection with the page_service). Is used for better logging and tracing
+///
+/// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look
+/// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`.
+/// See [`Id`] for alternative ways to serialize it.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
+pub struct ConnectionId(Id);
+
+id_newtype!(ConnectionId);
+
 // A pair uniquely identifying Neon instance.
 #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct TenantTimelineId {

libs/utils/src/lib.rs

@@ -48,6 +48,27 @@ pub mod nonblock;
 // Default signal handling
 pub mod signals;
 
+pub mod fs_ext;
+
+/// use with fail::cfg("$name", "return(2000)")
+#[macro_export]
+macro_rules! failpoint_sleep_millis_async {
+    ($name:literal) => {{
+        let should_sleep: Option<std::time::Duration> = (|| {
+            fail::fail_point!($name, |v: Option<_>| {
+                let millis = v.unwrap().parse::<u64>().unwrap();
+                Some(Duration::from_millis(millis))
+            });
+            None
+        })();
+        if let Some(d) = should_sleep {
+            tracing::info!("failpoint {:?}: sleeping for {:?}", $name, d);
+            tokio::time::sleep(d).await;
+            tracing::info!("failpoint {:?}: sleep done", $name);
+        }
+    }};
+}
+
 /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages
 ///
 /// we have several cases:

libs/utils/src/lsn.rs

@@ -138,7 +138,7 @@ impl FromStr for Lsn {
     ///
     /// If the input string is missing the '/' character, then use `Lsn::from_hex`
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let mut splitter = s.split('/');
+        let mut splitter = s.trim().split('/');
         if let (Some(left), Some(right), None) = (splitter.next(), splitter.next(), splitter.next())
         {
             let left_num = u32::from_str_radix(left, 16).map_err(|_| LsnParseError)?;
@@ -270,6 +270,11 @@ mod tests {
         );
         assert_eq!(Lsn::from_hex("0"), Ok(Lsn(0)));
         assert_eq!(Lsn::from_hex("F12345678AAAA5555"), Err(LsnParseError));
+
+        let expected_lsn = Lsn(0x3C490F8);
+        assert_eq!(" 0/3C490F8".parse(), Ok(expected_lsn));
+        assert_eq!("0/3C490F8 ".parse(), Ok(expected_lsn));
+        assert_eq!(" 0/3C490F8 ".parse(), Ok(expected_lsn));
     }
 
     #[test]

pageserver/Cargo.toml

@@ -35,6 +35,7 @@ itertools = "0.10.3"
 nix = "0.25"
 num-traits = "0.2.15"
 once_cell = "1.13.0"
+pin-project-lite = "0.2.7"
 postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
 postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
@@ -61,6 +62,7 @@ walkdir = "2.3.2"
 etcd_broker = { path = "../libs/etcd_broker" }
 metrics = { path = "../libs/metrics" }
 pageserver_api = { path = "../libs/pageserver_api" }
+postgres_connection = { path = "../libs/postgres_connection" }
 postgres_ffi = { path = "../libs/postgres_ffi" }
 pq_proto = { path = "../libs/pq_proto" }
 remote_storage = { path = "../libs/remote_storage" }
@@ -76,3 +78,7 @@ tempfile = "3.2"
 [[bench]]
 name = "bench_layer_map"
 harness = false
+
+[[bench]]
+name = "bench_walredo"
+harness = false

pageserver/src/auth.rs

@@ -0,0 +1,22 @@
+use anyhow::{bail, Result};
+use utils::auth::{Claims, Scope};
+use utils::id::TenantId;
+
+pub fn check_permission(claims: &Claims, tenant_id: Option<TenantId>) -> Result<()> {
+    match (&claims.scope, tenant_id) {
+        (Scope::Tenant, None) => {
+            bail!("Attempt to access management api with tenant scope. Permission denied")
+        }
+        (Scope::Tenant, Some(tenant_id)) => {
+            if claims.tenant_id.unwrap() != tenant_id {
+                bail!("Tenant id mismatch. Permission denied")
+            }
+            Ok(())
+        }
+        (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope
+        (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope
+        (Scope::SafekeeperData, _) => {
+            bail!("SafekeeperData scope makes no sense for Pageserver")
+        }
+    }
+}

pageserver/src/bin/pageserver.rs

@@ -1,5 +1,7 @@
 //! Main entry point for the Page Server executable.
 
+use std::env::{var, VarError};
+use std::sync::Arc;
 use std::{env, ops::ControlFlow, path::Path, str::FromStr};
 
 use anyhow::{anyhow, Context};
@@ -199,6 +201,20 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
     logging::init(conf.log_format)?;
     info!("version: {}", version());
 
+    // If any failpoints were set from FAILPOINTS environment variable,
+    // print them to the log for debugging purposes
+    let failpoints = fail::list();
+    if !failpoints.is_empty() {
+        info!(
+            "started with failpoints: {}",
+            failpoints
+                .iter()
+                .map(|(name, actions)| format!("{name}={actions}"))
+                .collect::<Vec<String>>()
+                .join(";")
+        )
+    }
+
     let lock_file_path = conf.workdir.join(PID_FILE_NAME);
     let lock_file = match lock_file::create_lock_file(&lock_file_path, Pid::this().to_string()) {
         lock_file::LockCreationResult::Created {
@@ -223,6 +239,8 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
     // we need to release the lock file only when the current process is gone
     let _ = Box::leak(Box::new(lock_file));
 
+    info!("Created PID file with PID {}", Pid::this().to_string());
+
     // TODO: Check that it looks like a valid repository before going further
 
     // bind sockets before daemonizing so we report errors early and do not return until we are listening
@@ -256,6 +274,23 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
     };
     info!("Using auth: {:#?}", conf.auth_type);
 
+    match var("ZENITH_AUTH_TOKEN") {
+        Ok(v) => {
+            info!("Loaded JWT token for authentication with Safekeeper");
+            pageserver::config::SAFEKEEPER_AUTH_TOKEN
+                .set(Arc::new(v))
+                .map_err(|_| anyhow!("Could not initialize SAFEKEEPER_AUTH_TOKEN"))?;
+        }
+        Err(VarError::NotPresent) => {
+            info!("No JWT token for authentication with Safekeeper detected");
+        }
+        Err(e) => {
+            return Err(e).with_context(|| {
+                "Failed to either load to detect non-present ZENITH_AUTH_TOKEN environment variable"
+            })
+        }
+    };
+
     let remote_storage = conf
         .remote_storage_config
         .as_ref()
@@ -264,7 +299,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
         })
         .transpose()
         .context("Failed to init generic remote storage")?;
-    let remote_index = {
+    {
         let _rt_guard = BACKGROUND_RUNTIME.enter();
         tenant_mgr::init_tenant_mgr(conf, remote_storage.clone())?
     };
@@ -276,7 +311,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
     {
         let _rt_guard = MGMT_REQUEST_RUNTIME.enter();
 
-        let router = http::make_router(conf, auth.clone(), remote_index, remote_storage)?;
+        let router = http::make_router(conf, auth.clone(), remote_storage)?;
         let service =
             utils::http::RouterService::new(router.build().map_err(|err| anyhow!(err))?).unwrap();
         let server = hyper::Server::from_tcp(http_listener)?

pageserver/src/config.rs

@@ -8,10 +8,13 @@ use anyhow::{anyhow, bail, ensure, Context, Result};
 use remote_storage::RemoteStorageConfig;
 use std::env;
 use utils::crashsafe::path_with_suffix_extension;
+use utils::id::ConnectionId;
 
+use once_cell::sync::OnceCell;
 use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
+use std::sync::Arc;
 use std::time::Duration;
 use toml_edit;
 use toml_edit::{Document, Item};
@@ -22,13 +25,9 @@ use utils::{
     postgres_backend::AuthType,
 };
 
-use crate::tenant::TIMELINES_SEGMENT_NAME;
+use crate::tenant::{TENANT_ATTACHING_MARKER_FILENAME, TIMELINES_SEGMENT_NAME};
 use crate::tenant_config::{TenantConf, TenantConfOpt};
-
-/// The name of the metadata file pageserver creates per timeline.
-pub const METADATA_FILE_NAME: &str = "metadata";
-pub const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit";
-const TENANT_CONFIG_NAME: &str = "config";
+use crate::{METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_UNINIT_MARK_SUFFIX};
 
 pub mod defaults {
     use crate::tenant_config::defaults::*;
@@ -144,6 +143,15 @@ pub struct PageServerConf {
     pub concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore,
 }
 
+/// We do not want to store this in a PageServerConf because the latter may be logged
+/// and/or serialized at a whim, while the token is secret. Currently this token is the
+/// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in
+/// the future, more tokens and auth may arrive for etcd and/or its rewrite (see
+/// https://github.com/neondatabase/neon/issues/2394), completely changing the logic.
+/// Hence, we resort to a global variable for now instead of passing the token from the
+/// startup code to the connection code through a dozen layers.
+pub static SAFEKEEPER_AUTH_TOKEN: OnceCell<Arc<String>> = OnceCell::new();
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum ProfilingConfig {
     Disabled,
@@ -389,6 +397,11 @@ impl PageServerConf {
         self.tenants_path().join(tenant_id.to_string())
     }
 
+    pub fn tenant_attaching_mark_file_path(&self, tenant_id: &TenantId) -> PathBuf {
+        self.tenant_path(tenant_id)
+            .join(TENANT_ATTACHING_MARKER_FILENAME)
+    }
+
     /// Points to a place in pageserver's local directory,
     /// where certain tenant's tenantconf file should be located.
     pub fn tenant_config_path(&self, tenant_id: TenantId) -> PathBuf {
@@ -414,6 +427,22 @@ impl PageServerConf {
         )
     }
 
+    pub fn traces_path(&self) -> PathBuf {
+        self.workdir.join("traces")
+    }
+
+    pub fn trace_path(
+        &self,
+        tenant_id: &TenantId,
+        timeline_id: &TimelineId,
+        connection_id: &ConnectionId,
+    ) -> PathBuf {
+        self.traces_path()
+            .join(tenant_id.to_string())
+            .join(timeline_id.to_string())
+            .join(connection_id.to_string())
+    }
+
     /// Points to a place in pageserver's local directory,
     /// where certain timeline's metadata file should be located.
     pub fn metadata_path(&self, timeline_id: TimelineId, tenant_id: TenantId) -> PathBuf {
@@ -597,8 +626,9 @@ impl PageServerConf {
         PathBuf::from(format!("../tmp_check/test_{test_name}"))
     }
 
-    #[cfg(test)]
     pub fn dummy_conf(repo_dir: PathBuf) -> Self {
+        let pg_distrib_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install");
+
         PageServerConf {
             id: NodeId(0),
             wait_lsn_timeout: Duration::from_secs(60),
@@ -609,7 +639,7 @@ impl PageServerConf {
             listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
             superuser: "cloud_admin".to_string(),
             workdir: repo_dir,
-            pg_distrib_dir: PathBuf::new(),
+            pg_distrib_dir,
             auth_type: AuthType::Trust,
             auth_validation_public_key_path: None,
             remote_storage_config: None,

pageserver/src/http/openapi_spec.yml

@@ -604,13 +604,7 @@ components:
         id:
           type: string
         state:
-            oneOf:
-              - type: string
-              - type: object
-                properties:
-                  background_jobs_running:
-                    type: boolean
-
+          type: string
         current_physical_size:
           type: integer
         has_in_progress_downloads:
@@ -667,6 +661,7 @@ components:
         - disk_consistent_lsn
         - awaits_download
         - state
+        - latest_gc_cutoff_lsn
       properties:
         timeline_id:
           type: string
@@ -711,6 +706,9 @@ components:
           type: boolean
         state:
           type: string
+        latest_gc_cutoff_lsn:
+          type: string
+          format: hex
 
         # These 'local' and 'remote' fields just duplicate some of the fields
         # above. They are kept for backwards-compatibility. They can be removed,

pageserver/src/http/routes.rs

@@ -3,31 +3,29 @@ use std::sync::Arc;
 use anyhow::{anyhow, Context, Result};
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
+use pageserver_api::models::TenantState;
 use remote_storage::GenericRemoteStorage;
 use tokio::task::JoinError;
 use tracing::*;
 
-use super::models::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo};
 use super::models::{
-    StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo,
-    TimelineCreateRequest,
+    LocalTimelineInfo, RemoteTimelineInfo, StatusResponse, TenantConfigRequest,
+    TenantCreateRequest, TenantCreateResponse, TenantInfo, TimelineCreateRequest, TimelineInfo,
 };
 use crate::pgdatadir_mapping::LsnForTimestamp;
-use crate::storage_sync;
-use crate::storage_sync::index::{RemoteIndex, RemoteTimeline};
-use crate::tenant::{TenantState, Timeline};
+use crate::tenant::Timeline;
 use crate::tenant_config::TenantConfOpt;
 use crate::{config::PageServerConf, tenant_mgr};
 use utils::{
     auth::JwtAuth,
     http::{
-        endpoint::{self, attach_openapi_ui, auth_middleware, check_permission},
+        endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with},
         error::{ApiError, HttpErrorBody},
         json::{json_request, json_response},
         request::parse_request_param,
         RequestExt, RouterBuilder,
     },
-    id::{TenantId, TenantTimelineId, TimelineId},
+    id::{TenantId, TimelineId},
     lsn::Lsn,
 };
 
@@ -40,7 +38,6 @@ use crate::CheckpointConfig;
 struct State {
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
-    remote_index: RemoteIndex,
     allowlist_routes: Vec<Uri>,
     remote_storage: Option<GenericRemoteStorage>,
 }
@@ -49,7 +46,6 @@ impl State {
     fn new(
         conf: &'static PageServerConf,
         auth: Option<Arc<JwtAuth>>,
-        remote_index: RemoteIndex,
         remote_storage: Option<GenericRemoteStorage>,
     ) -> anyhow::Result<Self> {
         let allowlist_routes = ["/v1/status", "/v1/doc", "/swagger.yml"]
@@ -60,7 +56,6 @@ impl State {
             conf,
             auth,
             allowlist_routes,
-            remote_index,
             remote_storage,
         })
     }
@@ -79,19 +74,41 @@ fn get_config(request: &Request<Body>) -> &'static PageServerConf {
     get_state(request).conf
 }
 
+fn check_permission(request: &Request<Body>, tenant_id: Option<TenantId>) -> Result<(), ApiError> {
+    check_permission_with(request, |claims| {
+        crate::auth::check_permission(claims, tenant_id)
+    })
+}
+
 // Helper function to construct a TimelineInfo struct for a timeline
-async fn build_timeline_info(
-    state: &State,
+fn build_timeline_info(
+    tenant_state: TenantState,
     timeline: &Arc<Timeline>,
     include_non_incremental_logical_size: bool,
     include_non_incremental_physical_size: bool,
+) -> anyhow::Result<TimelineInfo> {
+    let mut info = build_timeline_info_common(tenant_state, timeline)?;
+    if include_non_incremental_logical_size {
+        info.current_logical_size_non_incremental =
+            Some(timeline.get_current_logical_size_non_incremental(info.last_record_lsn)?);
+    }
+    if include_non_incremental_physical_size {
+        info.current_physical_size_non_incremental =
+            Some(timeline.get_physical_size_non_incremental()?)
+    }
+    Ok(info)
+}
+
+fn build_timeline_info_common(
+    tenant_state: TenantState,
+    timeline: &Arc<Timeline>,
 ) -> anyhow::Result<TimelineInfo> {
     let last_record_lsn = timeline.get_last_record_lsn();
     let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = {
         let guard = timeline.last_received_wal.lock().unwrap();
         if let Some(info) = guard.as_ref() {
             (
-                Some(info.wal_source_connstr.clone()),
+                Some(format!("{:?}", info.wal_source_connconf)), // Password is hidden, but it's for statistics only.
                 Some(info.last_received_msg_lsn),
                 Some(info.last_received_msg_ts),
             )
@@ -100,22 +117,6 @@ async fn build_timeline_info(
         }
     };
 
-    let (remote_consistent_lsn, awaits_download) = if let Some(remote_entry) = state
-        .remote_index
-        .read()
-        .await
-        .timeline_entry(&TenantTimelineId {
-            tenant_id: timeline.tenant_id,
-            timeline_id: timeline.timeline_id,
-        }) {
-        (
-            Some(remote_entry.metadata.disk_consistent_lsn()),
-            remote_entry.awaits_download,
-        )
-    } else {
-        (None, false)
-    };
-
     let ancestor_timeline_id = timeline.get_ancestor_timeline_id();
     let ancestor_lsn = match timeline.get_ancestor_lsn() {
         Lsn(0) => None,
@@ -130,6 +131,7 @@ async fn build_timeline_info(
     };
     let current_physical_size = Some(timeline.get_physical_size());
     let state = timeline.current_state();
+    let remote_consistent_lsn = timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));
 
     let info = TimelineInfo {
         tenant_id: timeline.tenant_id,
@@ -137,30 +139,25 @@ async fn build_timeline_info(
         ancestor_timeline_id,
         ancestor_lsn,
         disk_consistent_lsn: timeline.get_disk_consistent_lsn(),
+        remote_consistent_lsn,
         last_record_lsn,
         prev_record_lsn: Some(timeline.get_prev_record_lsn()),
         latest_gc_cutoff_lsn: *timeline.get_latest_gc_cutoff_lsn(),
         current_logical_size,
         current_physical_size,
-        current_logical_size_non_incremental: if include_non_incremental_logical_size {
-            Some(timeline.get_current_logical_size_non_incremental(last_record_lsn)?)
-        } else {
-            None
-        },
-        current_physical_size_non_incremental: if include_non_incremental_physical_size {
-            Some(timeline.get_physical_size_non_incremental()?)
-        } else {
-            None
-        },
+        current_logical_size_non_incremental: None,
+        current_physical_size_non_incremental: None,
         wal_source_connstr,
         last_received_msg_lsn,
         last_received_msg_ts,
         pg_version: timeline.pg_version,
 
-        remote_consistent_lsn,
-        awaits_download,
         state,
 
+        // XXX bring back tracking of downloads per timeline, or, introduce
+        // an 'Attaching' state for the timeline and get rid of this field.
+        awaits_download: tenant_state == TenantState::Attaching,
+
         // Duplicate some fields in 'local' and 'remote' fields, for backwards-compatility
         // with the control plane.
         local: LocalTimelineInfo {
@@ -170,7 +167,7 @@ async fn build_timeline_info(
             current_physical_size,
         },
         remote: RemoteTimelineInfo {
-            remote_consistent_lsn,
+            remote_consistent_lsn: Some(remote_consistent_lsn),
         },
     };
     Ok(info)
@@ -178,6 +175,7 @@ async fn build_timeline_info(
 
 // healthcheck handler
 async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
+    check_permission(&request, None)?;
     let config = get_config(&request);
     json_response(StatusCode::OK, StatusResponse { id: config.id })
 }
@@ -187,34 +185,28 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
     let request_data: TimelineCreateRequest = json_request(&mut request).await?;
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
+    let new_timeline_id = request_data
+        .new_timeline_id
+        .unwrap_or_else(TimelineId::generate);
 
     let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-    let new_timeline_info = async {
-        match tenant.create_timeline(
-            request_data.new_timeline_id.map(TimelineId::from),
-            request_data.ancestor_timeline_id.map(TimelineId::from),
-            request_data.ancestor_start_lsn,
-            request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
-        ).await {
-            Ok(Some(new_timeline)) => {
-                // Created. Construct a TimelineInfo for it.
-                let timeline_info = build_timeline_info(state, &new_timeline, false, false)
-                    .await
-                    .map_err(ApiError::InternalServerError)?;
-                Ok(Some(timeline_info))
-            }
-            Ok(None) => Ok(None), // timeline already exists
-            Err(err) => Err(ApiError::InternalServerError(err)),
+    match tenant.create_timeline(
+        new_timeline_id,
+        request_data.ancestor_timeline_id.map(TimelineId::from),
+        request_data.ancestor_start_lsn,
+        request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION)
+    )
+    .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
+    .await {
+        Ok(Some(new_timeline)) => {
+            // Created. Construct a TimelineInfo for it.
+            let timeline_info = build_timeline_info_common(tenant.current_state(), &new_timeline)
+                .map_err(ApiError::InternalServerError)?;
+            json_response(StatusCode::CREATED, timeline_info)
         }
+        Ok(None) => json_response(StatusCode::CONFLICT, ()), // timeline already exists
+        Err(err) => Err(ApiError::InternalServerError(err)),
     }
-    .instrument(info_span!("timeline_create", tenant = %tenant_id, new_timeline = ?request_data.new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version))
-        .await?;
-
-    Ok(match new_timeline_info {
-        Some(info) => json_response(StatusCode::CREATED, info)?,
-        None => json_response(StatusCode::CONFLICT, ())?,
-    })
 }
 
 async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -225,22 +217,21 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
         query_param_present(&request, "include-non-incremental-physical-size");
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
+    let _entered = info_span!("timeline_list", tenant = %tenant_id).entered();
 
-    let timelines = info_span!("timeline_list", tenant = %tenant_id).in_scope(|| {
+    let (tenant_state, timelines) = {
         let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
-        Ok(tenant.list_timelines())
-    })?;
+        (tenant.current_state(), tenant.list_timelines())
+    };
 
     let mut response_data = Vec::with_capacity(timelines.len());
     for timeline in timelines {
         let timeline_info = build_timeline_info(
-            state,
+            tenant_state,
             &timeline,
             include_non_incremental_logical_size,
             include_non_incremental_physical_size,
         )
-        .await
         .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}")
         .map_err(ApiError::InternalServerError)?;
 
@@ -289,24 +280,25 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
         query_param_present(&request, "include-non-incremental-physical-size");
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
-
     let timeline_info = async {
-        let timeline = tokio::task::spawn_blocking(move || {
-            tenant_mgr::get_tenant(tenant_id, true)?.get_timeline(timeline_id, false)
+        let (tenant_state, timeline) = tokio::task::spawn_blocking(move || {
+            let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::NotFound)?;
+            Ok((
+                tenant.current_state(),
+                tenant.get_timeline(timeline_id, false),
+            ))
         })
         .await
-        .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
+        .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
 
         let timeline = timeline.map_err(ApiError::NotFound)?;
 
         let timeline_info = build_timeline_info(
-            state,
+            tenant_state,
             &timeline,
             include_non_incremental_logical_size,
             include_non_incremental_physical_size,
         )
-        .await
         .context("Failed to get local timeline info: {e:#}")
         .map_err(ApiError::InternalServerError)?;
 
@@ -351,117 +343,28 @@ async fn tenant_attach_handler(request: Request<Body>) -> Result<Response<Body>,
 
     info!("Handling tenant attach {tenant_id}");
 
-    tokio::task::spawn_blocking(move || match tenant_mgr::get_tenant(tenant_id, false) {
-        Ok(tenant) => {
-            if tenant.list_timelines().is_empty() {
-                info!("Attaching to tenant {tenant_id} with zero timelines");
-                Ok(())
-            } else {
-                Err(ApiError::Conflict(
-                    "Tenant is already present locally".to_owned(),
-                ))
-            }
-        }
-        Err(_) => Ok(()),
-    })
-    .await
-    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
-
     let state = get_state(&request);
-    let remote_index = &state.remote_index;
 
-    let mut index_accessor = remote_index.write().await;
-    if let Some(tenant_entry) = index_accessor.tenant_entry_mut(&tenant_id) {
-        if tenant_entry.has_in_progress_downloads() {
-            return Err(ApiError::Conflict(
-                "Tenant download is already in progress".to_string(),
-            ));
-        }
-
-        for (timeline_id, remote_timeline) in tenant_entry.iter_mut() {
-            storage_sync::schedule_layer_download(tenant_id, *timeline_id);
-            remote_timeline.awaits_download = true;
-        }
-        return json_response(StatusCode::ACCEPTED, ());
-    }
-    // no tenant in the index, release the lock to make the potentially lengthy download operation
-    drop(index_accessor);
-
-    // download index parts for every tenant timeline
-    let remote_timelines = match gather_tenant_timelines_index_parts(state, tenant_id).await {
-        Ok(Some(remote_timelines)) => remote_timelines,
-        Ok(None) => return Err(ApiError::NotFound(anyhow!("Unknown remote tenant"))),
-        Err(e) => {
-            error!("Failed to retrieve remote tenant data: {:?}", e);
-            return Err(ApiError::NotFound(anyhow!(
-                "Failed to retrieve remote tenant"
-            )));
-        }
-    };
-
-    // recheck that download is not in progress because
-    // we've released the lock to avoid holding it during the download
-    let mut index_accessor = remote_index.write().await;
-    let tenant_entry = match index_accessor.tenant_entry_mut(&tenant_id) {
-        Some(tenant_entry) => {
-            if tenant_entry.has_in_progress_downloads() {
-                return Err(ApiError::Conflict(
-                    "Tenant download is already in progress".to_string(),
-                ));
-            }
-            tenant_entry
-        }
-        None => index_accessor.add_tenant_entry(tenant_id),
-    };
-
-    // populate remote index with the data from index part and create directories on the local filesystem
-    for (timeline_id, mut remote_timeline) in remote_timelines {
-        tokio::fs::create_dir_all(state.conf.timeline_path(&timeline_id, &tenant_id))
+    if let Some(remote_storage) = &state.remote_storage {
+        // FIXME: distinguish between "Tenant already exists" and other errors
+        tenant_mgr::attach_tenant(state.conf, tenant_id, remote_storage)
+            .instrument(info_span!("tenant_attach", tenant = %tenant_id))
             .await
-            .context("Failed to create new timeline directory")
             .map_err(ApiError::InternalServerError)?;
-
-        remote_timeline.awaits_download = true;
-        tenant_entry.insert(timeline_id, remote_timeline);
-        // schedule actual download
-        storage_sync::schedule_layer_download(tenant_id, timeline_id);
+    } else {
+        return Err(ApiError::BadRequest(anyhow!(
+            "attach_tenant is possible because pageserver was configured without remote storage"
+        )));
     }
 
     json_response(StatusCode::ACCEPTED, ())
 }
 
-/// Note: is expensive from s3 access perspective,
-/// for details see comment to `storage_sync::gather_tenant_timelines_index_parts`
-async fn gather_tenant_timelines_index_parts(
-    state: &State,
-    tenant_id: TenantId,
-) -> anyhow::Result<Option<Vec<(TimelineId, RemoteTimeline)>>> {
-    let index_parts = match state.remote_storage.as_ref() {
-        Some(storage) => {
-            storage_sync::gather_tenant_timelines_index_parts(state.conf, storage, tenant_id).await
-        }
-        None => return Ok(None),
-    }
-    .with_context(|| format!("Failed to download index parts for tenant {tenant_id}"))?;
-
-    let mut remote_timelines = Vec::with_capacity(index_parts.len());
-    for (timeline_id, index_part) in index_parts {
-        let timeline_path = state.conf.timeline_path(&timeline_id, &tenant_id);
-        let remote_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
-            .with_context(|| {
-                format!("Failed to convert index part into remote timeline for timeline {tenant_id}/{timeline_id}")
-            })?;
-        remote_timelines.push((timeline_id, remote_timeline));
-    }
-    Ok(Some(remote_timelines))
-}
-
 async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let state = get_state(&request);
     tenant_mgr::delete_timeline(tenant_id, timeline_id)
         .instrument(info_span!("timeline_delete", tenant = %tenant_id, timeline = %timeline_id))
         .await
@@ -470,12 +373,6 @@ async fn timeline_delete_handler(request: Request<Body>) -> Result<Response<Body
         // it.
         .map_err(ApiError::InternalServerError)?;
 
-    let mut remote_index = state.remote_index.write().await;
-    remote_index.remove_timeline_entry(TenantTimelineId {
-        tenant_id,
-        timeline_id,
-    });
-
     json_response(StatusCode::OK, ())
 }
 
@@ -492,22 +389,23 @@ async fn tenant_detach_handler(request: Request<Body>) -> Result<Response<Body>,
         // Replace this with better handling once the error type permits it.
         .map_err(ApiError::InternalServerError)?;
 
-    let mut remote_index = state.remote_index.write().await;
-    remote_index.remove_tenant_entry(&tenant_id);
-
     json_response(StatusCode::OK, ())
 }
 
 async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     check_permission(&request, None)?;
 
-    let state = get_state(&request);
-    // clone to avoid holding the lock while awaiting for blocking task
-    let remote_index = state.remote_index.read().await.clone();
-
     let response_data = tokio::task::spawn_blocking(move || {
         let _enter = info_span!("tenant_list").entered();
-        crate::tenant_mgr::list_tenant_info(&remote_index)
+        tenant_mgr::list_tenants()
+            .iter()
+            .map(|(id, state)| TenantInfo {
+                id: *id,
+                state: *state,
+                current_physical_size: None,
+                has_in_progress_downloads: Some(state.has_in_progress_downloads()),
+            })
+            .collect::<Vec<TenantInfo>>()
     })
     .await
     .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?;
@@ -519,58 +417,38 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    // if tenant is in progress of downloading it can be absent in global tenant map
-    let tenant = tenant_mgr::get_tenant(tenant_id, false);
+    let tenant_info = tokio::task::spawn_blocking(move || {
+        let _enter = info_span!("tenant_status_handler", tenant = %tenant_id).entered();
+        let tenant = tenant_mgr::get_tenant(tenant_id, false)?;
 
-    let state = get_state(&request);
-    let remote_index = &state.remote_index;
-
-    let index_accessor = remote_index.read().await;
-    let has_in_progress_downloads = index_accessor
-        .tenant_entry(&tenant_id)
-        .map(|t| t.has_in_progress_downloads())
-        .unwrap_or_else(|| {
-            info!("Tenant {tenant_id} not found in remote index");
-            false
-        });
-
-    let (tenant_state, current_physical_size) = match tenant {
-        Ok(tenant) => {
-            let timelines = tenant.list_timelines();
-            // Calculate total physical size of all timelines
-            let mut current_physical_size = 0;
-            for timeline in timelines {
-                current_physical_size += timeline.get_physical_size();
-            }
-
-            (tenant.current_state(), Some(current_physical_size))
+        // Calculate total physical size of all timelines
+        let mut current_physical_size = 0;
+        for timeline in tenant.list_timelines().iter() {
+            current_physical_size += timeline.get_physical_size();
         }
-        Err(e) => {
-            error!("Failed to get local tenant state: {e:#}");
-            if has_in_progress_downloads {
-                (TenantState::Paused, None)
-            } else {
-                (TenantState::Broken, None)
-            }
-        }
-    };
 
-    json_response(
-        StatusCode::OK,
-        TenantInfo {
+        let state = tenant.current_state();
+        let tenant_info = TenantInfo {
             id: tenant_id,
-            state: tenant_state,
-            current_physical_size,
-            has_in_progress_downloads: Some(has_in_progress_downloads),
-        },
-    )
+            state,
+            current_physical_size: Some(current_physical_size),
+            has_in_progress_downloads: Some(state.has_in_progress_downloads()),
+        };
+
+        Ok::<_, anyhow::Error>(tenant_info)
+    })
+    .await
+    .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))?
+    .map_err(ApiError::InternalServerError)?;
+
+    json_response(StatusCode::OK, tenant_info)
 }
 
 async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    let tenant = tenant_mgr::get_tenant(tenant_id, false).map_err(ApiError::InternalServerError)?;
+    let tenant = tenant_mgr::get_tenant(tenant_id, true).map_err(ApiError::InternalServerError)?;
 
     // this can be long operation, it currently is not backed by any request coalescing or similar
     let inputs = tenant
@@ -618,7 +496,6 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     check_permission(&request, None)?;
 
     let request_data: TenantCreateRequest = json_request(&mut request).await?;
-    let remote_index = get_state(&request).remote_index.clone();
 
     let mut tenant_conf = TenantConfOpt::default();
     if let Some(gc_period) = request_data.gc_period {
@@ -659,6 +536,9 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
     if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag {
         tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag);
     }
+    if let Some(trace_read_requests) = request_data.trace_read_requests {
+        tenant_conf.trace_read_requests = Some(trace_read_requests);
+    }
 
     tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
     if let Some(checkpoint_timeout) = request_data.checkpoint_timeout {
@@ -685,20 +565,42 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
         .map(TenantId::from)
         .unwrap_or_else(TenantId::generate);
 
-    let new_tenant_id = tokio::task::spawn_blocking(move || {
+    let new_tenant = tokio::task::spawn_blocking(move || {
         let _enter = info_span!("tenant_create", tenant = ?target_tenant_id).entered();
-        let conf = get_config(&request);
+        let state = get_state(&request);
 
-        tenant_mgr::create_tenant(conf, tenant_conf, target_tenant_id, remote_index)
-            // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
-            // with better error handling once the type permits it
-            .map_err(ApiError::InternalServerError)
+        tenant_mgr::create_tenant(
+            state.conf,
+            tenant_conf,
+            target_tenant_id,
+            state.remote_storage.clone(),
+        )
+        // FIXME: `create_tenant` can fail from both user and internal errors. Replace this
+        // with better error handling once the type permits it
+        .map_err(ApiError::InternalServerError)
     })
     .await
     .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??;
 
-    Ok(match new_tenant_id {
-        Some(id) => json_response(StatusCode::CREATED, TenantCreateResponse(id))?,
+    Ok(match new_tenant {
+        Some(tenant) => {
+            // We created the tenant. Existing API semantics are that the tenant
+            // is Active when this function returns.
+            if let res @ Err(_) = tenant.wait_to_become_active().await {
+                // This shouldn't happen because we just created the tenant directory
+                // in tenant_mgr::create_tenant, and there aren't any remote timelines
+                // to load, so, nothing can really fail during load.
+                // Don't do cleanup because we don't know how we got here.
+                // The tenant will likely be in `Broken` state and subsequent
+                // calls will fail.
+                res.context("created tenant failed to become active")
+                    .map_err(ApiError::InternalServerError)?;
+            }
+            json_response(
+                StatusCode::CREATED,
+                TenantCreateResponse(tenant.tenant_id()),
+            )?
+        }
         None => json_response(StatusCode::CONFLICT, ())?,
     })
 }
@@ -746,6 +648,9 @@ async fn tenant_config_handler(mut request: Request<Body>) -> Result<Response<Bo
     if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag {
         tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag);
     }
+    if let Some(trace_read_requests) = request_data.trace_read_requests {
+        tenant_conf.trace_read_requests = Some(trace_read_requests);
+    }
 
     tenant_conf.checkpoint_distance = request_data.checkpoint_distance;
     if let Some(checkpoint_timeout) = request_data.checkpoint_timeout {
@@ -821,22 +726,16 @@ async fn timeline_gc_handler(mut request: Request<Body>) -> Result<Response<Body
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
     check_permission(&request, Some(tenant_id))?;
 
-    // FIXME: currently this will return a 500 error on bad tenant id; it should be 4XX
-    let tenant = tenant_mgr::get_tenant(tenant_id, false).map_err(ApiError::NotFound)?;
     let gc_req: TimelineGcRequest = json_request(&mut request).await?;
 
-    let gc_horizon = gc_req.gc_horizon.unwrap_or_else(|| tenant.get_gc_horizon());
-
-    // Use tenant's pitr setting
-    let pitr = tenant.get_pitr_interval();
-    let result = tenant
-        .gc_iteration(Some(timeline_id), gc_horizon, pitr, true)
-        .instrument(info_span!("manual_gc", tenant = %tenant_id, timeline = %timeline_id))
+    let wait_task_done = tenant_mgr::immediate_gc(tenant_id, timeline_id, gc_req)?;
+    let gc_result = wait_task_done
         .await
-        // FIXME: `gc_iteration` can return an error for multiple reasons; we should handle it
-        // better once the types support it.
+        .context("wait for gc task")
+        .map_err(ApiError::InternalServerError)?
         .map_err(ApiError::InternalServerError)?;
-    json_response(StatusCode::OK, result)
+
+    json_response(StatusCode::OK, gc_result)
 }
 
 // Run compaction immediately on given timeline.
@@ -850,7 +749,10 @@ async fn timeline_compact_handler(request: Request<Body>) -> Result<Response<Bod
     let timeline = tenant
         .get_timeline(timeline_id, true)
         .map_err(ApiError::NotFound)?;
-    timeline.compact().map_err(ApiError::InternalServerError)?;
+    timeline
+        .compact()
+        .await
+        .map_err(ApiError::InternalServerError)?;
 
     json_response(StatusCode::OK, ())
 }
@@ -884,7 +786,6 @@ async fn handler_404(_: Request<Body>) -> Result<Response<Body>, ApiError> {
 pub fn make_router(
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
-    remote_index: RemoteIndex,
     remote_storage: Option<GenericRemoteStorage>,
 ) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
     let spec = include_bytes!("openapi_spec.yml");
@@ -921,8 +822,7 @@ pub fn make_router(
 
     Ok(router
         .data(Arc::new(
-            State::new(conf, auth, remote_index, remote_storage)
-                .context("Failed to initialize router state")?,
+            State::new(conf, auth, remote_storage).context("Failed to initialize router state")?,
         ))
         .get("/v1/status", status_handler)
         .put(

pageserver/src/lib.rs

@@ -1,30 +1,32 @@
+mod auth;
 pub mod basebackup;
 pub mod config;
 pub mod http;
 pub mod import_datadir;
 pub mod keyspace;
-pub mod metrics;
+pub(crate) mod metrics;
 pub mod page_cache;
 pub mod page_service;
 pub mod pgdatadir_mapping;
 pub mod profiling;
 pub mod repository;
-pub mod storage_sync;
+pub mod storage_sync2;
+pub use storage_sync2 as storage_sync;
 pub mod task_mgr;
 pub mod tenant;
 pub mod tenant_config;
 pub mod tenant_mgr;
 pub mod tenant_tasks;
+pub mod trace;
 pub mod virtual_file;
 pub mod walingest;
 pub mod walreceiver;
 pub mod walrecord;
 pub mod walredo;
 
-use std::collections::HashMap;
+use std::path::Path;
 
 use tracing::info;
-use utils::id::{TenantId, TimelineId};
 
 use crate::task_mgr::TaskKind;
 
@@ -70,7 +72,7 @@ pub async fn shutdown_pageserver(exit_code: i32) {
     //
     // FIXME: Does this wait for the sync tasks to finish syncing what's queued up?
     // Should it?
-    task_mgr::shutdown_tasks(Some(TaskKind::StorageSync), None, None).await;
+    task_mgr::shutdown_tasks(Some(TaskKind::RemoteUploadTask), None, None).await;
 
     // Shut down the HTTP endpoint last, so that you can still check the server's
     // status while it's shutting down.
@@ -105,23 +107,39 @@ fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_seconds
     }
 }
 
-/// A newtype to store arbitrary data grouped by tenant and timeline ids.
-/// One could use [`utils::id::TenantTimelineId`] for grouping, but that would
-/// not include the cases where a certain tenant has zero timelines.
-/// This is sometimes important: a tenant could be registered during initial load from FS,
-/// even if he has no timelines on disk.
-#[derive(Debug)]
-pub struct TenantTimelineValues<T>(HashMap<TenantId, HashMap<TimelineId, T>>);
+/// The name of the metadata file pageserver creates per timeline.
+/// Full path: `tenants/<tenant_id>/timelines/<timeline_id>/metadata`.
+pub const METADATA_FILE_NAME: &str = "metadata";
 
-impl<T> TenantTimelineValues<T> {
-    fn new() -> Self {
-        Self(HashMap::new())
+/// Per-tenant configuration file.
+/// Full path: `tenants/<tenant_id>/config`.
+pub const TENANT_CONFIG_NAME: &str = "config";
+
+/// A suffix used for various temporary files. Any temporary files found in the
+/// data directory at pageserver startup can be automatically removed.
+pub const TEMP_FILE_SUFFIX: &str = "___temp";
+
+/// A marker file to mark that a timeline directory was not fully initialized.
+/// If a timeline directory with this marker is encountered at pageserver startup,
+/// the timeline directory and the marker file are both removed.
+/// Full path: `tenants/<tenant_id>/timelines/<timeline_id>___uninit`.
+pub const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit";
+
+pub fn is_temporary(path: &Path) -> bool {
+    match path.file_name() {
+        Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX),
+        None => false,
     }
 }
 
-/// A suffix to be used during file sync from the remote storage,
-/// to ensure that we do not leave corrupted files that pretend to be layers.
-const TEMP_FILE_SUFFIX: &str = "___temp";
+pub fn is_uninit_mark(path: &Path) -> bool {
+    match path.file_name() {
+        Some(name) => name
+            .to_string_lossy()
+            .ends_with(TIMELINE_UNINIT_MARK_SUFFIX),
+        None => false,
+    }
+}
 
 #[cfg(test)]
 mod backoff_defaults_tests {

pageserver/src/metrics.rs

@@ -1,9 +1,8 @@
 use metrics::core::{AtomicU64, GenericCounter};
 use metrics::{
-    register_gauge_vec, register_histogram, register_histogram_vec, register_int_counter,
-    register_int_counter_vec, register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec,
-    GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge,
-    UIntGaugeVec,
+    register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec,
+    register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec, Histogram, HistogramVec,
+    IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
 };
 use once_cell::sync::Lazy;
 use utils::id::{TenantId, TimelineId};
@@ -200,63 +199,59 @@ pub static NUM_ONDISK_LAYERS: Lazy<IntGauge> = Lazy::new(|| {
         .expect("failed to define a metric")
 });
 
-pub static REMAINING_SYNC_ITEMS: Lazy<IntGauge> = Lazy::new(|| {
-    register_int_gauge!(
-        "pageserver_remote_storage_remaining_sync_items",
-        "Number of storage sync items left in the queue"
+// remote storage metrics
+
+pub static REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS: Lazy<IntGaugeVec> = Lazy::new(|| {
+    register_int_gauge_vec!(
+        "pageserver_remote_upload_queue_unfinished_tasks",
+        "Number of tasks in the upload queue that are not finished yet.",
+        &["tenant_id", "timeline_id", "file_kind", "op_kind"],
     )
-    .expect("failed to register pageserver remote storage remaining sync items int gauge")
+    .expect("failed to define a metric")
 });
 
-pub static IMAGE_SYNC_TIME: Lazy<GaugeVec> = Lazy::new(|| {
-    register_gauge_vec!(
-        "pageserver_remote_storage_image_sync_duration",
-        "Time spent to synchronize (up/download) a whole pageserver image",
-        &["tenant_id", "timeline_id"],
-    )
-    .expect("failed to register per-timeline pageserver image sync time vec")
-});
+#[derive(Debug, Clone, Copy)]
+pub enum RemoteOpKind {
+    Upload,
+    Download,
+    Delete,
+}
+impl RemoteOpKind {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Upload => "upload",
+            Self::Download => "download",
+            Self::Delete => "delete",
+        }
+    }
+}
 
-pub static IMAGE_SYNC_OPERATION_KINDS: &[&str] = &["upload", "download", "delete"];
-pub static IMAGE_SYNC_STATUS: &[&str] = &["success", "failure", "abort"];
+#[derive(Debug, Clone, Copy)]
+pub enum RemoteOpFileKind {
+    Layer,
+    Index,
+}
+impl RemoteOpFileKind {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::Layer => "layer",
+            Self::Index => "index",
+        }
+    }
+}
 
-pub static IMAGE_SYNC_COUNT: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "pageserver_remote_storage_image_sync_count",
-        "Number of synchronization operations executed for pageserver images. \
-        Grouped by tenant, timeline, operation_kind and status",
-        &["tenant_id", "timeline_id", "operation_kind", "status"]
-    )
-    .expect("failed to register pageserver image sync count vec")
-});
+pub static REMOTE_OPERATION_KINDS: &[&str] = &["upload", "download", "delete"];
+pub static REMOTE_OPERATION_FILE_KINDS: &[&str] = &["layer", "index"];
+pub static REMOTE_OPERATION_STATUSES: &[&str] = &["success", "failure"];
 
-pub static IMAGE_SYNC_TIME_HISTOGRAM: Lazy<HistogramVec> = Lazy::new(|| {
+pub static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
     register_histogram_vec!(
-        "pageserver_remote_storage_image_sync_seconds",
-        "Time took to synchronize (download or upload) a whole pageserver image. \
-        Grouped by operation_kind and status",
-        &["operation_kind", "status"],
-        vec![0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 3.0, 10.0, 20.0]
+        "pageserver_remote_operation_seconds",
+        "Time spent on remote storage operations. \
+        Grouped by tenant, timeline, operation_kind and status",
+        &["tenant_id", "timeline_id", "file_kind", "op_kind", "status"]
     )
-    .expect("failed to register pageserver image sync time histogram vec")
-});
-
-pub static REMOTE_INDEX_UPLOAD: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "pageserver_remote_storage_remote_index_uploads_total",
-        "Number of remote index uploads",
-        &["tenant_id", "timeline_id"],
-    )
-    .expect("failed to register pageserver remote index upload vec")
-});
-
-pub static NO_LAYERS_UPLOAD: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "pageserver_remote_storage_no_layers_uploads_total",
-        "Number of skipped uploads due to no layers",
-        &["tenant_id", "timeline_id"],
-    )
-    .expect("failed to register pageserver no layers upload vec")
+    .expect("failed to define a metric")
 });
 
 pub static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {
@@ -473,16 +468,90 @@ impl Drop for TimelineMetrics {
             let _ = SMGR_QUERY_TIME.remove_label_values(&[op, tenant_id, timeline_id]);
         }
 
-        for op in IMAGE_SYNC_OPERATION_KINDS {
-            for status in IMAGE_SYNC_STATUS {
-                let _ = IMAGE_SYNC_COUNT.remove_label_values(&[tenant_id, timeline_id, op, status]);
+        let _ = REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS.remove_label_values(&[tenant_id, timeline_id]);
+        for file_kind in REMOTE_OPERATION_FILE_KINDS {
+            for op in REMOTE_OPERATION_KINDS {
+                for status in REMOTE_OPERATION_STATUSES {
+                    let _ = REMOTE_OPERATION_TIME.remove_label_values(&[
+                        tenant_id,
+                        timeline_id,
+                        file_kind,
+                        op,
+                        status,
+                    ]);
+                }
             }
         }
-
-        let _ = IMAGE_SYNC_TIME.remove_label_values(&[tenant_id, timeline_id]);
     }
 }
 
 pub fn remove_tenant_metrics(tenant_id: &TenantId) {
     let _ = STORAGE_TIME.remove_label_values(&["gc", &tenant_id.to_string(), "-"]);
 }
+
+use futures::Future;
+use pin_project_lite::pin_project;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use std::time::Instant;
+
+/// Wrapper future that measures the time spent by a remote storage operation,
+/// and records the time and success/failure as a prometheus metric.
+pub trait MeasureRemoteOp: Sized {
+    fn measure_remote_op(
+        self,
+        tenant_id: TenantId,
+        timeline_id: TimelineId,
+        file_kind: RemoteOpFileKind,
+        op: RemoteOpKind,
+    ) -> MeasuredRemoteOp<Self> {
+        let start = Instant::now();
+        MeasuredRemoteOp {
+            inner: self,
+            tenant_id,
+            timeline_id,
+            file_kind,
+            op,
+            start,
+        }
+    }
+}
+
+impl<T: Sized> MeasureRemoteOp for T {}
+
+pin_project! {
+    pub struct MeasuredRemoteOp<F>
+    {
+        #[pin]
+        inner: F,
+        tenant_id: TenantId,
+        timeline_id: TimelineId,
+        file_kind: RemoteOpFileKind,
+        op: RemoteOpKind,
+        start: Instant,
+    }
+}
+
+impl<F: Future<Output = Result<O, E>>, O, E> Future for MeasuredRemoteOp<F> {
+    type Output = Result<O, E>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.project();
+        let poll_result = this.inner.poll(cx);
+        if let Poll::Ready(ref res) = poll_result {
+            let duration = this.start.elapsed();
+            let status = if res.is_ok() { &"success" } else { &"failure" };
+            REMOTE_OPERATION_TIME
+                .get_metric_with_label_values(&[
+                    &this.tenant_id.to_string(),
+                    &this.timeline_id.to_string(),
+                    this.file_kind.as_str(),
+                    this.op.as_str(),
+                    status,
+                ])
+                .unwrap()
+                .observe(duration.as_secs_f64());
+        }
+        poll_result
+    }
+}

pageserver/src/page_service.rs

@@ -13,7 +13,6 @@ use anyhow::{bail, ensure, Context, Result};
 use bytes::Buf;
 use bytes::Bytes;
 use futures::{Stream, StreamExt};
-use futures::stream::FuturesOrdered;
 use pageserver_api::models::{
     PagestreamBeMessage, PagestreamDbSizeRequest, PagestreamDbSizeResponse,
     PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse,
@@ -26,12 +25,14 @@ use std::net::TcpListener;
 use std::str;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::pin;
 use tokio_util::io::StreamReader;
 use tokio_util::io::SyncIoBridge;
 use tracing::*;
+use utils::id::ConnectionId;
 use utils::{
-    auth::{self, Claims, JwtAuth, Scope},
+    auth::{Claims, JwtAuth, Scope},
     id::{TenantId, TimelineId},
     lsn::Lsn,
     postgres_backend::AuthType,
@@ -39,14 +40,17 @@ use utils::{
     simple_rcu::RcuReadGuard,
 };
 
+use crate::auth::check_permission;
 use crate::basebackup;
-use crate::config::PageServerConf;
+use crate::config::{PageServerConf, ProfilingConfig};
 use crate::import_datadir::import_wal_from_tar;
 use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME};
+use crate::profiling::profpoint_start;
 use crate::task_mgr;
 use crate::task_mgr::TaskKind;
-use crate::tenant::Timeline;
+use crate::tenant::{Tenant, Timeline};
 use crate::tenant_mgr;
+use crate::trace::Tracer;
 use crate::CheckpointConfig;
 
 use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID;
@@ -74,6 +78,12 @@ fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Byt
                         FeMessage::CopyData(bytes) => bytes,
                         FeMessage::CopyDone => { break },
                         FeMessage::Sync => continue,
+                        FeMessage::Terminate => {
+                            let msg = format!("client terminated connection with Terminate message during COPY");
+                            pgb.write_message(&BeMessage::ErrorResponse(&msg))?;
+                            Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?;
+                            break;
+                        }
                         m => {
                             let msg = format!("unexpected message {:?}", m);
                             pgb.write_message(&BeMessage::ErrorResponse(&msg))?;
@@ -85,10 +95,10 @@ fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Byt
                     yield copy_data_bytes;
                 }
                 Ok(None) => {
-                    let msg = "client closed connection";
+                    let msg = "client closed connection during COPY";
                     pgb.write_message(&BeMessage::ErrorResponse(msg))?;
                     pgb.flush().await?;
-                    Err(io::Error::new(io::ErrorKind::Other, msg))?;
+                    Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?;
                 }
                 Err(e) => {
                     Err(io::Error::new(io::ErrorKind::Other, e))?;
@@ -269,17 +279,29 @@ impl PageServerHandler {
         //       so there is no need to reset the association
         task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
 
+        // Make request tracer if needed
+        let tenant = get_active_tenant_with_timeout(tenant_id).await?;
+        let mut tracer = if tenant.get_trace_read_requests() {
+            let connection_id = ConnectionId::generate();
+            let path = tenant
+                .conf
+                .trace_path(&tenant_id, &timeline_id, &connection_id);
+            Some(Tracer::new(path))
+        } else {
+            None
+        };
+
         // Check that the timeline exists
-        let timeline = get_local_timeline(tenant_id, timeline_id)?;
+        let timeline = tenant.get_timeline(timeline_id, true)?;
 
         // switch client to COPYBOTH
         pgb.write_message(&BeMessage::CopyBothResponse)?;
         pgb.flush().await?;
 
-        let mut inprogress_requests = FuturesOrdered::new();
+        let metrics = PageRequestMetrics::new(&tenant_id, &timeline_id);
 
         loop {
-            tokio::select! {
+            let msg = tokio::select! {
                 biased;
 
                 _ = task_mgr::shutdown_watcher() => {
@@ -288,58 +310,57 @@ impl PageServerHandler {
                     break;
                 }
 
-                response = inprogress_requests.next(), if !inprogress_requests.is_empty() => {
-                    let response: Bytes = response.unwrap()?;
-                    pgb.write_message(&BeMessage::CopyData(&response))?;
-                    pgb.flush().await?;
-                    continue;
+                msg = pgb.read_message() => { msg }
+            };
+
+            let copy_data_bytes = match msg? {
+                Some(FeMessage::CopyData(bytes)) => bytes,
+                Some(FeMessage::Terminate) => break,
+                Some(m) => {
+                    bail!("unexpected message: {m:?} during COPY");
                 }
+                None => break, // client disconnected
+            };
 
-                msg = pgb.read_message() => {
-                    let copy_data_bytes = match msg? {
-                        Some(FeMessage::CopyData(bytes)) => bytes,
-                        Some(m) => {
-                            bail!("unexpected message: {m:?} during COPY");
-                        }
-                        None => break, // client disconnected
-                    };
+            trace!("query: {copy_data_bytes:?}");
 
-                    trace!("query: {copy_data_bytes:?}");
+            // Trace request if needed
+            if let Some(t) = tracer.as_mut() {
+                t.trace(&copy_data_bytes)
+            }
 
-                    let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader())?;
+            let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader())?;
 
-                    let timeline = Arc::clone(&timeline);
-                    let task = async move {
-                        let response = match neon_fe_msg {
-                            PagestreamFeMessage::Exists(req) => {
-                                Self::handle_get_rel_exists_request(&timeline, &req).await
-                            }
-                            PagestreamFeMessage::Nblocks(req) => {
-                                Self::handle_get_nblocks_request(&timeline, &req).await
-                            }
-                            PagestreamFeMessage::GetPage(req) => {
-                                Self::handle_get_page_at_lsn_request(&timeline, &req).await
-                            }
-                            PagestreamFeMessage::DbSize(req) => {
-                                Self::handle_db_size_request(&timeline, &req).await
-                            }
-                        };
-
-                        let response = response.unwrap_or_else(|e| {
-                            // print the all details to the log with {:#}, but for the client the
-                            // error message is enough
-                            error!("error reading relation or page version: {:?}", e);
-                            PagestreamBeMessage::Error(PagestreamErrorResponse {
-                                message: e.to_string(),
-                            })
-                        });
-                        let response: Bytes = response.serialize();
-                        response
-                    };
-                    inprogress_requests.push_back(tokio::spawn(task));
-                    continue;
+            let response = match neon_fe_msg {
+                PagestreamFeMessage::Exists(req) => {
+                    let _timer = metrics.get_rel_exists.start_timer();
+                    self.handle_get_rel_exists_request(&timeline, &req).await
+                }
+                PagestreamFeMessage::Nblocks(req) => {
+                    let _timer = metrics.get_rel_size.start_timer();
+                    self.handle_get_nblocks_request(&timeline, &req).await
+                }
+                PagestreamFeMessage::GetPage(req) => {
+                    let _timer = metrics.get_page_at_lsn.start_timer();
+                    self.handle_get_page_at_lsn_request(&timeline, &req).await
+                }
+                PagestreamFeMessage::DbSize(req) => {
+                    let _timer = metrics.get_db_size.start_timer();
+                    self.handle_db_size_request(&timeline, &req).await
                 }
             };
+
+            let response = response.unwrap_or_else(|e| {
+                // print the all details to the log with {:#}, but for the client the
+                // error message is enough
+                error!("error reading relation or page version: {:?}", e);
+                PagestreamBeMessage::Error(PagestreamErrorResponse {
+                    message: e.to_string(),
+                })
+            });
+
+            pgb.write_message(&BeMessage::CopyData(&response.serialize()))?;
+            pgb.flush().await?;
         }
         Ok(())
     }
@@ -357,7 +378,7 @@ impl PageServerHandler {
         task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
         // Create empty timeline
         info!("creating new timeline");
-        let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
+        let tenant = get_active_tenant_with_timeout(tenant_id).await?;
         let timeline = tenant.create_empty_timeline(timeline_id, base_lsn, pg_version)?;
 
         // TODO mark timeline as not ready until it reaches end_lsn.
@@ -412,7 +433,7 @@ impl PageServerHandler {
     ) -> anyhow::Result<()> {
         task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
 
-        let timeline = get_local_timeline(tenant_id, timeline_id)?;
+        let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
         ensure!(timeline.get_last_record_lsn() == start_lsn);
 
         // TODO leave clean state on error. For now you can use detach to clean
@@ -511,8 +532,9 @@ impl PageServerHandler {
         Ok(lsn)
     }
 
-    #[instrument(skip(timeline, req), fields(rel = %req.rel, req_lsn = %req.lsn))]
+    #[instrument(skip(self, timeline, req), fields(rel = %req.rel, req_lsn = %req.lsn))]
     async fn handle_get_rel_exists_request(
+        &self,
         timeline: &Timeline,
         req: &PagestreamExistsRequest,
     ) -> Result<PagestreamBeMessage> {
@@ -527,8 +549,9 @@ impl PageServerHandler {
         }))
     }
 
-    #[instrument(skip(timeline, req), fields(rel = %req.rel, req_lsn = %req.lsn))]
+    #[instrument(skip(self, timeline, req), fields(rel = %req.rel, req_lsn = %req.lsn))]
     async fn handle_get_nblocks_request(
+        &self,
         timeline: &Timeline,
         req: &PagestreamNblocksRequest,
     ) -> Result<PagestreamBeMessage> {
@@ -543,8 +566,9 @@ impl PageServerHandler {
         }))
     }
 
-    #[instrument(skip(timeline, req), fields(dbnode = %req.dbnode, req_lsn = %req.lsn))]
+    #[instrument(skip(self, timeline, req), fields(dbnode = %req.dbnode, req_lsn = %req.lsn))]
     async fn handle_db_size_request(
+        &self,
         timeline: &Timeline,
         req: &PagestreamDbSizeRequest,
     ) -> Result<PagestreamBeMessage> {
@@ -562,8 +586,9 @@ impl PageServerHandler {
         }))
     }
 
-    #[instrument(skip(timeline, req), fields(rel = %req.rel, blkno = %req.blkno, req_lsn = %req.lsn))]
+    #[instrument(skip(self, timeline, req), fields(rel = %req.rel, blkno = %req.blkno, req_lsn = %req.lsn))]
     async fn handle_get_page_at_lsn_request(
+        &self,
         timeline: &Timeline,
         req: &PagestreamGetPageRequest,
     ) -> Result<PagestreamBeMessage> {
@@ -582,7 +607,7 @@ impl PageServerHandler {
         // FIXME: this profiling now happens at different place than it used to. The
         // current profiling is based on a thread-local variable, so it doesn't work
         // across awaits
-        //let _profiling_guard = profpoint_start(self.conf, ProfilingConfig::PageRequests);
+        let _profiling_guard = profpoint_start(self.conf, ProfilingConfig::PageRequests);
         let page = timeline.get_rel_page_at_lsn(req.rel, req.blkno, lsn, req.latest)?;
 
         Ok(PagestreamBeMessage::GetPage(PagestreamGetPageResponse {
@@ -590,8 +615,9 @@ impl PageServerHandler {
         }))
     }
 
-    #[instrument(skip(pgb))]
+    #[instrument(skip(self, pgb))]
     async fn handle_basebackup_request(
+        &self,
         pgb: &mut PostgresBackend,
         tenant_id: TenantId,
         timeline_id: TimelineId,
@@ -600,7 +626,7 @@ impl PageServerHandler {
         full_backup: bool,
     ) -> anyhow::Result<()> {
         // check that the timeline exists
-        let timeline = get_local_timeline(tenant_id, timeline_id)?;
+        let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
         let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
         if let Some(lsn) = lsn {
             // Backup was requested at a particular LSN. Wait for it to arrive.
@@ -647,7 +673,7 @@ impl PageServerHandler {
             .claims
             .as_ref()
             .expect("claims presence already checked");
-        auth::check_permission(claims, tenant_id)
+        check_permission(claims, tenant_id)
     }
 }
 
@@ -724,7 +750,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
             };
 
             // Check that the timeline exists
-            Self::handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, None, false)
+            self.handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, None, false)
                 .await?;
             pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
         }
@@ -742,7 +768,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
             let timeline_id = TimelineId::from_str(params[1])?;
 
             self.check_permission(Some(tenant_id))?;
-            let timeline = get_local_timeline(tenant_id, timeline_id)?;
+            let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
 
             let end_of_timeline = timeline.get_last_record_rlsn();
 
@@ -784,7 +810,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
             self.check_permission(Some(tenant_id))?;
 
             // Check that the timeline exists
-            Self::handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, prev_lsn, true)
+            self.handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, prev_lsn, true)
                 .await?;
             pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
         } else if query_string.starts_with("import basebackup ") {
@@ -865,7 +891,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
 
             self.check_permission(Some(tenant_id))?;
 
-            let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
+            let tenant = get_active_tenant_with_timeout(tenant_id).await?;
             pgb.write_message(&BeMessage::RowDescription(&[
                 RowDescriptor::int8_col(b"checkpoint_distance"),
                 RowDescriptor::int8_col(b"checkpoint_timeout"),
@@ -909,8 +935,28 @@ impl postgres_backend_async::Handler for PageServerHandler {
     }
 }
 
-fn get_local_timeline(tenant_id: TenantId, timeline_id: TimelineId) -> Result<Arc<Timeline>> {
-    tenant_mgr::get_tenant(tenant_id, true)
+/// Get active tenant.
+///
+/// If the tenant is Loading, waits for it to become Active, for up to 30 s. That
+/// ensures that queries don't fail immediately after pageserver startup, because
+/// all tenants are still loading.
+async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> Result<Arc<Tenant>> {
+    let tenant = tenant_mgr::get_tenant(tenant_id, false)?;
+    match tokio::time::timeout(Duration::from_secs(30), tenant.wait_to_become_active()).await {
+        Ok(wait_result) => wait_result
+            // no .context(), the error message is good enough and some tests depend on it
+            .map(move |()| tenant),
+        Err(_) => anyhow::bail!("Timeout waiting for tenant {tenant_id} to become Active"),
+    }
+}
+
+/// Shorthand for getting a reference to a Timeline of an Active tenant.
+async fn get_active_timeline_with_timeout(
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+) -> Result<Arc<Timeline>> {
+    get_active_tenant_with_timeout(tenant_id)
+        .await
         .and_then(|tenant| tenant.get_timeline(timeline_id, true))
 }
 

pageserver/src/repository.rs

@@ -189,7 +189,7 @@ impl Value {
 ///
 /// Result of performing GC
 ///
-#[derive(Default, Serialize)]
+#[derive(Default, Serialize, Debug)]
 pub struct GcResult {
     pub layers_total: u64,
     pub layers_needed_by_cutoff: u64,

pageserver/src/storage_sync/delete.rs

@@ -1,235 +0,0 @@
-//! Timeline synchronization logic to delete a bulk of timeline's remote files from the remote storage.
-
-use std::path::Path;
-
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use tracing::{debug, error, info};
-
-use crate::storage_sync::{SyncQueue, SyncTask};
-use remote_storage::GenericRemoteStorage;
-use utils::id::TenantTimelineId;
-
-use super::{LayersDeletion, SyncData};
-
-/// Attempts to remove the timleline layers from the remote storage.
-/// If the task had not adjusted the metadata before, the deletion will fail.
-pub(super) async fn delete_timeline_layers(
-    storage: &GenericRemoteStorage,
-    sync_queue: &SyncQueue,
-    sync_id: TenantTimelineId,
-    mut delete_data: SyncData<LayersDeletion>,
-) -> bool {
-    if !delete_data.data.deletion_registered {
-        error!("Cannot delete timeline layers before the deletion metadata is not registered, reenqueueing");
-        delete_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Delete(delete_data));
-        return false;
-    }
-
-    if delete_data.data.layers_to_delete.is_empty() {
-        info!("No layers to delete, skipping");
-        return true;
-    }
-
-    let layers_to_delete = delete_data
-        .data
-        .layers_to_delete
-        .drain()
-        .collect::<Vec<_>>();
-    debug!("Layers to delete: {layers_to_delete:?}");
-    info!("Deleting {} timeline layers", layers_to_delete.len());
-
-    let mut delete_tasks = layers_to_delete
-        .into_iter()
-        .map(|local_layer_path| async {
-            match remove_storage_object(storage, &local_layer_path).await {
-                Ok(()) => Ok(local_layer_path),
-                Err(e) => Err((e, local_layer_path)),
-            }
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errored = false;
-    while let Some(deletion_result) = delete_tasks.next().await {
-        match deletion_result {
-            Ok(local_layer_path) => {
-                debug!(
-                    "Successfully deleted layer {} for timeline {sync_id}",
-                    local_layer_path.display()
-                );
-                delete_data.data.deleted_layers.insert(local_layer_path);
-            }
-            Err((e, local_layer_path)) => {
-                errored = true;
-                error!(
-                    "Failed to delete layer {} for timeline {sync_id}: {e:?}",
-                    local_layer_path.display()
-                );
-                delete_data.data.layers_to_delete.insert(local_layer_path);
-            }
-        }
-    }
-
-    if errored {
-        debug!("Reenqueuing failed delete task for timeline {sync_id}");
-        delete_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Delete(delete_data));
-    } else {
-        info!("Successfully deleted all layers");
-    }
-    errored
-}
-
-async fn remove_storage_object(
-    storage: &GenericRemoteStorage,
-    local_layer_path: &Path,
-) -> anyhow::Result<()> {
-    let storage_path = storage
-        .remote_object_id(local_layer_path)
-        .with_context(|| {
-            format!(
-                "Failed to get the layer storage path for local path '{}'",
-                local_layer_path.display()
-            )
-        })?;
-
-    storage.delete(&storage_path).await.with_context(|| {
-        format!(
-            "Failed to delete remote layer from storage at '{:?}'",
-            storage_path
-        )
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{collections::HashSet, num::NonZeroUsize};
-
-    use itertools::Itertools;
-    use tempfile::tempdir;
-    use tokio::fs;
-    use utils::lsn::Lsn;
-
-    use crate::{
-        storage_sync::test_utils::{create_local_timeline, dummy_metadata},
-        tenant::harness::{TenantHarness, TIMELINE_ID},
-    };
-    use remote_storage::{LocalFs, RemoteStorage};
-
-    use super::*;
-
-    #[tokio::test]
-    async fn delete_timeline_negative() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("delete_timeline_negative")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_path_buf(),
-            harness.conf.workdir.clone(),
-        )?);
-
-        let deleted = delete_timeline_layers(
-            &storage,
-            &sync_queue,
-            sync_id,
-            SyncData {
-                retries: 1,
-                data: LayersDeletion {
-                    deleted_layers: HashSet::new(),
-                    layers_to_delete: HashSet::new(),
-                    deletion_registered: false,
-                },
-            },
-        )
-        .await;
-
-        assert!(
-            !deleted,
-            "Should not start the deletion for task with delete metadata unregistered"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn delete_timeline() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("delete_timeline")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let layer_files = ["a", "b", "c", "d"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_path_buf(),
-            harness.conf.workdir.clone(),
-        )?);
-
-        let local_storage = storage.as_local().unwrap();
-
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-        for (local_path, _metadata) in timeline_upload.layers_to_upload {
-            let remote_path =
-                local_storage.resolve_in_storage(&local_storage.remote_object_id(&local_path)?)?;
-            let remote_parent_dir = remote_path.parent().unwrap();
-            if !remote_parent_dir.exists() {
-                fs::create_dir_all(&remote_parent_dir).await?;
-            }
-            fs::copy(&local_path, &remote_path).await?;
-        }
-        assert_eq!(
-            local_storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|remote_path| local_storage.local_path(&remote_path).unwrap())
-                .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
-                .sorted()
-                .collect::<Vec<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_str| layer_str.to_string())
-                .sorted()
-                .collect::<Vec<_>>(),
-            "Expect to have all layer files remotely before deletion"
-        );
-
-        let deleted = delete_timeline_layers(
-            &storage,
-            &sync_queue,
-            sync_id,
-            SyncData {
-                retries: current_retries,
-                data: LayersDeletion {
-                    deleted_layers: HashSet::new(),
-                    layers_to_delete: HashSet::from([
-                        local_timeline_path.join("a"),
-                        local_timeline_path.join("c"),
-                        local_timeline_path.join("something_different"),
-                    ]),
-                    deletion_registered: true,
-                },
-            },
-        )
-        .await;
-        assert!(deleted, "Should be able to delete timeline files");
-
-        assert_eq!(
-            local_storage
-                .list()
-                .await?
-                .into_iter()
-                .map(|remote_path| local_storage.local_path(&remote_path).unwrap())
-                .filter_map(|local_path| { Some(local_path.file_name()?.to_str()?.to_owned()) })
-                .sorted()
-                .collect::<Vec<_>>(),
-            vec!["b".to_string(), "d".to_string()],
-            "Expect to have only non-deleted files remotely"
-        );
-
-        Ok(())
-    }
-}

pageserver/src/storage_sync/download.rs

@@ -1,695 +0,0 @@
-//! Timeline synchronization logic to fetch the layer files from remote storage into pageserver's local directory.
-
-use std::{
-    collections::{HashMap, HashSet},
-    fmt::Debug,
-    mem,
-    path::Path,
-};
-
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use remote_storage::{DownloadError, GenericRemoteStorage};
-use tokio::{
-    fs,
-    io::{self, AsyncWriteExt},
-};
-use tracing::{debug, error, info, warn};
-
-use crate::{
-    config::PageServerConf,
-    storage_sync::{index::LayerFileMetadata, SyncTask},
-    TEMP_FILE_SUFFIX,
-};
-use utils::{
-    crashsafe::path_with_suffix_extension,
-    id::{TenantId, TenantTimelineId, TimelineId},
-};
-
-use super::{
-    index::{IndexPart, RemoteTimeline},
-    LayersDownload, SyncData, SyncQueue,
-};
-
-// We collect timelines remotely available for each tenant
-// in case we failed to gather all index parts (due to an error)
-// Poisoned variant is returned.
-// When data is received succesfully without errors Present variant is used.
-pub enum TenantIndexParts {
-    Poisoned {
-        present: HashMap<TimelineId, IndexPart>,
-        missing: HashSet<TimelineId>,
-    },
-    Present(HashMap<TimelineId, IndexPart>),
-}
-
-impl TenantIndexParts {
-    fn add_poisoned(&mut self, timeline_id: TimelineId) {
-        match self {
-            TenantIndexParts::Poisoned { missing, .. } => {
-                missing.insert(timeline_id);
-            }
-            TenantIndexParts::Present(present) => {
-                *self = TenantIndexParts::Poisoned {
-                    present: mem::take(present),
-                    missing: HashSet::from([timeline_id]),
-                }
-            }
-        }
-    }
-}
-
-impl Default for TenantIndexParts {
-    fn default() -> Self {
-        TenantIndexParts::Present(HashMap::default())
-    }
-}
-
-pub async fn download_index_parts(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    keys: HashSet<TenantTimelineId>,
-) -> HashMap<TenantId, TenantIndexParts> {
-    let mut index_parts: HashMap<TenantId, TenantIndexParts> = HashMap::new();
-
-    let mut part_downloads = keys
-        .into_iter()
-        .map(|id| async move { (id, download_index_part(conf, storage, id).await) })
-        .collect::<FuturesUnordered<_>>();
-
-    while let Some((id, part_upload_result)) = part_downloads.next().await {
-        match part_upload_result {
-            Ok(index_part) => {
-                debug!("Successfully fetched index part for {id}");
-                match index_parts.entry(id.tenant_id).or_default() {
-                    TenantIndexParts::Poisoned { present, .. } => {
-                        present.insert(id.timeline_id, index_part);
-                    }
-                    TenantIndexParts::Present(parts) => {
-                        parts.insert(id.timeline_id, index_part);
-                    }
-                }
-            }
-            Err(download_error) => {
-                match download_error {
-                    DownloadError::NotFound => {
-                        // thats ok because it means that we didnt upload something we have locally for example
-                    }
-                    e => {
-                        let tenant_parts = index_parts.entry(id.tenant_id).or_default();
-                        tenant_parts.add_poisoned(id.timeline_id);
-                        error!(
-                            "Failed to fetch index part for {id}: {e} poisoning tenant index parts"
-                        );
-                    }
-                }
-            }
-        }
-    }
-
-    index_parts
-}
-
-/// Note: The function is rather expensive from s3 access point of view, it will execute ceil(N/1000) + N requests.
-/// At least one request to obtain a list of tenant timelines (more requests is there are more than 1000 timelines).
-/// And then will attempt to download all index files that belong to these timelines.
-pub async fn gather_tenant_timelines_index_parts(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    tenant_id: TenantId,
-) -> anyhow::Result<HashMap<TimelineId, IndexPart>> {
-    let tenant_path = conf.timelines_path(&tenant_id);
-    let timeline_sync_ids = get_timeline_sync_ids(storage, &tenant_path, tenant_id)
-        .await
-        .with_context(|| format!("Failed to list timeline sync ids for tenat {tenant_id}"))?;
-
-    match download_index_parts(conf, storage, timeline_sync_ids)
-        .await
-        .remove(&tenant_id)
-        .ok_or_else(|| anyhow::anyhow!("Missing tenant index parts. This is a bug."))?
-    {
-        TenantIndexParts::Poisoned { missing, .. } => {
-            anyhow::bail!("Failed to download index parts for all timelines. Missing {missing:?}")
-        }
-        TenantIndexParts::Present(parts) => Ok(parts),
-    }
-}
-
-/// Retrieves index data from the remote storage for a given timeline.
-async fn download_index_part(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    sync_id: TenantTimelineId,
-) -> Result<IndexPart, DownloadError> {
-    let index_part_path = conf
-        .metadata_path(sync_id.timeline_id, sync_id.tenant_id)
-        .with_file_name(IndexPart::FILE_NAME);
-    let mut index_part_download = storage
-        .download_storage_object(None, &index_part_path)
-        .await?;
-
-    let mut index_part_bytes = Vec::new();
-    io::copy(
-        &mut index_part_download.download_stream,
-        &mut index_part_bytes,
-    )
-    .await
-    .with_context(|| {
-        format!(
-            "Failed to download an index part into file '{}'",
-            index_part_path.display()
-        )
-    })
-    .map_err(DownloadError::Other)?;
-
-    let index_part: IndexPart = serde_json::from_slice(&index_part_bytes)
-        .with_context(|| {
-            format!(
-                "Failed to deserialize index part file into file '{}'",
-                index_part_path.display()
-            )
-        })
-        .map_err(DownloadError::Other)?;
-
-    let missing_files = index_part.missing_files();
-    if !missing_files.is_empty() {
-        warn!("Found missing layers in index part for timeline {sync_id}: {missing_files:?}");
-    }
-
-    Ok(index_part)
-}
-
-/// Timeline download result, with extra data, needed for downloading.
-#[derive(Debug)]
-pub(super) enum DownloadedTimeline {
-    /// Remote timeline data is either absent or corrupt, no download possible.
-    Abort,
-    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
-    /// Initial download failed due to some error, the download task is rescheduled for another retry.
-    FailedAndRescheduled,
-    /// Remote timeline data is found, its latest checkpoint's metadata contents (disk_consistent_lsn) is known.
-    /// Initial download successful.
-    Successful(SyncData<LayersDownload>),
-}
-
-/// Attempts to download all given timeline's layers.
-/// Timeline files that already exist locally are skipped during the download, but the local metadata file is
-/// updated in the end, if the remote one contains a newer disk_consistent_lsn.
-///
-/// On an error, bumps the retries count and updates the files to skip with successful downloads, rescheduling the task.
-pub(super) async fn download_timeline_layers<'a>(
-    conf: &'static PageServerConf,
-    storage: &'a GenericRemoteStorage,
-    sync_queue: &'a SyncQueue,
-    remote_timeline: Option<&'a RemoteTimeline>,
-    sync_id: TenantTimelineId,
-    mut download_data: SyncData<LayersDownload>,
-) -> DownloadedTimeline {
-    let remote_timeline = match remote_timeline {
-        Some(remote_timeline) => {
-            if !remote_timeline.awaits_download {
-                error!("Timeline with sync id {sync_id} is not awaiting download");
-                return DownloadedTimeline::Abort;
-            }
-            remote_timeline
-        }
-        None => {
-            error!("Timeline with sync id {sync_id} is not present in the remote index");
-            return DownloadedTimeline::Abort;
-        }
-    };
-
-    let download = &mut download_data.data;
-
-    let layers_to_download = remote_timeline
-        .stored_files()
-        .iter()
-        .filter_map(|(layer_path, metadata)| {
-            if !download.layers_to_skip.contains(layer_path) {
-                Some((layer_path.to_owned(), metadata.to_owned()))
-            } else {
-                None
-            }
-        })
-        .collect::<Vec<_>>();
-
-    debug!("Layers to download: {layers_to_download:?}");
-    info!("Downloading {} timeline layers", layers_to_download.len());
-
-    if layers_to_download.is_empty() {
-        info!("No layers to download after filtering, skipping");
-        return DownloadedTimeline::Successful(download_data);
-    }
-
-    let mut download_tasks = layers_to_download
-        .into_iter()
-        .map(|(layer_destination_path, metadata)| async move {
-
-            match layer_destination_path.metadata() {
-                Ok(m) if m.is_file() => {
-                    // the file exists from earlier round when we failed after renaming it as
-                    // layer_destination_path
-                    let verified = if let Some(expected) = metadata.file_size() {
-                        m.len() == expected
-                    } else {
-                        // behaviour before recording metadata was to accept any existing
-                        true
-                    };
-
-                    if verified {
-                        debug!(
-                            "Layer already exists locally, skipping download: {}",
-                            layer_destination_path.display()
-                        );
-                        return Ok((layer_destination_path, LayerFileMetadata::new(m.len())))
-                    } else {
-                        // no need to remove it, it will be overwritten by fs::rename
-                        // after successful download
-                        warn!("Downloaded layer exists already but layer file metadata mismatches: {}, metadata {:?}", layer_destination_path.display(), metadata);
-                    }
-                }
-                Ok(m) => {
-                    return Err(anyhow::anyhow!("Downloaded layer destination exists but is not a file: {m:?}, target needs to be removed/archived manually: {layer_destination_path:?}"));
-                }
-                Err(_) => {
-                    // behave as the file didn't exist
-                }
-            }
-
-            // Perform a rename inspired by durable_rename from file_utils.c.
-            // The sequence:
-            //     write(tmp)
-            //     fsync(tmp)
-            //     rename(tmp, new)
-            //     fsync(new)
-            //     fsync(parent)
-            // For more context about durable_rename check this email from postgres mailing list:
-            // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
-            // If pageserver crashes the temp file will be deleted on startup and re-downloaded.
-            let temp_file_path =
-                path_with_suffix_extension(&layer_destination_path, TEMP_FILE_SUFFIX);
-
-            // TODO: this doesn't use the cached fd for some reason?
-            let mut destination_file =
-                fs::File::create(&temp_file_path).await.with_context(|| {
-                    format!(
-                        "Failed to create a destination file for layer '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            let mut layer_download = storage.download_storage_object(None, &layer_destination_path)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to initiate the download the layer for {sync_id} into file '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            let bytes_amount = io::copy(&mut layer_download.download_stream, &mut destination_file)
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to download the layer for {sync_id} into file '{}'",
-                        temp_file_path.display()
-                    )
-                })?;
-
-            // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
-            // A file will not be closed immediately when it goes out of scope if there are any IO operations
-            // that have not yet completed. To ensure that a file is closed immediately when it is dropped,
-            // you should call flush before dropping it.
-            //
-            // From the tokio code I see that it waits for pending operations to complete. There shouldn't be any because
-            // we assume that `destination_file` file is fully written. I.e there is no pending .write(...).await operations.
-            // But for additional safety let's check/wait for any pending operations.
-            destination_file.flush().await.with_context(|| {
-                format!(
-                    "failed to flush source file at {}",
-                    temp_file_path.display()
-                )
-            })?;
-
-            match metadata.file_size() {
-                Some(expected) if expected != bytes_amount => {
-                    anyhow::bail!(
-                        "According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'",
-                        temp_file_path.display()
-                    );
-                },
-                Some(_) | None => {
-                    // matches, or upgrading from an earlier IndexPart version
-                }
-            }
-
-            // not using sync_data because it can lose file size update
-            destination_file.sync_all().await.with_context(|| {
-                format!(
-                    "failed to fsync source file at {}",
-                    temp_file_path.display()
-                )
-            })?;
-            drop(destination_file);
-
-            fail::fail_point!("remote-storage-download-pre-rename", |_| {
-                anyhow::bail!("remote-storage-download-pre-rename failpoint triggered")
-            });
-
-            fs::rename(&temp_file_path, &layer_destination_path).await?;
-
-            fsync_path(&layer_destination_path).await.with_context(|| {
-                format!(
-                    "Cannot fsync layer destination path {}",
-                    layer_destination_path.display(),
-                )
-            })?;
-
-            Ok::<_, anyhow::Error>((layer_destination_path, LayerFileMetadata::new(bytes_amount)))
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errors_happened = false;
-    // keep files we've downloaded to remove them from layers_to_skip if directory fsync fails
-    let mut undo = HashSet::new();
-    while let Some(download_result) = download_tasks.next().await {
-        match download_result {
-            Ok((downloaded_path, metadata)) => {
-                undo.insert(downloaded_path.clone());
-                download.layers_to_skip.insert(downloaded_path.clone());
-                // what if the key existed already? ignore, because then we would had
-                // downloaded a partial file, and had to retry
-                download.gathered_metadata.insert(downloaded_path, metadata);
-            }
-            Err(e) => {
-                errors_happened = true;
-                error!("Failed to download a layer for timeline {sync_id}: {e:?}");
-            }
-        }
-    }
-
-    // fsync timeline directory which is a parent directory for downloaded files
-    let TenantTimelineId {
-        tenant_id,
-        timeline_id,
-    } = &sync_id;
-    let timeline_dir = conf.timeline_path(timeline_id, tenant_id);
-    if let Err(e) = fsync_path(&timeline_dir).await {
-        error!(
-            "Cannot fsync parent directory {} error {}",
-            timeline_dir.display(),
-            e
-        );
-        for item in undo {
-            download.layers_to_skip.remove(&item);
-            // intentionally don't clear the gathered_metadata because it exists for fsync_path
-            // failure on parent directory
-        }
-        errors_happened = true;
-    }
-
-    if errors_happened {
-        debug!("Reenqueuing failed download task for timeline {sync_id}");
-        download_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Download(download_data));
-        DownloadedTimeline::FailedAndRescheduled
-    } else {
-        info!("Successfully downloaded all layers");
-        DownloadedTimeline::Successful(download_data)
-    }
-}
-
-async fn get_timeline_sync_ids(
-    storage: &GenericRemoteStorage,
-    tenant_path: &Path,
-    tenant_id: TenantId,
-) -> anyhow::Result<HashSet<TenantTimelineId>> {
-    let tenant_storage_path = storage.remote_object_id(tenant_path).with_context(|| {
-        format!(
-            "Failed to get tenant storage path for local path '{}'",
-            tenant_path.display()
-        )
-    })?;
-
-    let timelines = storage
-        .list_prefixes(Some(&tenant_storage_path))
-        .await
-        .with_context(|| {
-            format!(
-                "Failed to list tenant storage path {tenant_storage_path:?} to get remote timelines to download"
-            )
-        })?;
-
-    if timelines.is_empty() {
-        anyhow::bail!("no timelines found on the remote storage")
-    }
-
-    let mut sync_ids = HashSet::new();
-
-    for timeline_remote_storage_key in timelines {
-        let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| {
-            anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}")
-        })?;
-
-        let timeline_id: TimelineId = object_name.parse().with_context(|| {
-            format!("failed to parse object name into timeline id '{object_name}'")
-        })?;
-
-        sync_ids.insert(TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        });
-    }
-
-    Ok(sync_ids)
-}
-
-async fn fsync_path(path: impl AsRef<Path>) -> Result<(), io::Error> {
-    fs::File::open(path).await?.sync_all().await
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{
-        collections::{BTreeSet, HashSet},
-        num::NonZeroUsize,
-        path::PathBuf,
-    };
-
-    use remote_storage::{LocalFs, RemoteStorage};
-    use tempfile::tempdir;
-    use utils::lsn::Lsn;
-
-    use crate::{
-        storage_sync::{
-            index::RelativePath,
-            test_utils::{create_local_timeline, dummy_metadata},
-        },
-        tenant::harness::{TenantHarness, TIMELINE_ID},
-    };
-
-    use super::*;
-
-    #[tokio::test]
-    async fn download_timeline() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("download_timeline")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let layer_files = ["a", "b", "layer_to_skip", "layer_to_keep_locally"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-
-        for local_path in timeline_upload.layers_to_upload.keys() {
-            let remote_path =
-                local_storage.resolve_in_storage(&storage.remote_object_id(local_path)?)?;
-            let remote_parent_dir = remote_path.parent().unwrap();
-            if !remote_parent_dir.exists() {
-                fs::create_dir_all(&remote_parent_dir).await?;
-            }
-            fs::copy(&local_path, &remote_path).await?;
-        }
-        let mut read_dir = fs::read_dir(&local_timeline_path).await?;
-        while let Some(dir_entry) = read_dir.next_entry().await? {
-            if dir_entry.file_name().to_str() == Some("layer_to_keep_locally") {
-                continue;
-            } else {
-                fs::remove_file(dir_entry.path()).await?;
-            }
-        }
-
-        let mut remote_timeline = RemoteTimeline::new(metadata.clone());
-        remote_timeline.awaits_download = true;
-        remote_timeline.add_timeline_layers(layer_files.iter().map(|layer| {
-            let layer_path = local_timeline_path.join(layer);
-
-            // this could had also been LayerFileMetadata::default(), but since in this test we
-            // don't do the merge operation done by storage_sync::download_timeline_data, it would
-            // not be merged back to timeline.
-            let metadata_from_upload = timeline_upload
-                .layers_to_upload
-                .get(&layer_path)
-                .expect("layer must exist in previously uploaded paths")
-                .to_owned();
-            (layer_path, metadata_from_upload)
-        }));
-
-        let download_data = match download_timeline_layers(
-            harness.conf,
-            &storage,
-            &sync_queue,
-            Some(&remote_timeline),
-            sync_id,
-            SyncData::new(
-                current_retries,
-                LayersDownload::from_skipped_layers(HashSet::from([
-                    local_timeline_path.join("layer_to_skip")
-                ])),
-            ),
-        )
-        .await
-        {
-            DownloadedTimeline::Successful(data) => data,
-            wrong_result => {
-                panic!("Expected a successful download for timeline, but got: {wrong_result:?}")
-            }
-        };
-
-        assert_eq!(
-            current_retries, download_data.retries,
-            "On successful download, retries are not expected to change"
-        );
-        assert_eq!(
-            download_data
-                .data
-                .layers_to_skip
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer| local_timeline_path.join(layer))
-                .collect(),
-            "On successful download, layers to skip should contain all downloaded files and present layers that were skipped"
-        );
-
-        let mut downloaded_files = BTreeSet::new();
-        let mut read_dir = fs::read_dir(&local_timeline_path).await?;
-        while let Some(dir_entry) = read_dir.next_entry().await? {
-            downloaded_files.insert(dir_entry.path());
-        }
-
-        assert_eq!(
-            downloaded_files,
-            layer_files
-                .iter()
-                .filter(|layer| layer != &&"layer_to_skip")
-                .map(|layer| local_timeline_path.join(layer))
-                .collect(),
-            "On successful download, all layers that were not skipped, should be downloaded"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn download_timeline_negatives() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("download_timeline_negatives")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-
-        let empty_remote_timeline_download = download_timeline_layers(
-            harness.conf,
-            &storage,
-            &sync_queue,
-            None,
-            sync_id,
-            SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
-        )
-        .await;
-        assert!(
-            matches!(empty_remote_timeline_download, DownloadedTimeline::Abort),
-            "Should not allow downloading for empty remote timeline"
-        );
-
-        let not_expecting_download_remote_timeline = RemoteTimeline::new(dummy_metadata(Lsn(5)));
-        assert!(
-            !not_expecting_download_remote_timeline.awaits_download,
-            "Should not expect download for the timeline"
-        );
-        let already_downloading_remote_timeline_download = download_timeline_layers(
-            harness.conf,
-            &storage,
-            &sync_queue,
-            Some(&not_expecting_download_remote_timeline),
-            sync_id,
-            SyncData::new(0, LayersDownload::from_skipped_layers(HashSet::new())),
-        )
-        .await;
-        assert!(
-            matches!(
-                already_downloading_remote_timeline_download,
-                DownloadedTimeline::Abort,
-            ),
-            "Should not allow downloading for remote timeline that does not expect it"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_download_index_part() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("test_download_index_part")?;
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-
-        let index_part = IndexPart::new(
-            HashSet::from([
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
-            ]),
-            HashSet::from([RelativePath::new(
-                &local_timeline_path,
-                local_timeline_path.join("three"),
-            )?]),
-            metadata.disk_consistent_lsn(),
-            metadata.to_bytes()?,
-        );
-
-        let local_index_part_path = harness
-            .conf
-            .metadata_path(sync_id.timeline_id, sync_id.tenant_id)
-            .with_file_name(IndexPart::FILE_NAME);
-        let index_part_remote_id = local_storage.remote_object_id(&local_index_part_path)?;
-        let index_part_local_path = PathBuf::from(index_part_remote_id.to_string());
-        fs::create_dir_all(index_part_local_path.parent().unwrap()).await?;
-        fs::write(&index_part_local_path, serde_json::to_vec(&index_part)?).await?;
-
-        let downloaded_index_part = download_index_part(harness.conf, &storage, sync_id).await?;
-
-        assert_eq!(
-            downloaded_index_part, index_part,
-            "Downloaded index part should be the same as the one in storage"
-        );
-
-        Ok(())
-    }
-}

pageserver/src/storage_sync/index.rs

@@ -1,712 +0,0 @@
-//! In-memory index to track the tenant files on the remote storage.
-//! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
-//! remote timeline layers and its metadata.
-
-use std::ops::{Deref, DerefMut};
-use std::{
-    collections::{HashMap, HashSet},
-    path::{Path, PathBuf},
-    sync::Arc,
-};
-
-use anyhow::{anyhow, Context, Ok};
-use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
-use tokio::sync::RwLock;
-use tracing::log::warn;
-
-use crate::{config::PageServerConf, tenant::metadata::TimelineMetadata};
-use utils::{
-    id::{TenantId, TenantTimelineId, TimelineId},
-    lsn::Lsn,
-};
-
-use super::download::TenantIndexParts;
-
-/// A part of the filesystem path, that needs a root to become a path again.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct RelativePath(String);
-
-impl RelativePath {
-    /// Attempts to strip off the base from path, producing a relative path or an error.
-    pub fn new<P: AsRef<Path>>(base: &Path, path: P) -> anyhow::Result<Self> {
-        let path = path.as_ref();
-        let relative = path.strip_prefix(base).with_context(|| {
-            format!(
-                "path '{}' is not relative to base '{}'",
-                path.display(),
-                base.display()
-            )
-        })?;
-        Ok(RelativePath(relative.to_string_lossy().to_string()))
-    }
-
-    /// Joins the relative path with the base path.
-    fn as_path(&self, base: &Path) -> PathBuf {
-        base.join(&self.0)
-    }
-}
-
-#[derive(Debug, Clone, Default)]
-pub struct TenantEntry(HashMap<TimelineId, RemoteTimeline>);
-
-impl TenantEntry {
-    pub fn has_in_progress_downloads(&self) -> bool {
-        self.values()
-            .any(|remote_timeline| remote_timeline.awaits_download)
-    }
-}
-
-impl Deref for TenantEntry {
-    type Target = HashMap<TimelineId, RemoteTimeline>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for TenantEntry {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-impl From<HashMap<TimelineId, RemoteTimeline>> for TenantEntry {
-    fn from(inner: HashMap<TimelineId, RemoteTimeline>) -> Self {
-        Self(inner)
-    }
-}
-
-/// An index to track tenant files that exist on the remote storage.
-#[derive(Debug, Clone, Default)]
-pub struct RemoteTimelineIndex {
-    entries: HashMap<TenantId, TenantEntry>,
-}
-
-/// A wrapper to synchronize the access to the index, should be created and used before dealing with any [`RemoteTimelineIndex`].
-#[derive(Default)]
-pub struct RemoteIndex(Arc<RwLock<RemoteTimelineIndex>>);
-
-impl RemoteIndex {
-    pub fn from_parts(
-        conf: &'static PageServerConf,
-        index_parts: HashMap<TenantId, TenantIndexParts>,
-    ) -> anyhow::Result<Self> {
-        let mut entries: HashMap<TenantId, TenantEntry> = HashMap::new();
-
-        for (tenant_id, index_parts) in index_parts {
-            match index_parts {
-                // TODO: should we schedule a retry so it can be recovered? otherwise we can revive it only through detach/attach or pageserver restart
-                TenantIndexParts::Poisoned { missing, ..} => warn!("skipping tenant_id set up for remote index because the index download has failed for timeline(s): {missing:?}"),
-                TenantIndexParts::Present(timelines) => {
-                    for (timeline_id, index_part) in timelines {
-                        let timeline_path = conf.timeline_path(&timeline_id, &tenant_id);
-                        let remote_timeline =
-                            RemoteTimeline::from_index_part(&timeline_path, index_part)
-                                .context("Failed to restore remote timeline data from index part")?;
-
-                        entries
-                            .entry(tenant_id)
-                            .or_default()
-                            .insert(timeline_id, remote_timeline);
-                    }
-                },
-            }
-        }
-
-        Ok(Self(Arc::new(RwLock::new(RemoteTimelineIndex { entries }))))
-    }
-
-    pub async fn read(&self) -> tokio::sync::RwLockReadGuard<'_, RemoteTimelineIndex> {
-        self.0.read().await
-    }
-
-    pub async fn write(&self) -> tokio::sync::RwLockWriteGuard<'_, RemoteTimelineIndex> {
-        self.0.write().await
-    }
-}
-
-impl Clone for RemoteIndex {
-    fn clone(&self) -> Self {
-        Self(Arc::clone(&self.0))
-    }
-}
-
-impl RemoteTimelineIndex {
-    pub fn timeline_entry(
-        &self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: &TenantTimelineId,
-    ) -> Option<&RemoteTimeline> {
-        self.entries.get(tenant_id)?.get(timeline_id)
-    }
-
-    pub fn timeline_entry_mut(
-        &mut self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: &TenantTimelineId,
-    ) -> Option<&mut RemoteTimeline> {
-        self.entries.get_mut(tenant_id)?.get_mut(timeline_id)
-    }
-
-    pub fn add_timeline_entry(
-        &mut self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: TenantTimelineId,
-        entry: RemoteTimeline,
-    ) {
-        self.entries
-            .entry(tenant_id)
-            .or_default()
-            .insert(timeline_id, entry);
-    }
-
-    pub fn remove_timeline_entry(
-        &mut self,
-        TenantTimelineId {
-            tenant_id,
-            timeline_id,
-        }: TenantTimelineId,
-    ) -> Option<RemoteTimeline> {
-        self.entries
-            .entry(tenant_id)
-            .or_default()
-            .remove(&timeline_id)
-    }
-
-    pub fn tenant_entry(&self, tenant_id: &TenantId) -> Option<&TenantEntry> {
-        self.entries.get(tenant_id)
-    }
-
-    pub fn tenant_entry_mut(&mut self, tenant_id: &TenantId) -> Option<&mut TenantEntry> {
-        self.entries.get_mut(tenant_id)
-    }
-
-    pub fn add_tenant_entry(&mut self, tenant_id: TenantId) -> &mut TenantEntry {
-        self.entries.entry(tenant_id).or_default()
-    }
-
-    pub fn remove_tenant_entry(&mut self, tenant_id: &TenantId) -> Option<TenantEntry> {
-        self.entries.remove(tenant_id)
-    }
-
-    pub fn set_awaits_download(
-        &mut self,
-        id: &TenantTimelineId,
-        awaits_download: bool,
-    ) -> anyhow::Result<()> {
-        self.timeline_entry_mut(id)
-            .ok_or_else(|| anyhow!("unknown timeline sync {id}"))?
-            .awaits_download = awaits_download;
-        Ok(())
-    }
-}
-
-/// Restored index part data about the timeline, stored in the remote index.
-#[derive(Debug, Clone)]
-pub struct RemoteTimeline {
-    timeline_layers: HashMap<PathBuf, LayerFileMetadata>,
-    missing_layers: HashMap<PathBuf, LayerFileMetadata>,
-
-    pub metadata: TimelineMetadata,
-    pub awaits_download: bool,
-}
-
-impl RemoteTimeline {
-    pub fn new(metadata: TimelineMetadata) -> Self {
-        Self {
-            timeline_layers: HashMap::default(),
-            missing_layers: HashMap::default(),
-            metadata,
-            awaits_download: false,
-        }
-    }
-
-    pub fn add_timeline_layers(
-        &mut self,
-        new_layers: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
-    ) {
-        self.timeline_layers.extend(new_layers);
-    }
-
-    pub fn add_upload_failures(
-        &mut self,
-        upload_failures: impl IntoIterator<Item = (PathBuf, LayerFileMetadata)>,
-    ) {
-        self.missing_layers.extend(upload_failures);
-    }
-
-    pub fn remove_layers(&mut self, layers_to_remove: &HashSet<PathBuf>) {
-        self.timeline_layers
-            .retain(|layer, _| !layers_to_remove.contains(layer));
-        self.missing_layers
-            .retain(|layer, _| !layers_to_remove.contains(layer));
-    }
-
-    /// Lists all layer files in the given remote timeline. Omits the metadata file.
-    pub fn stored_files(&self) -> &HashMap<PathBuf, LayerFileMetadata> {
-        &self.timeline_layers
-    }
-
-    /// Combines metadata gathered or verified during downloading needed layer files to metadata on
-    /// the [`RemoteIndex`], so it can be uploaded later.
-    pub fn merge_metadata_from_downloaded(
-        &mut self,
-        downloaded: &HashMap<PathBuf, LayerFileMetadata>,
-    ) {
-        downloaded.iter().for_each(|(path, metadata)| {
-            if let Some(upgraded) = self.timeline_layers.get_mut(path) {
-                upgraded.merge(metadata);
-            }
-        });
-    }
-
-    pub fn from_index_part(timeline_path: &Path, index_part: IndexPart) -> anyhow::Result<Self> {
-        let metadata = TimelineMetadata::from_bytes(&index_part.metadata_bytes)?;
-        let default_metadata = &IndexLayerMetadata::default();
-
-        let find_metadata = |key: &RelativePath| -> LayerFileMetadata {
-            index_part
-                .layer_metadata
-                .get(key)
-                .unwrap_or(default_metadata)
-                .into()
-        };
-
-        Ok(Self {
-            timeline_layers: index_part
-                .timeline_layers
-                .iter()
-                .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
-                .collect(),
-            missing_layers: index_part
-                .missing_layers
-                .iter()
-                .map(|layer_path| (layer_path.as_path(timeline_path), find_metadata(layer_path)))
-                .collect(),
-            metadata,
-            awaits_download: false,
-        })
-    }
-}
-
-/// Metadata gathered for each of the layer files.
-///
-/// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
-/// might have less or more metadata depending if upgrading or rolling back an upgrade.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-#[cfg_attr(test, derive(Default))]
-pub struct LayerFileMetadata {
-    file_size: Option<u64>,
-}
-
-impl From<&'_ IndexLayerMetadata> for LayerFileMetadata {
-    fn from(other: &IndexLayerMetadata) -> Self {
-        LayerFileMetadata {
-            file_size: other.file_size,
-        }
-    }
-}
-
-impl LayerFileMetadata {
-    pub fn new(file_size: u64) -> Self {
-        LayerFileMetadata {
-            file_size: Some(file_size),
-        }
-    }
-
-    pub fn file_size(&self) -> Option<u64> {
-        self.file_size
-    }
-
-    /// Metadata has holes due to version upgrades. This method is called to upgrade self with the
-    /// other value.
-    ///
-    /// This is called on the possibly outdated version.
-    pub fn merge(&mut self, other: &Self) {
-        self.file_size = other.file_size.or(self.file_size);
-    }
-}
-
-/// Part of the remote index, corresponding to a certain timeline.
-/// Contains the data about all files in the timeline, present remotely and its metadata.
-///
-/// This type needs to be backwards and forwards compatible. When changing the fields,
-/// remember to add a test case for the changed version.
-#[serde_as]
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
-pub struct IndexPart {
-    /// Debugging aid describing the version of this type.
-    #[serde(default)]
-    version: usize,
-
-    /// Each of the layers present on remote storage.
-    ///
-    /// Additional metadata can might exist in `layer_metadata`.
-    timeline_layers: HashSet<RelativePath>,
-
-    /// Currently is not really used in pageserver,
-    /// present to manually keep track of the layer files that pageserver might never retrieve.
-    ///
-    /// Such "holes" might appear if any upload task was evicted on an error threshold:
-    /// the this layer will only be rescheduled for upload on pageserver restart.
-    missing_layers: HashSet<RelativePath>,
-
-    /// Per layer file metadata, which can be present for a present or missing layer file.
-    ///
-    /// Older versions of `IndexPart` will not have this property or have only a part of metadata
-    /// that latest version stores.
-    #[serde(default)]
-    layer_metadata: HashMap<RelativePath, IndexLayerMetadata>,
-
-    #[serde_as(as = "DisplayFromStr")]
-    disk_consistent_lsn: Lsn,
-    metadata_bytes: Vec<u8>,
-}
-
-impl IndexPart {
-    /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
-    /// used to understand later versions.
-    ///
-    /// Version is currently informative only.
-    const LATEST_VERSION: usize = 1;
-    pub const FILE_NAME: &'static str = "index_part.json";
-
-    #[cfg(test)]
-    pub fn new(
-        timeline_layers: HashSet<RelativePath>,
-        missing_layers: HashSet<RelativePath>,
-        disk_consistent_lsn: Lsn,
-        metadata_bytes: Vec<u8>,
-    ) -> Self {
-        Self {
-            version: Self::LATEST_VERSION,
-            timeline_layers,
-            missing_layers,
-            layer_metadata: HashMap::default(),
-            disk_consistent_lsn,
-            metadata_bytes,
-        }
-    }
-
-    pub fn missing_files(&self) -> &HashSet<RelativePath> {
-        &self.missing_layers
-    }
-
-    pub fn from_remote_timeline(
-        timeline_path: &Path,
-        remote_timeline: RemoteTimeline,
-    ) -> anyhow::Result<Self> {
-        let metadata_bytes = remote_timeline.metadata.to_bytes()?;
-
-        let mut layer_metadata = HashMap::new();
-
-        let mut missing_layers = HashSet::new();
-
-        separate_paths_and_metadata(
-            timeline_path,
-            &remote_timeline.missing_layers,
-            &mut missing_layers,
-            &mut layer_metadata,
-        )
-        .context("Failed to convert missing layers' paths to relative ones")?;
-
-        let mut timeline_layers = HashSet::new();
-
-        separate_paths_and_metadata(
-            timeline_path,
-            &remote_timeline.timeline_layers,
-            &mut timeline_layers,
-            &mut layer_metadata,
-        )
-        .context("Failed to convert timeline layers' paths to relative ones")?;
-
-        Ok(Self {
-            version: Self::LATEST_VERSION,
-            timeline_layers,
-            missing_layers,
-            layer_metadata,
-            disk_consistent_lsn: remote_timeline.metadata.disk_consistent_lsn(),
-            metadata_bytes,
-        })
-    }
-}
-
-/// Serialized form of [`LayerFileMetadata`].
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
-pub struct IndexLayerMetadata {
-    file_size: Option<u64>,
-}
-
-impl From<&'_ LayerFileMetadata> for IndexLayerMetadata {
-    fn from(other: &'_ LayerFileMetadata) -> Self {
-        IndexLayerMetadata {
-            file_size: other.file_size,
-        }
-    }
-}
-
-fn separate_paths_and_metadata(
-    timeline_path: &Path,
-    input: &HashMap<PathBuf, LayerFileMetadata>,
-    output: &mut HashSet<RelativePath>,
-    layer_metadata: &mut HashMap<RelativePath, IndexLayerMetadata>,
-) -> anyhow::Result<()> {
-    for (path, metadata) in input {
-        let rel_path = RelativePath::new(timeline_path, path)?;
-        let metadata = IndexLayerMetadata::from(metadata);
-
-        layer_metadata.insert(rel_path.clone(), metadata);
-        output.insert(rel_path);
-    }
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::BTreeSet;
-
-    use super::*;
-    use crate::tenant::harness::{TenantHarness, TIMELINE_ID};
-    use crate::DEFAULT_PG_VERSION;
-
-    #[test]
-    fn index_part_conversion() {
-        let harness = TenantHarness::create("index_part_conversion").unwrap();
-        let timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let metadata = TimelineMetadata::new(
-            Lsn(5).align(),
-            Some(Lsn(4)),
-            None,
-            Lsn(3),
-            Lsn(2),
-            Lsn(1),
-            DEFAULT_PG_VERSION,
-        );
-        let remote_timeline = RemoteTimeline {
-            timeline_layers: HashMap::from([
-                (timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
-                (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
-            ]),
-            missing_layers: HashMap::from([
-                (timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
-                (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
-            ]),
-            metadata: metadata.clone(),
-            awaits_download: false,
-        };
-
-        let index_part = IndexPart::from_remote_timeline(&timeline_path, remote_timeline.clone())
-            .expect("Correct remote timeline should be convertible to index part");
-
-        assert_eq!(
-            index_part.timeline_layers.iter().collect::<BTreeSet<_>>(),
-            BTreeSet::from([
-                &RelativePath("layer_1".to_string()),
-                &RelativePath("layer_2".to_string())
-            ]),
-            "Index part should have all remote timeline layers after the conversion"
-        );
-        assert_eq!(
-            index_part.missing_layers.iter().collect::<BTreeSet<_>>(),
-            BTreeSet::from([
-                &RelativePath("missing_1".to_string()),
-                &RelativePath("missing_2".to_string())
-            ]),
-            "Index part should have all missing remote timeline layers after the conversion"
-        );
-        assert_eq!(
-            index_part.disk_consistent_lsn,
-            metadata.disk_consistent_lsn(),
-            "Index part should have disk consistent lsn from the timeline"
-        );
-        assert_eq!(
-            index_part.metadata_bytes,
-            metadata
-                .to_bytes()
-                .expect("Failed to serialize correct metadata into bytes"),
-            "Index part should have all missing remote timeline layers after the conversion"
-        );
-
-        let restored_timeline = RemoteTimeline::from_index_part(&timeline_path, index_part)
-            .expect("Correct index part should be convertible to remote timeline");
-
-        let original_metadata = &remote_timeline.metadata;
-        let restored_metadata = &restored_timeline.metadata;
-        // we have to compare the metadata this way, since its header is different after creation and restoration,
-        // but that is now consireded ok.
-        assert_eq!(
-            original_metadata.disk_consistent_lsn(),
-            restored_metadata.disk_consistent_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.prev_record_lsn(),
-            restored_metadata.prev_record_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.ancestor_timeline(),
-            restored_metadata.ancestor_timeline(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.ancestor_lsn(),
-            restored_metadata.ancestor_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.latest_gc_cutoff_lsn(),
-            restored_metadata.latest_gc_cutoff_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-        assert_eq!(
-            original_metadata.initdb_lsn(),
-            restored_metadata.initdb_lsn(),
-            "remote timeline -> index part -> remote timeline conversion should not alter metadata"
-        );
-
-        assert_eq!(
-            remote_timeline.awaits_download, restored_timeline.awaits_download,
-            "remote timeline -> index part -> remote timeline conversion should not loose download flag"
-        );
-
-        assert_eq!(
-            remote_timeline
-                .timeline_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            restored_timeline
-                .timeline_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            "remote timeline -> index part -> remote timeline conversion should not loose layer data"
-        );
-        assert_eq!(
-            remote_timeline
-                .missing_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            restored_timeline
-                .missing_layers
-                .into_iter()
-                .collect::<BTreeSet<_>>(),
-            "remote timeline -> index part -> remote timeline conversion should not loose missing file data"
-        );
-    }
-
-    #[test]
-    fn index_part_conversion_negatives() {
-        let harness = TenantHarness::create("index_part_conversion_negatives").unwrap();
-        let timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let metadata = TimelineMetadata::new(
-            Lsn(5).align(),
-            Some(Lsn(4)),
-            None,
-            Lsn(3),
-            Lsn(2),
-            Lsn(1),
-            DEFAULT_PG_VERSION,
-        );
-
-        let conversion_result = IndexPart::from_remote_timeline(
-            &timeline_path,
-            RemoteTimeline {
-                timeline_layers: HashMap::from([
-                    (PathBuf::from("bad_path"), LayerFileMetadata::new(1)),
-                    (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
-                ]),
-                missing_layers: HashMap::from([
-                    (timeline_path.join("missing_1"), LayerFileMetadata::new(3)),
-                    (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
-                ]),
-                metadata: metadata.clone(),
-                awaits_download: false,
-            },
-        );
-        assert!(conversion_result.is_err(), "Should not be able to convert metadata with layer paths that are not in the timeline directory");
-
-        let conversion_result = IndexPart::from_remote_timeline(
-            &timeline_path,
-            RemoteTimeline {
-                timeline_layers: HashMap::from([
-                    (timeline_path.join("layer_1"), LayerFileMetadata::new(1)),
-                    (timeline_path.join("layer_2"), LayerFileMetadata::new(2)),
-                ]),
-                missing_layers: HashMap::from([
-                    (PathBuf::from("bad_path"), LayerFileMetadata::new(3)),
-                    (timeline_path.join("missing_2"), LayerFileMetadata::new(4)),
-                ]),
-                metadata,
-                awaits_download: false,
-            },
-        );
-        assert!(conversion_result.is_err(), "Should not be able to convert metadata with missing layer paths that are not in the timeline directory");
-    }
-
-    #[test]
-    fn v0_indexpart_is_parsed() {
-        let example = r#"{
-            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
-            "missing_layers":["not_a_real_layer_but_adding_coverage"],
-            "disk_consistent_lsn":"0/16960E8",
-            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-        }"#;
-
-        let expected = IndexPart {
-            version: 0,
-            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
-            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
-            layer_metadata: HashMap::default(),
-            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
-            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
-        };
-
-        let part = serde_json::from_str::<IndexPart>(example).unwrap();
-        assert_eq!(part, expected);
-    }
-
-    #[test]
-    fn v1_indexpart_is_parsed() {
-        let example = r#"{
-            "version":1,
-            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
-            "missing_layers":["not_a_real_layer_but_adding_coverage"],
-            "layer_metadata":{
-                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
-                "not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
-            },
-            "disk_consistent_lsn":"0/16960E8",
-            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-        }"#;
-
-        let expected = IndexPart {
-            // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
-            version: 1,
-            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
-            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
-            layer_metadata: HashMap::from([
-                (RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned()), IndexLayerMetadata {
-                    file_size: Some(25600000),
-                }),
-                (RelativePath("not_a_real_layer_but_adding_coverage".to_owned()), IndexLayerMetadata {
-                    // serde_json should always parse this but this might be a double with jq for
-                    // example.
-                    file_size: Some(9007199254741001),
-                })
-            ]),
-            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
-            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
-        };
-
-        let part = serde_json::from_str::<IndexPart>(example).unwrap();
-        assert_eq!(part, expected);
-    }
-}

pageserver/src/storage_sync/upload.rs

@@ -1,479 +0,0 @@
-//! Timeline synchronization logic to compress and upload to the remote storage all new timeline files from the checkpoints.
-
-use std::{fmt::Debug, path::PathBuf};
-
-use anyhow::Context;
-use futures::stream::{FuturesUnordered, StreamExt};
-use remote_storage::GenericRemoteStorage;
-use tokio::fs;
-use tracing::{debug, error, info, warn};
-
-use utils::id::TenantTimelineId;
-
-use super::{
-    index::{IndexPart, RemoteTimeline},
-    LayersUpload, SyncData, SyncQueue,
-};
-use crate::metrics::NO_LAYERS_UPLOAD;
-use crate::{config::PageServerConf, storage_sync::SyncTask};
-
-/// Serializes and uploads the given index part data to the remote storage.
-pub(super) async fn upload_index_part(
-    conf: &'static PageServerConf,
-    storage: &GenericRemoteStorage,
-    sync_id: TenantTimelineId,
-    index_part: IndexPart,
-) -> anyhow::Result<()> {
-    let index_part_bytes = serde_json::to_vec(&index_part)
-        .context("Failed to serialize index part file into bytes")?;
-    let index_part_size = index_part_bytes.len();
-    let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes));
-
-    let index_part_path = conf
-        .metadata_path(sync_id.timeline_id, sync_id.tenant_id)
-        .with_file_name(IndexPart::FILE_NAME);
-    storage
-        .upload_storage_object(
-            Box::new(index_part_bytes),
-            index_part_size,
-            &index_part_path,
-        )
-        .await
-        .with_context(|| format!("Failed to upload index part for '{sync_id}'"))
-}
-
-/// Timeline upload result, with extra data, needed for uploading.
-#[derive(Debug)]
-pub(super) enum UploadedTimeline {
-    /// Upload failed due to some error, the upload task is rescheduled for another retry.
-    FailedAndRescheduled(anyhow::Error),
-    /// No issues happened during the upload, all task files were put into the remote storage.
-    Successful(SyncData<LayersUpload>),
-}
-
-/// Attempts to upload given layer files.
-/// No extra checks for overlapping files is made and any files that are already present remotely will be overwritten, if submitted during the upload.
-///
-/// On an error, bumps the retries count and reschedules the entire task.
-pub(super) async fn upload_timeline_layers<'a>(
-    storage: &'a GenericRemoteStorage,
-    sync_queue: &SyncQueue,
-    remote_timeline: Option<&'a RemoteTimeline>,
-    sync_id: TenantTimelineId,
-    mut upload_data: SyncData<LayersUpload>,
-) -> UploadedTimeline {
-    let upload = &mut upload_data.data;
-    let new_upload_lsn = upload
-        .metadata
-        .as_ref()
-        .map(|meta| meta.disk_consistent_lsn());
-
-    let already_uploaded_layers = remote_timeline
-        .map(|timeline| {
-            timeline
-                .stored_files()
-                .keys()
-                .cloned()
-                .collect::<std::collections::HashSet<_>>()
-        })
-        .unwrap_or_default();
-
-    let layers_to_upload = upload
-        .layers_to_upload
-        .iter()
-        .filter_map(|(k, v)| {
-            if !already_uploaded_layers.contains(k) {
-                Some((k.to_owned(), v.to_owned()))
-            } else {
-                None
-            }
-        })
-        .collect::<Vec<_>>();
-
-    if layers_to_upload.is_empty() {
-        debug!("No layers to upload after filtering, aborting");
-        NO_LAYERS_UPLOAD
-            .with_label_values(&[
-                &sync_id.tenant_id.to_string(),
-                &sync_id.timeline_id.to_string(),
-            ])
-            .inc();
-        return UploadedTimeline::Successful(upload_data);
-    }
-
-    debug!("Layers to upload: {layers_to_upload:?}");
-    info!(
-        "Uploading {} timeline layers, new lsn: {new_upload_lsn:?}",
-        layers_to_upload.len(),
-    );
-
-    let mut upload_tasks = layers_to_upload
-        .into_iter()
-        .map(|(source_path, known_metadata)| async move {
-            let source_file = match fs::File::open(&source_path).await.with_context(|| {
-                format!(
-                    "Failed to upen a source file for layer '{}'",
-                    source_path.display()
-                )
-            }) {
-                Ok(file) => file,
-                Err(e) => return Err(UploadError::MissingLocalFile(source_path, e)),
-            };
-
-            let fs_size = source_file
-                .metadata()
-                .await
-                .with_context(|| {
-                    format!(
-                        "Failed to get the source file metadata for layer '{}'",
-                        source_path.display()
-                    )
-                })
-                .map_err(UploadError::Other)?
-                .len();
-
-            // FIXME: this looks bad
-            if let Some(metadata_size) = known_metadata.file_size() {
-                if metadata_size != fs_size {
-                    return Err(UploadError::Other(anyhow::anyhow!(
-                        "File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}"
-                    )));
-                }
-            } else {
-                // this is a silly state we would like to avoid
-            }
-
-            let fs_size = usize::try_from(fs_size).with_context(|| format!("File {source_path:?} size {fs_size} could not be converted to usize"))
-                .map_err(UploadError::Other)?;
-
-            match storage
-                .upload_storage_object(Box::new(source_file), fs_size, &source_path)
-                .await
-                .with_context(|| format!("Failed to upload layer file for {sync_id}"))
-            {
-                Ok(()) => Ok(source_path),
-                Err(e) => Err(UploadError::MissingLocalFile(source_path, e)),
-            }
-        })
-        .collect::<FuturesUnordered<_>>();
-
-    let mut errors = Vec::new();
-    while let Some(upload_result) = upload_tasks.next().await {
-        match upload_result {
-            Ok(uploaded_path) => {
-                let metadata = upload
-                    .layers_to_upload
-                    .remove(&uploaded_path)
-                    .expect("metadata should always exist, assuming no double uploads");
-                upload.uploaded_layers.insert(uploaded_path, metadata);
-            }
-            Err(e) => match e {
-                UploadError::Other(e) => {
-                    error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
-                    errors.push(format!("{e:#}"));
-                }
-                UploadError::MissingLocalFile(source_path, e) => {
-                    if source_path.exists() {
-                        error!("Failed to upload a layer for timeline {sync_id}: {e:?}");
-                        errors.push(format!("{e:#}"));
-                    } else {
-                        // We have run the upload sync task, but the file we wanted to upload is gone.
-                        // This is "fine" due the asynchronous nature of the sync loop: it only reacts to events and might need to
-                        // retry the upload tasks, if S3 or network is down: but during this time, pageserver might still operate and
-                        // run compaction/gc tasks, removing redundant files from disk.
-                        // It's not good to pause GC/compaction because of those and we would rather skip such uploads.
-                        //
-                        // Yet absence of such files might also mean that the timeline metadata file was updated (GC moves the Lsn forward, for instance).
-                        // We don't try to read a more recent version, since it could contain `disk_consistent_lsn` that does not have its upload finished yet.
-                        // This will create "missing" layers and make data inconsistent.
-                        // Instead, we only update the metadata when it was submitted in an upload task as a checkpoint result.
-                        upload.layers_to_upload.remove(&source_path);
-                        warn!(
-                            "Missing locally a layer file {} scheduled for upload, skipping",
-                            source_path.display()
-                        );
-                    }
-                }
-            },
-        }
-    }
-
-    if errors.is_empty() {
-        info!("Successfully uploaded all layers");
-        UploadedTimeline::Successful(upload_data)
-    } else {
-        debug!("Reenqueuing failed upload task for timeline {sync_id}");
-        upload_data.retries += 1;
-        sync_queue.push(sync_id, SyncTask::Upload(upload_data));
-        UploadedTimeline::FailedAndRescheduled(anyhow::anyhow!(
-            "Errors appeared during layer uploads: {:?}",
-            errors
-        ))
-    }
-}
-
-enum UploadError {
-    MissingLocalFile(PathBuf, anyhow::Error),
-    Other(anyhow::Error),
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{
-        collections::{BTreeSet, HashSet},
-        num::NonZeroUsize,
-    };
-
-    use remote_storage::{LocalFs, RemoteStorage};
-    use tempfile::tempdir;
-    use utils::lsn::Lsn;
-
-    use crate::{
-        storage_sync::{
-            index::RelativePath,
-            test_utils::{create_local_timeline, dummy_metadata},
-        },
-        tenant::harness::{TenantHarness, TIMELINE_ID},
-    };
-
-    use super::{upload_index_part, *};
-
-    #[tokio::test]
-    async fn regular_layer_upload() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("regular_layer_upload")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let layer_files = ["a", "b"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_path_buf(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let current_retries = 3;
-        let metadata = dummy_metadata(Lsn(0x30));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let mut timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layer_files, metadata.clone()).await?;
-        timeline_upload.metadata = None;
-
-        assert!(
-            local_storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-
-        let upload_result = upload_timeline_layers(
-            &storage,
-            &sync_queue,
-            None,
-            sync_id,
-            SyncData::new(current_retries, timeline_upload.clone()),
-        )
-        .await;
-
-        let upload_data = match upload_result {
-            UploadedTimeline::Successful(upload_data) => upload_data,
-            wrong_result => {
-                panic!("Expected a successful upload for timeline, but got: {wrong_result:?}")
-            }
-        };
-
-        assert_eq!(
-            current_retries, upload_data.retries,
-            "On successful upload, retries are not expected to change"
-        );
-        let upload = &upload_data.data;
-        assert!(
-            upload.layers_to_upload.is_empty(),
-            "Successful upload should have no layers left to upload"
-        );
-        assert_eq!(
-            upload
-                .uploaded_layers
-                .keys()
-                .cloned()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_file| local_timeline_path.join(layer_file))
-                .collect(),
-            "Successful upload should have all layers uploaded"
-        );
-        assert_eq!(
-            upload.metadata, None,
-            "Successful upload without metadata should not have it returned either"
-        );
-
-        let storage_files = local_storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            layer_files.len(),
-            "All layers should be uploaded"
-        );
-        assert_eq!(
-            storage_files
-                .into_iter()
-                .map(|storage_path| local_storage.local_path(&storage_path))
-                .collect::<anyhow::Result<BTreeSet<_>>>()?,
-            layer_files
-                .into_iter()
-                .map(|file| local_timeline_path.join(file))
-                .collect(),
-            "Uploaded files should match with the local ones"
-        );
-
-        Ok(())
-    }
-
-    // Currently, GC can run between upload retries, removing local layers scheduled for upload. Test this scenario.
-    #[tokio::test]
-    async fn layer_upload_after_local_fs_update() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("layer_upload_after_local_fs_update")?;
-        let sync_queue = SyncQueue::new(NonZeroUsize::new(100).unwrap());
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let layer_files = ["a1", "b1"];
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let current_retries = 5;
-        let metadata = dummy_metadata(Lsn(0x40));
-
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-        let layers_to_upload = {
-            let mut layers = layer_files.to_vec();
-            layers.push("layer_to_remove");
-            layers
-        };
-        let timeline_upload =
-            create_local_timeline(&harness, TIMELINE_ID, &layers_to_upload, metadata.clone())
-                .await?;
-        assert!(
-            local_storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-
-        fs::remove_file(local_timeline_path.join("layer_to_remove")).await?;
-
-        let upload_result = upload_timeline_layers(
-            &storage,
-            &sync_queue,
-            None,
-            sync_id,
-            SyncData::new(current_retries, timeline_upload.clone()),
-        )
-        .await;
-
-        let upload_data = match upload_result {
-            UploadedTimeline::Successful(upload_data) => upload_data,
-            wrong_result => panic!(
-                "Expected a successful after local fs upload for timeline, but got: {wrong_result:?}"
-            ),
-        };
-
-        assert_eq!(
-            current_retries, upload_data.retries,
-            "On successful upload, retries are not expected to change"
-        );
-        let upload = &upload_data.data;
-        assert!(
-                upload.layers_to_upload.is_empty(),
-                "Successful upload should have no layers left to upload, even those that were removed from the local fs"
-            );
-        assert_eq!(
-            upload
-                .uploaded_layers
-                .keys()
-                .cloned()
-                .collect::<BTreeSet<_>>(),
-            layer_files
-                .iter()
-                .map(|layer_file| local_timeline_path.join(layer_file))
-                .collect(),
-            "Successful upload should have all layers uploaded"
-        );
-        assert_eq!(
-            upload.metadata,
-            Some(metadata),
-            "Successful upload should not change its metadata"
-        );
-
-        let storage_files = local_storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            layer_files.len(),
-            "All layers should be uploaded"
-        );
-        assert_eq!(
-            storage_files
-                .into_iter()
-                .map(|storage_path| local_storage.local_path(&storage_path))
-                .collect::<anyhow::Result<BTreeSet<_>>>()?,
-            layer_files
-                .into_iter()
-                .map(|file| local_timeline_path.join(file))
-                .collect(),
-            "Uploaded files should match with the local ones"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_upload_index_part() -> anyhow::Result<()> {
-        let harness = TenantHarness::create("test_upload_index_part")?;
-        let sync_id = TenantTimelineId::new(harness.tenant_id, TIMELINE_ID);
-
-        let storage = GenericRemoteStorage::new(LocalFs::new(
-            tempdir()?.path().to_owned(),
-            harness.conf.workdir.clone(),
-        )?);
-        let local_storage = storage.as_local().unwrap();
-        let metadata = dummy_metadata(Lsn(0x40));
-        let local_timeline_path = harness.timeline_path(&TIMELINE_ID);
-
-        let index_part = IndexPart::new(
-            HashSet::from([
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("one"))?,
-                RelativePath::new(&local_timeline_path, local_timeline_path.join("two"))?,
-            ]),
-            HashSet::from([RelativePath::new(
-                &local_timeline_path,
-                local_timeline_path.join("three"),
-            )?]),
-            metadata.disk_consistent_lsn(),
-            metadata.to_bytes()?,
-        );
-
-        assert!(
-            local_storage.list().await?.is_empty(),
-            "Storage should be empty before any uploads are made"
-        );
-        upload_index_part(harness.conf, &storage, sync_id, index_part.clone()).await?;
-
-        let storage_files = local_storage.list().await?;
-        assert_eq!(
-            storage_files.len(),
-            1,
-            "Should have only the index part file uploaded"
-        );
-
-        let index_part_path = storage_files.first().unwrap();
-        assert_eq!(
-            index_part_path.object_name(),
-            Some(IndexPart::FILE_NAME),
-            "Remote index part should have the correct name"
-        );
-        let remote_index_part: IndexPart = serde_json::from_slice(
-            &fs::read(local_storage.resolve_in_storage(index_part_path)?).await?,
-        )?;
-        assert_eq!(
-            index_part, remote_index_part,
-            "Remote index part should match the local one"
-        );
-
-        Ok(())
-    }
-}

pageserver/src/storage_sync2/delete.rs

@@ -0,0 +1,38 @@
+//! Helper functions to delete files from remote storage with a RemoteStorage
+use anyhow::Context;
+use std::path::Path;
+use tracing::debug;
+
+use remote_storage::GenericRemoteStorage;
+
+pub(super) async fn delete_layer(
+    storage: &GenericRemoteStorage,
+    local_layer_path: &Path,
+) -> anyhow::Result<()> {
+    fail::fail_point!("before-delete-layer", |_| {
+        anyhow::bail!("failpoint before-delete-layer")
+    });
+    debug!(
+        "Deleting layer from remote storage: {:?}",
+        local_layer_path.display()
+    );
+
+    let storage_path = storage
+        .remote_object_id(local_layer_path)
+        .with_context(|| {
+            format!(
+                "Failed to get the layer storage path for local path '{}'",
+                local_layer_path.display()
+            )
+        })?;
+
+    // XXX: If the deletion fails because the object already didn't exist,
+    // it would be good to just issue a warning but consider it success.
+    // https://github.com/neondatabase/neon/issues/2934
+    storage.delete(&storage_path).await.with_context(|| {
+        format!(
+            "Failed to delete remote layer from storage at '{:?}'",
+            storage_path
+        )
+    })
+}

pageserver/src/storage_sync2/download.rs

@@ -0,0 +1,257 @@
+//! Helper functions to download files from remote storage with a RemoteStorage
+use std::collections::HashSet;
+use std::path::Path;
+
+use anyhow::{bail, Context};
+use futures::stream::{FuturesUnordered, StreamExt};
+use tokio::fs;
+use tokio::io::AsyncWriteExt;
+use tracing::debug;
+
+use crate::config::PageServerConf;
+use crate::storage_sync::index::LayerFileMetadata;
+use remote_storage::{DownloadError, GenericRemoteStorage};
+use utils::crashsafe::path_with_suffix_extension;
+use utils::id::{TenantId, TimelineId};
+
+use super::index::IndexPart;
+use super::RelativePath;
+
+async fn fsync_path(path: impl AsRef<std::path::Path>) -> Result<(), std::io::Error> {
+    fs::File::open(path).await?.sync_all().await
+}
+
+///
+/// If 'metadata' is given, we will validate that the downloaded file's size matches that
+/// in the metadata. (In the future, we might do more cross-checks, like CRC validation)
+///
+/// Returns the size of the downloaded file.
+pub async fn download_layer_file<'a>(
+    conf: &'static PageServerConf,
+    storage: &'a GenericRemoteStorage,
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+    path: &'a RelativePath,
+    layer_metadata: &'a LayerFileMetadata,
+) -> anyhow::Result<u64> {
+    let timeline_path = conf.timeline_path(&timeline_id, &tenant_id);
+
+    let local_path = path.to_local_path(&timeline_path);
+
+    let layer_storage_path = storage.remote_object_id(&local_path).with_context(|| {
+        format!(
+            "Failed to get the layer storage path for local path '{}'",
+            local_path.display()
+        )
+    })?;
+
+    // Perform a rename inspired by durable_rename from file_utils.c.
+    // The sequence:
+    //     write(tmp)
+    //     fsync(tmp)
+    //     rename(tmp, new)
+    //     fsync(new)
+    //     fsync(parent)
+    // For more context about durable_rename check this email from postgres mailing list:
+    // https://www.postgresql.org/message-id/56583BDD.9060302@2ndquadrant.com
+    // If pageserver crashes the temp file will be deleted on startup and re-downloaded.
+    let temp_file_path = path_with_suffix_extension(&local_path, TEMP_DOWNLOAD_EXTENSION);
+
+    // TODO: this doesn't use the cached fd for some reason?
+    let mut destination_file = fs::File::create(&temp_file_path).await.with_context(|| {
+        format!(
+            "Failed to create a destination file for layer '{}'",
+            temp_file_path.display()
+        )
+    })?;
+    let mut download = storage
+        .download(&layer_storage_path)
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to open a download stream for layer with remote storage path '{layer_storage_path:?}'"
+            )
+        })?;
+    let bytes_amount = tokio::io::copy(&mut download.download_stream, &mut destination_file).await.with_context(|| {
+        format!(
+            "Failed to download layer with remote storage path '{layer_storage_path:?}' into file '{}'", temp_file_path.display()
+        )
+    })?;
+
+    // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that:
+    // A file will not be closed immediately when it goes out of scope if there are any IO operations
+    // that have not yet completed. To ensure that a file is closed immediately when it is dropped,
+    // you should call flush before dropping it.
+    //
+    // From the tokio code I see that it waits for pending operations to complete. There shouldt be any because
+    // we assume that `destination_file` file is fully written. I e there is no pending .write(...).await operations.
+    // But for additional safety lets check/wait for any pending operations.
+    destination_file.flush().await.with_context(|| {
+        format!(
+            "failed to flush source file at {}",
+            temp_file_path.display()
+        )
+    })?;
+
+    match layer_metadata.file_size() {
+        Some(expected) if expected != bytes_amount => {
+            anyhow::bail!(
+                "According to layer file metadata should had downloaded {expected} bytes but downloaded {bytes_amount} bytes into file '{}'",
+                temp_file_path.display()
+            );
+        }
+        Some(_) | None => {
+            // matches, or upgrading from an earlier IndexPart version
+        }
+    }
+
+    // not using sync_data because it can lose file size update
+    destination_file.sync_all().await.with_context(|| {
+        format!(
+            "failed to fsync source file at {}",
+            temp_file_path.display()
+        )
+    })?;
+    drop(destination_file);
+
+    fail::fail_point!("remote-storage-download-pre-rename", |_| {
+        bail!("remote-storage-download-pre-rename failpoint triggered")
+    });
+
+    fs::rename(&temp_file_path, &local_path).await?;
+
+    fsync_path(&local_path)
+        .await
+        .with_context(|| format!("Could not fsync layer file {}", local_path.display(),))?;
+
+    tracing::info!("download complete: {}", local_path.display());
+
+    Ok(bytes_amount)
+}
+
+const TEMP_DOWNLOAD_EXTENSION: &str = "temp_download";
+
+pub fn is_temp_download_file(path: &Path) -> bool {
+    let extension = path.extension().map(|pname| {
+        pname
+            .to_str()
+            .expect("paths passed to this function must be valid Rust strings")
+    });
+    match extension {
+        Some(TEMP_DOWNLOAD_EXTENSION) => true,
+        Some(_) => false,
+        None => false,
+    }
+}
+
+/// List timelines of given tenant in remote storage
+pub async fn list_remote_timelines<'a>(
+    storage: &'a GenericRemoteStorage,
+    conf: &'static PageServerConf,
+    tenant_id: TenantId,
+) -> anyhow::Result<Vec<(TimelineId, IndexPart)>> {
+    let tenant_path = conf.timelines_path(&tenant_id);
+    let tenant_storage_path = storage.remote_object_id(&tenant_path).with_context(|| {
+        format!(
+            "Failed to get tenant storage path for local path '{}'",
+            tenant_path.display()
+        )
+    })?;
+
+    let timelines = storage
+        .list_prefixes(Some(&tenant_storage_path))
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to list tenant storage path {tenant_storage_path:?} to get remote timelines to download"
+            )
+        })?;
+
+    if timelines.is_empty() {
+        anyhow::bail!("no timelines found on the remote storage")
+    }
+
+    let mut timeline_ids = HashSet::new();
+    let mut part_downloads = FuturesUnordered::new();
+
+    for timeline_remote_storage_key in timelines {
+        let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| {
+            anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}")
+        })?;
+
+        let timeline_id: TimelineId = object_name.parse().with_context(|| {
+            format!("failed to parse object name into timeline id '{object_name}'")
+        })?;
+
+        // list_prefixes returns all files with the prefix. If we haven't seen this timeline ID
+        // yet, launch a download task for it.
+        if !timeline_ids.contains(&timeline_id) {
+            timeline_ids.insert(timeline_id);
+            let storage_clone = storage.clone();
+            part_downloads.push(async move {
+                (
+                    timeline_id,
+                    download_index_part(conf, &storage_clone, tenant_id, timeline_id).await,
+                )
+            });
+        }
+    }
+
+    // Wait for all the download tasks to complete.
+    let mut timeline_parts = Vec::new();
+    while let Some((timeline_id, part_upload_result)) = part_downloads.next().await {
+        let index_part = part_upload_result
+            .with_context(|| format!("Failed to fetch index part for timeline {timeline_id}"))?;
+
+        debug!("Successfully fetched index part for timeline {timeline_id}");
+        timeline_parts.push((timeline_id, index_part));
+    }
+    Ok(timeline_parts)
+}
+
+pub async fn download_index_part(
+    conf: &'static PageServerConf,
+    storage: &GenericRemoteStorage,
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+) -> Result<IndexPart, DownloadError> {
+    let index_part_path = conf
+        .metadata_path(timeline_id, tenant_id)
+        .with_file_name(IndexPart::FILE_NAME);
+    let part_storage_path = storage
+        .remote_object_id(&index_part_path)
+        .with_context(|| {
+            format!(
+                "Failed to get the index part storage path for local path '{}'",
+                index_part_path.display()
+            )
+        })
+        .map_err(DownloadError::BadInput)?;
+
+    let mut index_part_download = storage.download(&part_storage_path).await?;
+
+    let mut index_part_bytes = Vec::new();
+    tokio::io::copy(
+        &mut index_part_download.download_stream,
+        &mut index_part_bytes,
+    )
+    .await
+    .with_context(|| {
+        format!(
+            "Failed to download an index part into file '{}'",
+            index_part_path.display()
+        )
+    })
+    .map_err(DownloadError::Other)?;
+
+    let index_part: IndexPart = serde_json::from_slice(&index_part_bytes)
+        .with_context(|| {
+            format!(
+                "Failed to deserialize index part file into file '{}'",
+                index_part_path.display()
+            )
+        })
+        .map_err(DownloadError::Other)?;
+
+    Ok(index_part)
+}

pageserver/src/storage_sync2/index.rs

@@ -0,0 +1,248 @@
+//! In-memory index to track the tenant files on the remote storage.
+//! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about
+//! remote timeline layers and its metadata.
+
+use std::{
+    collections::{HashMap, HashSet},
+    path::{Path, PathBuf},
+};
+
+use anyhow::{Context, Ok};
+use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
+
+use crate::tenant::metadata::TimelineMetadata;
+
+use utils::lsn::Lsn;
+
+/// A part of the filesystem path, that needs a root to become a path again.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct RelativePath(String);
+
+impl RelativePath {
+    /// Attempts to strip off the base from path, producing a relative path or an error.
+    pub fn from_local_path(timeline_path: &Path, path: &Path) -> anyhow::Result<RelativePath> {
+        let relative = path.strip_prefix(timeline_path).with_context(|| {
+            format!(
+                "path '{}' is not relative to base '{}'",
+                path.display(),
+                timeline_path.display()
+            )
+        })?;
+        Ok(Self::from_filename(relative))
+    }
+
+    pub fn from_filename(path: &Path) -> RelativePath {
+        RelativePath(path.to_string_lossy().to_string())
+    }
+
+    pub fn to_local_path(&self, timeline_path: &Path) -> PathBuf {
+        timeline_path.join(&self.0)
+    }
+}
+
+/// Metadata gathered for each of the layer files.
+///
+/// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which
+/// might have less or more metadata depending if upgrading or rolling back an upgrade.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[cfg_attr(test, derive(Default))]
+pub struct LayerFileMetadata {
+    file_size: Option<u64>,
+}
+
+impl From<&'_ IndexLayerMetadata> for LayerFileMetadata {
+    fn from(other: &IndexLayerMetadata) -> Self {
+        LayerFileMetadata {
+            file_size: other.file_size,
+        }
+    }
+}
+
+impl LayerFileMetadata {
+    pub fn new(file_size: u64) -> Self {
+        LayerFileMetadata {
+            file_size: Some(file_size),
+        }
+    }
+
+    /// This is used to initialize the metadata for remote layers, for which
+    /// the metadata was missing from the index part file.
+    pub const MISSING: Self = LayerFileMetadata { file_size: None };
+
+    pub fn file_size(&self) -> Option<u64> {
+        self.file_size
+    }
+
+    /// Metadata has holes due to version upgrades. This method is called to upgrade self with the
+    /// other value.
+    ///
+    /// This is called on the possibly outdated version.
+    pub fn merge(&mut self, other: &Self) {
+        self.file_size = other.file_size.or(self.file_size);
+    }
+}
+
+/// In-memory representation of an `index_part.json` file
+///
+/// Contains the data about all files in the timeline, present remotely and its metadata.
+///
+/// This type needs to be backwards and forwards compatible. When changing the fields,
+/// remember to add a test case for the changed version.
+#[serde_as]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+pub struct IndexPart {
+    /// Debugging aid describing the version of this type.
+    #[serde(default)]
+    version: usize,
+
+    /// Each of the layers present on remote storage.
+    ///
+    /// Additional metadata can might exist in `layer_metadata`.
+    pub timeline_layers: HashSet<RelativePath>,
+
+    /// FIXME: unused field. This should be removed, but that changes the on-disk format,
+    /// so we need to make sure we're backwards- (and maybe forwards-) compatible
+    missing_layers: HashSet<RelativePath>,
+
+    /// Per layer file metadata, which can be present for a present or missing layer file.
+    ///
+    /// Older versions of `IndexPart` will not have this property or have only a part of metadata
+    /// that latest version stores.
+    #[serde(default)]
+    pub layer_metadata: HashMap<RelativePath, IndexLayerMetadata>,
+
+    // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata.
+    // It's duplicated here for convenience.
+    #[serde_as(as = "DisplayFromStr")]
+    pub disk_consistent_lsn: Lsn,
+    metadata_bytes: Vec<u8>,
+}
+
+impl IndexPart {
+    /// When adding or modifying any parts of `IndexPart`, increment the version so that it can be
+    /// used to understand later versions.
+    ///
+    /// Version is currently informative only.
+    const LATEST_VERSION: usize = 1;
+    pub const FILE_NAME: &'static str = "index_part.json";
+
+    pub fn new(
+        layers_and_metadata: HashMap<RelativePath, LayerFileMetadata>,
+        disk_consistent_lsn: Lsn,
+        metadata_bytes: Vec<u8>,
+    ) -> Self {
+        let mut timeline_layers = HashSet::new();
+        let mut layer_metadata = HashMap::new();
+
+        separate_paths_and_metadata(
+            &layers_and_metadata,
+            &mut timeline_layers,
+            &mut layer_metadata,
+        );
+
+        Self {
+            version: Self::LATEST_VERSION,
+            timeline_layers,
+            missing_layers: HashSet::new(),
+            layer_metadata,
+            disk_consistent_lsn,
+            metadata_bytes,
+        }
+    }
+
+    pub fn parse_metadata(&self) -> anyhow::Result<TimelineMetadata> {
+        TimelineMetadata::from_bytes(&self.metadata_bytes)
+    }
+}
+
+/// Serialized form of [`LayerFileMetadata`].
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)]
+pub struct IndexLayerMetadata {
+    file_size: Option<u64>,
+}
+
+impl From<&'_ LayerFileMetadata> for IndexLayerMetadata {
+    fn from(other: &'_ LayerFileMetadata) -> Self {
+        IndexLayerMetadata {
+            file_size: other.file_size,
+        }
+    }
+}
+
+fn separate_paths_and_metadata(
+    input: &HashMap<RelativePath, LayerFileMetadata>,
+    output: &mut HashSet<RelativePath>,
+    layer_metadata: &mut HashMap<RelativePath, IndexLayerMetadata>,
+) {
+    for (path, metadata) in input {
+        let metadata = IndexLayerMetadata::from(metadata);
+        layer_metadata.insert(path.clone(), metadata);
+        output.insert(path.clone());
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn v0_indexpart_is_parsed() {
+        let example = r#"{
+            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
+            "missing_layers":["not_a_real_layer_but_adding_coverage"],
+            "disk_consistent_lsn":"0/16960E8",
+            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+        }"#;
+
+        let expected = IndexPart {
+            version: 0,
+            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
+            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
+            layer_metadata: HashMap::default(),
+            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
+            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
+        };
+
+        let part = serde_json::from_str::<IndexPart>(example).unwrap();
+        assert_eq!(part, expected);
+    }
+
+    #[test]
+    fn v1_indexpart_is_parsed() {
+        let example = r#"{
+            "version":1,
+            "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"],
+            "missing_layers":["not_a_real_layer_but_adding_coverage"],
+            "layer_metadata":{
+                "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 },
+                "not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 }
+            },
+            "disk_consistent_lsn":"0/16960E8",
+            "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+        }"#;
+
+        let expected = IndexPart {
+            // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
+            version: 1,
+            timeline_layers: [RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned())].into_iter().collect(),
+            missing_layers: [RelativePath("not_a_real_layer_but_adding_coverage".to_owned())].into_iter().collect(),
+            layer_metadata: HashMap::from([
+                (RelativePath("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".to_owned()), IndexLayerMetadata {
+                    file_size: Some(25600000),
+                }),
+                (RelativePath("not_a_real_layer_but_adding_coverage".to_owned()), IndexLayerMetadata {
+                    // serde_json should always parse this but this might be a double with jq for
+                    // example.
+                    file_size: Some(9007199254741001),
+                })
+            ]),
+            disk_consistent_lsn: "0/16960E8".parse::<Lsn>().unwrap(),
+            metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(),
+        };
+
+        let part = serde_json::from_str::<IndexPart>(example).unwrap();
+        assert_eq!(part, expected);
+    }
+}

pageserver/src/storage_sync2/upload.rs

@@ -0,0 +1,104 @@
+//! Helper functions to upload files to remote storage with a RemoteStorage
+
+use anyhow::{bail, Context};
+use fail::fail_point;
+use std::path::Path;
+use tokio::fs;
+
+use super::index::IndexPart;
+use crate::config::PageServerConf;
+use crate::storage_sync::LayerFileMetadata;
+use remote_storage::GenericRemoteStorage;
+use utils::id::{TenantId, TimelineId};
+
+/// Serializes and uploads the given index part data to the remote storage.
+pub(super) async fn upload_index_part<'a>(
+    conf: &'static PageServerConf,
+    storage: &'a GenericRemoteStorage,
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+    index_part: &'a IndexPart,
+) -> anyhow::Result<()> {
+    fail_point!("before-upload-index", |_| {
+        bail!("failpoint before-upload-index")
+    });
+    let index_part_bytes = serde_json::to_vec(&index_part)
+        .context("Failed to serialize index part file into bytes")?;
+    let index_part_size = index_part_bytes.len();
+    let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes));
+
+    let index_part_path = conf
+        .metadata_path(timeline_id, tenant_id)
+        .with_file_name(IndexPart::FILE_NAME);
+    storage
+        .upload_storage_object(
+            Box::new(index_part_bytes),
+            index_part_size,
+            &index_part_path,
+        )
+        .await
+        .with_context(|| format!("Failed to upload index part for '{tenant_id} / {timeline_id}'"))
+}
+
+/// Attempts to upload given layer files.
+/// No extra checks for overlapping files is made and any files that are already present remotely will be overwritten, if submitted during the upload.
+///
+/// On an error, bumps the retries count and reschedules the entire task.
+pub(super) async fn upload_timeline_layer(
+    storage: &GenericRemoteStorage,
+    source_path: &Path,
+    known_metadata: &LayerFileMetadata,
+) -> anyhow::Result<()> {
+    fail_point!("before-upload-layer", |_| {
+        bail!("failpoint before-upload-layer")
+    });
+    let storage_path = storage.remote_object_id(source_path).with_context(|| {
+        format!(
+            "Failed to get the layer storage path for local path '{}'",
+            source_path.display()
+        )
+    })?;
+
+    let source_file = fs::File::open(&source_path).await.with_context(|| {
+        format!(
+            "Failed to open a source file for layer '{}'",
+            source_path.display()
+        )
+    })?;
+
+    let fs_size = source_file
+        .metadata()
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to get the source file metadata for layer '{}'",
+                source_path.display()
+            )
+        })?
+        .len();
+
+    // FIXME: this looks bad
+    if let Some(metadata_size) = known_metadata.file_size() {
+        if metadata_size != fs_size {
+            bail!("File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}");
+        }
+    } else {
+        // this is a silly state we would like to avoid
+    }
+
+    let fs_size = usize::try_from(fs_size).with_context(|| {
+        format!("File {source_path:?} size {fs_size} could not be converted to usize")
+    })?;
+
+    storage
+        .upload(Box::new(source_file), fs_size, &storage_path, None)
+        .await
+        .with_context(|| {
+            format!(
+                "Failed to upload a layer from local path '{}'",
+                source_path.display()
+            )
+        })?;
+
+    Ok(())
+}

pageserver/src/task_mgr.rs

@@ -197,8 +197,8 @@ pub enum TaskKind {
     // Task that flushes frozen in-memory layers to disk
     LayerFlushTask,
 
-    // Task that manages the remote upload queue
-    StorageSync,
+    // Task that uploads a file to remote storage
+    RemoteUploadTask,
 
     // task that handles the initial downloading of all tenants
     InitialLoad,

pageserver/src/tenant/metadata.rs

@@ -242,7 +242,8 @@ pub fn save_metadata(
     let mut file = VirtualFile::open_with_options(
         &path,
         OpenOptions::new().write(true).create_new(first_save),
-    )?;
+    )
+    .context("open_with_options")?;
 
     let metadata_bytes = data.to_bytes().context("Failed to get metadata bytes")?;
 
@@ -264,6 +265,26 @@ pub fn save_metadata(
     Ok(())
 }
 
+pub fn load_metadata(
+    conf: &'static PageServerConf,
+    timeline_id: TimelineId,
+    tenant_id: TenantId,
+) -> anyhow::Result<TimelineMetadata> {
+    let metadata_path = conf.metadata_path(timeline_id, tenant_id);
+    let metadata_bytes = std::fs::read(&metadata_path).with_context(|| {
+        format!(
+            "Failed to read metadata bytes from path {}",
+            metadata_path.display()
+        )
+    })?;
+    TimelineMetadata::from_bytes(&metadata_bytes).with_context(|| {
+        format!(
+            "Failed to parse metadata bytes from path {}",
+            metadata_path.display()
+        )
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

pageserver/src/tenant/size.rs

@@ -183,6 +183,19 @@ pub(super) async fn gather_inputs(
             }
         }
 
+        // all timelines also have an end point if they have made any progress
+        if last_record_lsn > timeline.get_ancestor_lsn()
+            && !interesting_lsns
+                .iter()
+                .any(|(lsn, _)| lsn == &last_record_lsn)
+        {
+            updates.push(Update {
+                lsn: last_record_lsn,
+                command: Command::EndOfBranch,
+                timeline_id: timeline.timeline_id,
+            });
+        }
+
         timeline_inputs.insert(
             timeline.timeline_id,
             TimelineInputs {
@@ -270,48 +283,22 @@ impl ModelInputs {
         // impossible to always determine the a one main branch.
         let mut storage = tenant_size_model::Storage::<Option<TimelineId>>::new(None);
 
-        // tracking these not to require modifying the current implementation of the size model,
-        // which works in relative LSNs and sizes.
-        let mut last_state: HashMap<TimelineId, (Lsn, u64)> = HashMap::new();
-
         for update in &self.updates {
             let Update {
                 lsn,
                 command: op,
                 timeline_id,
             } = update;
+            let Lsn(now) = *lsn;
             match op {
                 Command::Update(sz) => {
-                    let latest = last_state.get_mut(timeline_id).ok_or_else(|| {
-                        anyhow::anyhow!(
-                        "ordering-mismatch: there must had been a previous state for {timeline_id}"
-                    )
-                    })?;
-
-                    let lsn_bytes = {
-                        let Lsn(now) = lsn;
-                        let Lsn(prev) = latest.0;
-                        debug_assert!(prev <= *now, "self.updates should had been sorted");
-                        now - prev
-                    };
-
-                    let size_diff =
-                        i64::try_from(*sz as i128 - latest.1 as i128).with_context(|| {
-                            format!("size difference i64 overflow for {timeline_id}")
-                        })?;
-
-                    storage.modify_branch(&Some(*timeline_id), "".into(), lsn_bytes, size_diff);
-                    *latest = (*lsn, *sz);
+                    storage.insert_point(&Some(*timeline_id), "".into(), now, Some(*sz));
+                }
+                Command::EndOfBranch => {
+                    storage.insert_point(&Some(*timeline_id), "".into(), now, None);
                 }
                 Command::BranchFrom(parent) => {
                     storage.branch(parent, Some(*timeline_id));
-
-                    let size = parent
-                        .as_ref()
-                        .and_then(|id| last_state.get(id))
-                        .map(|x| x.1)
-                        .unwrap_or(0);
-                    last_state.insert(*timeline_id, (*lsn, size));
                 }
             }
         }
@@ -320,10 +307,7 @@ impl ModelInputs {
     }
 }
 
-/// Single size model update.
-///
-/// Sizing model works with relative increments over latest branch state.
-/// Updates are absolute, so additional state needs to be tracked when applying.
+/// A point of interest in the tree of branches
 #[serde_with::serde_as]
 #[derive(
     Debug, PartialEq, PartialOrd, Eq, Ord, Clone, Copy, serde::Serialize, serde::Deserialize,
@@ -342,6 +326,7 @@ struct Update {
 enum Command {
     Update(u64),
     BranchFrom(#[serde_as(as = "Option<serde_with::DisplayFromStr>")] Option<TimelineId>),
+    EndOfBranch,
 }
 
 impl std::fmt::Debug for Command {
@@ -351,6 +336,7 @@ impl std::fmt::Debug for Command {
         match self {
             Self::Update(arg0) => write!(f, "Update({arg0})"),
             Self::BranchFrom(arg0) => write!(f, "BranchFrom({arg0:?})"),
+            Self::EndOfBranch => write!(f, "EndOfBranch"),
         }
     }
 }

pageserver/src/tenant/timeline.rs

@@ -14,11 +14,13 @@ use std::cmp::{max, min, Ordering};
 use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::ops::{Deref, Range};
-use std::path::PathBuf;
-use std::sync::atomic::{self, AtomicBool, AtomicI64, Ordering as AtomicOrdering};
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering};
 use std::sync::{Arc, Mutex, MutexGuard, RwLock};
 use std::time::{Duration, Instant, SystemTime};
 
+use crate::storage_sync::index::{IndexPart, RelativePath};
+use crate::storage_sync::RemoteTimelineClient;
 use crate::tenant::{
     delta_layer::{DeltaLayer, DeltaLayerWriter},
     ephemeral_file::is_ephemeral_file,
@@ -31,7 +33,7 @@ use crate::tenant::{
     storage_layer::{Layer, ValueReconstructResult, ValueReconstructState},
 };
 
-use crate::config::{PageServerConf, METADATA_FILE_NAME};
+use crate::config::PageServerConf;
 use crate::keyspace::{KeyPartitioning, KeySpace};
 use crate::metrics::TimelineMetrics;
 use crate::pgdatadir_mapping::BlockNumber;
@@ -40,6 +42,7 @@ use crate::pgdatadir_mapping::{is_rel_fsm_block_key, is_rel_vm_block_key};
 use crate::tenant_config::TenantConfOpt;
 use pageserver_api::reltag::RelTag;
 
+use postgres_connection::PgConnectionConfig;
 use postgres_ffi::to_pg_timestamp;
 use utils::{
     id::{TenantId, TimelineId},
@@ -50,16 +53,21 @@ use utils::{
 
 use crate::repository::GcResult;
 use crate::repository::{Key, Value};
-use crate::task_mgr;
 use crate::task_mgr::TaskKind;
 use crate::walreceiver::{is_etcd_client_initialized, spawn_connection_manager_task};
 use crate::walredo::WalRedoManager;
 use crate::CheckpointConfig;
+use crate::METADATA_FILE_NAME;
 use crate::ZERO_PAGE;
-use crate::{
-    page_cache,
-    storage_sync::{self, index::LayerFileMetadata},
-};
+use crate::{is_temporary, task_mgr};
+use crate::{page_cache, storage_sync::index::LayerFileMetadata};
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+enum FlushLoopState {
+    NotStarted,
+    Running,
+    Exited,
+}
 
 pub struct Timeline {
     conf: &'static PageServerConf,
@@ -79,6 +87,13 @@ pub struct Timeline {
     // WAL redo manager
     walredo_mgr: Arc<dyn WalRedoManager + Sync + Send>,
 
+    /// Remote storage client.
+    ///
+    /// If Some, use it to upload all newly created layers to the remote storage,
+    /// and keep remote metadata file in sync. In the future, also use it to download
+    /// layer files on-demand.
+    pub remote_client: Option<Arc<RemoteTimelineClient>>,
+
     // What page versions do we hold in the repository? If we get a
     // request > last_record_lsn, we need to wait until we receive all
     // the WAL up to the request. The SeqWait provides functions for
@@ -111,9 +126,6 @@ pub struct Timeline {
     // Metrics
     metrics: TimelineMetrics,
 
-    /// If `true`, will backup its files that appear after each checkpointing to the remote storage.
-    upload_layers: AtomicBool,
-
     /// Ensures layers aren't frozen by checkpointer between
     /// [`Timeline::get_layer_for_write`] and layer reads.
     /// Locked automatically by [`TimelineWriter`] and checkpointer.
@@ -122,7 +134,7 @@ pub struct Timeline {
     write_lock: Mutex<()>,
 
     /// Used to avoid multiple `flush_loop` tasks running
-    flush_loop_started: Mutex<bool>,
+    flush_loop_state: Mutex<FlushLoopState>,
 
     /// layer_flush_start_tx can be used to wake up the layer-flushing task.
     /// The value is a counter, incremented every time a new flush cycle is requested.
@@ -136,7 +148,7 @@ pub struct Timeline {
     /// A lock to ensure that no layer of the timeline is removed concurrently by other tasks.
     /// This lock is acquired in [`Timeline::gc`], [`Timeline::compact`],
     /// and [`Tenant::delete_timeline`].
-    layer_removal_cs: Mutex<()>,
+    pub(super) layer_removal_cs: tokio::sync::Mutex<()>,
 
     // Needed to ensure that we can't create a branch at a point that was already garbage collected
     pub latest_gc_cutoff_lsn: Rcu<Lsn>,
@@ -288,7 +300,7 @@ impl LogicalSize {
 }
 
 pub struct WalReceiverInfo {
-    pub wal_source_connstr: String,
+    pub wal_source_connconf: PgConnectionConfig,
     pub last_received_msg_lsn: Lsn,
     pub last_received_msg_ts: u128,
 }
@@ -400,6 +412,14 @@ impl Timeline {
         self.disk_consistent_lsn.load()
     }
 
+    pub fn get_remote_consistent_lsn(&self) -> Option<Lsn> {
+        if let Some(remote_client) = &self.remote_client {
+            remote_client.last_uploaded_consistent_lsn()
+        } else {
+            None
+        }
+    }
+
     /// Get the physical size of the timeline at the latest LSN
     pub fn get_physical_size(&self) -> u64 {
         self.metrics.current_physical_size_gauge.get()
@@ -484,15 +504,15 @@ impl Timeline {
             CheckpointConfig::Forced => {
                 self.freeze_inmem_layer(false);
                 self.flush_frozen_layers_and_wait().await?;
-                self.compact()
+                self.compact().await
             }
         }
     }
 
-    pub fn compact(&self) -> anyhow::Result<()> {
+    pub async fn compact(&self) -> anyhow::Result<()> {
         let last_record_lsn = self.get_last_record_lsn();
 
-        // Last record Lsn could be zero in case the timelie was just created
+        // Last record Lsn could be zero in case the timeline was just created
         if !last_record_lsn.is_valid() {
             warn!("Skipping compaction for potentially just initialized timeline, it has invalid last record lsn: {last_record_lsn}");
             return Ok(());
@@ -532,7 +552,12 @@ impl Timeline {
         // Below are functions compact_level0() and create_image_layers()
         // but they are a bit ad hoc and don't quite work like it's explained
         // above. Rewrite it.
-        let _layer_removal_cs = self.layer_removal_cs.lock().unwrap();
+        let _layer_removal_cs = self.layer_removal_cs.lock().await;
+        // Is the timeline being deleted?
+        let state = *self.state.borrow();
+        if state == TimelineState::Stopping {
+            anyhow::bail!("timeline is Stopping");
+        }
 
         let target_file_size = self.get_checkpoint_distance();
 
@@ -546,20 +571,15 @@ impl Timeline {
                 // 2. Create new image layers for partitions that have been modified
                 // "enough".
                 let layer_paths_to_upload = self.create_image_layers(&partitioning, lsn, false)?;
-                if !layer_paths_to_upload.is_empty()
-                    && self.upload_layers.load(atomic::Ordering::Relaxed)
-                {
-                    storage_sync::schedule_layer_upload(
-                        self.tenant_id,
-                        self.timeline_id,
-                        layer_paths_to_upload,
-                        None,
-                    );
+                if let Some(remote_client) = &self.remote_client {
+                    for (path, layer_metadata) in layer_paths_to_upload {
+                        remote_client.schedule_layer_file_upload(&path, &layer_metadata)?;
+                    }
                 }
 
                 // 3. Compact
                 let timer = self.metrics.compact_time_histo.start_timer();
-                self.compact_level0(target_file_size)?;
+                self.compact_level0(target_file_size).await?;
                 timer.stop_and_record();
             }
             Err(err) => {
@@ -648,8 +668,8 @@ impl Timeline {
             (TimelineState::Broken, _) => {
                 error!("Ignoring state update {new_state:?} for broken tenant");
             }
-            (TimelineState::Paused, TimelineState::Active) => {
-                debug!("Not activating a paused timeline");
+            (TimelineState::Stopping, TimelineState::Active) => {
+                debug!("Not activating a Stopping timeline");
             }
             (_, new_state) => {
                 self.state.send_replace(new_state);
@@ -719,7 +739,7 @@ impl Timeline {
         timeline_id: TimelineId,
         tenant_id: TenantId,
         walredo_mgr: Arc<dyn WalRedoManager + Send + Sync>,
-        upload_layers: bool,
+        remote_client: Option<RemoteTimelineClient>,
         pg_version: u32,
     ) -> Self {
         let disk_consistent_lsn = metadata.disk_consistent_lsn();
@@ -738,6 +758,8 @@ impl Timeline {
 
             walredo_mgr,
 
+            remote_client: remote_client.map(Arc::new),
+
             // initialize in-memory 'last_record_lsn' from 'disk_consistent_lsn'.
             last_record_lsn: SeqWait::new(RecordLsn {
                 last: disk_consistent_lsn,
@@ -753,15 +775,13 @@ impl Timeline {
 
             metrics: TimelineMetrics::new(&tenant_id, &timeline_id),
 
-            upload_layers: AtomicBool::new(upload_layers),
-
-            flush_loop_started: Mutex::new(false),
+            flush_loop_state: Mutex::new(FlushLoopState::NotStarted),
 
             layer_flush_start_tx,
             layer_flush_done_tx,
 
             write_lock: Mutex::new(()),
-            layer_removal_cs: Mutex::new(()),
+            layer_removal_cs: Default::default(),
 
             gc_info: RwLock::new(GcInfo {
                 retain_lsns: Vec::new(),
@@ -794,13 +814,23 @@ impl Timeline {
     }
 
     pub(super) fn maybe_spawn_flush_loop(self: &Arc<Self>) {
-        let mut flush_loop_started = self.flush_loop_started.lock().unwrap();
-        if *flush_loop_started {
-            info!(
-                "skipping attempt to start flush_loop twice {}/{}",
-                self.tenant_id, self.timeline_id
-            );
-            return;
+        let mut flush_loop_state = self.flush_loop_state.lock().unwrap();
+        match *flush_loop_state {
+            FlushLoopState::NotStarted => (),
+            FlushLoopState::Running => {
+                info!(
+                    "skipping attempt to start flush_loop twice {}/{}",
+                    self.tenant_id, self.timeline_id
+                );
+                return;
+            }
+            FlushLoopState::Exited => {
+                warn!(
+                    "ignoring attempt to restart exited flush_loop {}/{}",
+                    self.tenant_id, self.timeline_id
+                );
+                return;
+            }
         }
 
         let layer_flush_start_rx = self.layer_flush_start_tx.subscribe();
@@ -813,11 +843,16 @@ impl Timeline {
                     Some(self.timeline_id),
                     "layer flush task",
                     false,
-                    async move { self_clone.flush_loop(layer_flush_start_rx).await; Ok(()) }
+                    async move {
+                         self_clone.flush_loop(layer_flush_start_rx).await;
+                         let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap();
+                         assert_eq!(*flush_loop_state, FlushLoopState::Running);
+                         *flush_loop_state  = FlushLoopState::Exited;
+                         Ok(()) }
                     .instrument(info_span!(parent: None, "layer flush task", tenant = %self.tenant_id, timeline = %self.timeline_id))
                 );
 
-        *flush_loop_started = true;
+        *flush_loop_state = FlushLoopState::Running;
     }
 
     pub(super) fn launch_wal_receiver(self: &Arc<Self>) {
@@ -852,6 +887,7 @@ impl Timeline {
             walreceiver_connect_timeout,
             lagging_wal_timeout,
             max_lsn_wal_lag,
+            crate::config::SAFEKEEPER_AUTH_TOKEN.get().cloned(),
         );
     }
 
@@ -873,6 +909,7 @@ impl Timeline {
 
         for direntry in fs::read_dir(timeline_path)? {
             let direntry = direntry?;
+            let direntry_path = direntry.path();
             let fname = direntry.file_name();
             let fname = fname.to_string_lossy();
 
@@ -884,7 +921,7 @@ impl Timeline {
                         imgfilename, self.timeline_id, disk_consistent_lsn
                     );
 
-                    rename_to_backup(direntry.path())?;
+                    rename_to_backup(&direntry_path)?;
                     continue;
                 }
 
@@ -908,7 +945,7 @@ impl Timeline {
                         deltafilename, self.timeline_id, disk_consistent_lsn
                     );
 
-                    rename_to_backup(direntry.path())?;
+                    rename_to_backup(&direntry_path)?;
                     continue;
                 }
 
@@ -921,10 +958,23 @@ impl Timeline {
                 num_layers += 1;
             } else if fname == METADATA_FILE_NAME || fname.ends_with(".old") {
                 // ignore these
+            } else if crate::storage_sync::is_temp_download_file(&direntry_path) {
+                info!(
+                    "skipping temp download file, reconcile_with_remote will resume / clean up: {}",
+                    fname
+                );
             } else if is_ephemeral_file(&fname) {
                 // Delete any old ephemeral files
                 trace!("deleting old ephemeral file in timeline dir: {}", fname);
-                fs::remove_file(direntry.path())?;
+                fs::remove_file(&direntry_path)?;
+            } else if is_temporary(&direntry_path) {
+                info!("removing temp timeline file at {}", direntry_path.display());
+                fs::remove_file(&direntry_path).with_context(|| {
+                    format!(
+                        "failed to remove temp download file at {}",
+                        direntry_path.display()
+                    )
+                })?;
             } else {
                 warn!("unrecognized filename in timeline dir: {}", fname);
             }
@@ -945,10 +995,222 @@ impl Timeline {
         Ok(())
     }
 
-    pub(super) fn layer_removal_guard(&self) -> anyhow::Result<MutexGuard<()>> {
-        self.layer_removal_cs
-            .try_lock()
-            .map_err(|e| anyhow!("cannot lock compaction critical section {e}"))
+    async fn download_missing(
+        &self,
+        index_part: &IndexPart,
+        remote_client: &RemoteTimelineClient,
+        mut local_filenames: HashSet<PathBuf>,
+        up_to_date_disk_consistent_lsn: Lsn,
+    ) -> anyhow::Result<HashSet<PathBuf>> {
+        let mut remote_filenames: HashSet<PathBuf> = HashSet::new();
+        for fname in index_part.timeline_layers.iter() {
+            remote_filenames.insert(fname.to_local_path(&PathBuf::from("")));
+        }
+
+        // Are there any local files that exist, with a size that doesn't match
+        // with the size stored in the remote index file?
+        // If so, rename_to_backup those files so that we re-download them later.
+        local_filenames.retain(|path| {
+            let layer_metadata = index_part
+                .layer_metadata
+                .get(&RelativePath::from_filename(path))
+                .map(LayerFileMetadata::from)
+                .unwrap_or(LayerFileMetadata::MISSING);
+
+            if let Some(remote_size) = layer_metadata.file_size() {
+                let local_path = self.conf.timeline_path(&self.timeline_id, &self.tenant_id).join(&path);
+                match local_path.metadata() {
+                    Ok(metadata) => {
+                        let local_size = metadata.len();
+
+                        if local_size != remote_size {
+                            warn!("removing local file \"{}\" because it has unexpected length {}; length in remote index is {}",
+                                  path.display(),
+                                  local_size,
+                                  remote_size);
+                            if let Err(err) = rename_to_backup(&local_path) {
+                                error!("could not rename file \"{}\": {:?}",
+                                       local_path.display(), err);
+                            }
+                            self.metrics.current_physical_size_gauge.sub(local_size);
+                            false
+                        } else {
+                            true
+                        }
+                    }
+                    Err(err) => {
+                        error!("could not get size of local file \"{}\": {:?}", path.display(), err);
+                        true
+                    }
+                }
+            } else {
+                true
+            }
+        });
+
+        // Are we missing some files that are present in remote storage?
+        // Download them now.
+        // TODO Downloading many files this way is not efficient.
+        //     Better to use FuturesUnordered. Maybe keep as is because:
+        //    a) inplace download is a throw-away code, on-demand patch doesnt need that
+        //    b) typical case now is that there is nothing to sync, this downloads a lot
+        //       1) if there was another pageserver that came and generated new files
+        //       2) during attach of a timeline with big history which we currently do not do
+        for path in remote_filenames.difference(&local_filenames) {
+            let fname = path.to_str().unwrap();
+            info!("remote layer file {fname} does not exist locally");
+
+            let layer_metadata = index_part
+                .layer_metadata
+                .get(&RelativePath::from_filename(path))
+                .map(LayerFileMetadata::from)
+                .unwrap_or(LayerFileMetadata::MISSING);
+
+            if let Some(imgfilename) = ImageFileName::parse_str(fname) {
+                if imgfilename.lsn > up_to_date_disk_consistent_lsn {
+                    warn!(
+                        "found future image layer {} on timeline {} remote_consistent_lsn is {}",
+                        imgfilename, self.timeline_id, up_to_date_disk_consistent_lsn
+                    );
+                    continue;
+                }
+
+                trace!("downloading image file: {}", file = path.display());
+                let sz = remote_client
+                    .download_layer_file(&RelativePath::from_filename(path), &layer_metadata)
+                    .await
+                    .context("download image layer")?;
+                trace!("done");
+
+                let image_layer =
+                    ImageLayer::new(self.conf, self.timeline_id, self.tenant_id, &imgfilename);
+
+                self.layers
+                    .write()
+                    .unwrap()
+                    .insert_historic(Arc::new(image_layer));
+                self.metrics.current_physical_size_gauge.add(sz);
+            } else if let Some(deltafilename) = DeltaFileName::parse_str(fname) {
+                // Create a DeltaLayer struct for each delta file.
+                // The end-LSN is exclusive, while disk_consistent_lsn is
+                // inclusive. For example, if disk_consistent_lsn is 100, it is
+                // OK for a delta layer to have end LSN 101, but if the end LSN
+                // is 102, then it might not have been fully flushed to disk
+                // before crash.
+                if deltafilename.lsn_range.end > up_to_date_disk_consistent_lsn + 1 {
+                    warn!(
+                        "found future delta layer {} on timeline {} remote_consistent_lsn is {}",
+                        deltafilename, self.timeline_id, up_to_date_disk_consistent_lsn
+                    );
+                    continue;
+                }
+
+                trace!("downloading image file: {}", file = path.display());
+                let sz = remote_client
+                    .download_layer_file(&RelativePath::from_filename(path), &layer_metadata)
+                    .await
+                    .context("download delta layer")?;
+                trace!("done");
+
+                let delta_layer =
+                    DeltaLayer::new(self.conf, self.timeline_id, self.tenant_id, &deltafilename);
+
+                self.layers
+                    .write()
+                    .unwrap()
+                    .insert_historic(Arc::new(delta_layer));
+                self.metrics.current_physical_size_gauge.add(sz);
+            } else {
+                bail!("unexpected layer filename in remote storage: {}", fname);
+            }
+        }
+
+        // now these are local only filenames
+        let local_only_filenames = local_filenames
+            .difference(&remote_filenames)
+            .cloned()
+            .collect();
+        Ok(local_only_filenames)
+    }
+
+    ///
+    /// This function will synchronize local data with what we have in remote storage.
+    /// 1. It will download missing layer files.
+    /// 2. It will update local metadata if remote one has greater `disk_consistent_lsn`.
+    /// 3. It will upload files that are missing on the remote
+    /// 4. It will update index file on the remote accordingly
+    /// TODO may be a bit cleaner to do things based on populated remote client,
+    ///     and then do things based on its upload_queue.latest_files
+    ///
+    /// This is used during tenant attach. The layer map must have been loaded
+    /// with local filesystem contents already.
+    ///
+    /// The caller should provide IndexPart if it exists on the remote storage. If it's None,
+    /// we assume that it is missing on the remote storage, which means that we initialized
+    /// a timeline and then restarted before successful upload was performed
+    ///
+    #[instrument(skip(self, index_part, up_to_date_metadata))]
+    pub async fn reconcile_with_remote(
+        &self,
+        up_to_date_metadata: &TimelineMetadata,
+        index_part: Option<&IndexPart>,
+    ) -> anyhow::Result<()> {
+        info!("starting");
+        let remote_client = self
+            .remote_client
+            .as_ref()
+            .ok_or_else(|| anyhow!("cannot download without remote storage"))?;
+
+        let disk_consistent_lsn = up_to_date_metadata.disk_consistent_lsn();
+
+        // Build a map of local layers for quick lookups
+        let mut local_filenames: HashSet<PathBuf> = HashSet::new();
+        for layer in self.layers.read().unwrap().iter_historic_layers() {
+            local_filenames.insert(layer.filename());
+        }
+
+        let local_only_filenames = match index_part {
+            Some(index_part) => {
+                info!(
+                    "initializing upload queue from remote index with {} layer files",
+                    index_part.timeline_layers.len()
+                );
+                remote_client.init_upload_queue(index_part)?;
+                let local_only_filenames = self
+                    .download_missing(
+                        index_part,
+                        remote_client,
+                        local_filenames,
+                        disk_consistent_lsn,
+                    )
+                    .await?;
+                local_only_filenames
+            }
+            None => {
+                info!("initializing upload queue as empty");
+                remote_client.init_upload_queue_for_empty_remote(up_to_date_metadata)?;
+                local_filenames
+            }
+        };
+
+        // Are there local files that don't exist remotely? Schedule uploads for them
+        let timeline_path = self.conf.timeline_path(&self.timeline_id, &self.tenant_id);
+        for fname in &local_only_filenames {
+            let absolute = timeline_path.join(fname);
+            let sz = absolute
+                .metadata()
+                .with_context(|| format!("failed to get file {} metadata", fname.display()))?
+                .len();
+            info!("scheduling {} for upload", fname.display());
+            remote_client.schedule_layer_file_upload(&absolute, &LayerFileMetadata::new(sz))?;
+        }
+        if !local_only_filenames.is_empty() {
+            remote_client.schedule_index_upload(up_to_date_metadata)?;
+        }
+
+        info!("Done");
+
+        Ok(())
     }
 
     fn try_spawn_size_init_task(self: &Arc<Self>, init_lsn: Lsn) {
@@ -989,7 +1251,7 @@ impl Timeline {
                                         match new_state {
                                             // we're running this job for active timelines only
                                             TimelineState::Active => continue,
-                                            TimelineState::Broken | TimelineState::Paused | TimelineState::Suspended => return Some(new_state),
+                                            TimelineState::Broken | TimelineState::Stopping | TimelineState::Suspended => return Some(new_state),
                                         }
                                     }
                                     Err(_sender_dropped_error) => return None,
@@ -1365,8 +1627,9 @@ impl Timeline {
         // finished, instead of some other flush that was started earlier.
         let mut my_flush_request = 0;
 
-        if !&*self.flush_loop_started.lock().unwrap() {
-            anyhow::bail!("cannot flush frozen layers when flush_loop is not running")
+        let flush_loop_state = { *self.flush_loop_state.lock().unwrap() };
+        if flush_loop_state != FlushLoopState::Running {
+            anyhow::bail!("cannot flush frozen layers when flush_loop is not running, state is {flush_loop_state:?}")
         }
 
         self.layer_flush_start_tx.send_modify(|counter| {
@@ -1449,7 +1712,8 @@ impl Timeline {
         // After crash, we will restart WAL streaming and processing from that point.
         if disk_consistent_lsn != old_disk_consistent_lsn {
             assert!(disk_consistent_lsn > old_disk_consistent_lsn);
-            self.update_metadata_file(disk_consistent_lsn, layer_paths_to_upload)?;
+            self.update_metadata_file(disk_consistent_lsn, layer_paths_to_upload)
+                .context("update_metadata_file")?;
             // Also update the in-memory copy
             self.disk_consistent_lsn.store(disk_consistent_lsn);
         }
@@ -1504,15 +1768,18 @@ impl Timeline {
             self.tenant_id,
             &metadata,
             false,
-        )?;
+        )
+        .context("save_metadata")?;
 
-        if self.can_upload_layers() {
-            storage_sync::schedule_layer_upload(
-                self.tenant_id,
-                self.timeline_id,
-                layer_paths_to_upload,
-                Some(metadata),
-            );
+        if let Some(remote_client) = &self.remote_client {
+            for (path, layer_metadata) in layer_paths_to_upload {
+                remote_client
+                    .schedule_layer_file_upload(&path, &layer_metadata)
+                    .context("schedule_layer_file_upload")?;
+            }
+            remote_client
+                .schedule_index_upload(&metadata)
+                .context("schedule_layer_file_upload")?;
         }
 
         Ok(())
@@ -1708,19 +1975,25 @@ impl Timeline {
 
         Ok(layer_paths_to_upload)
     }
+}
+#[derive(Default)]
+struct CompactLevel0Phase1Result {
+    new_layers: Vec<DeltaLayer>,
+    deltas_to_compact: Vec<Arc<dyn Layer>>,
+}
 
-    ///
-    /// Collect a bunch of Level 0 layer files, and compact and reshuffle them as
-    /// as Level 1 files.
-    ///
-    fn compact_level0(&self, target_file_size: u64) -> anyhow::Result<()> {
+impl Timeline {
+    async fn compact_level0_phase1(
+        &self,
+        target_file_size: u64,
+    ) -> anyhow::Result<CompactLevel0Phase1Result> {
         let layers = self.layers.read().unwrap();
         let mut level0_deltas = layers.get_level0_deltas()?;
         drop(layers);
 
         // Only compact if enough layers have accumulated.
         if level0_deltas.is_empty() || level0_deltas.len() < self.get_compaction_threshold() {
-            return Ok(());
+            return Ok(Default::default());
         }
 
         // Gather the files to compact in this iteration.
@@ -1955,6 +2228,35 @@ impl Timeline {
             layer_paths.pop().unwrap();
         }
 
+        drop(all_keys_iter); // So that deltas_to_compact is no longer borrowed
+
+        Ok(CompactLevel0Phase1Result {
+            new_layers,
+            deltas_to_compact,
+        })
+    }
+
+    ///
+    /// Collect a bunch of Level 0 layer files, and compact and reshuffle them as
+    /// as Level 1 files.
+    ///
+    async fn compact_level0(&self, target_file_size: u64) -> anyhow::Result<()> {
+        let CompactLevel0Phase1Result {
+            new_layers,
+            deltas_to_compact,
+        } = self.compact_level0_phase1(target_file_size).await?;
+
+        // Before deleting any layers, we need to wait for their upload ops to finish.
+        // See storage_sync module level comment on consistency.
+        // Do it here because we don't want to hold self.layers.write() while waiting.
+        if let Some(remote_client) = &self.remote_client {
+            info!("waiting for upload ops to complete");
+            remote_client
+                .wait_completion()
+                .await
+                .context("wait for layer upload ops to complete")?;
+        }
+
         let mut layers = self.layers.write().unwrap();
         let mut new_layer_paths = HashMap::with_capacity(new_layers.len());
         for l in new_layers {
@@ -1962,6 +2264,13 @@ impl Timeline {
 
             let metadata = new_delta_path.metadata()?;
 
+            if let Some(remote_client) = &self.remote_client {
+                remote_client.schedule_layer_file_upload(
+                    &new_delta_path,
+                    &LayerFileMetadata::new(metadata.len()),
+                )?;
+            }
+
             // update the timeline's physical size
             self.metrics.current_physical_size_gauge.add(metadata.len());
 
@@ -1971,32 +2280,22 @@ impl Timeline {
 
         // Now that we have reshuffled the data to set of new delta layers, we can
         // delete the old ones
-        let mut layer_paths_do_delete = HashSet::with_capacity(deltas_to_compact.len());
-        drop(all_keys_iter);
+        let mut layer_paths_to_delete = Vec::with_capacity(deltas_to_compact.len());
         for l in deltas_to_compact {
             if let Some(path) = l.local_path() {
                 self.metrics
                     .current_physical_size_gauge
                     .sub(path.metadata()?.len());
-                layer_paths_do_delete.insert(path);
+                layer_paths_to_delete.push(path);
             }
             l.delete()?;
             layers.remove_historic(l);
         }
         drop(layers);
 
-        if self.can_upload_layers() {
-            storage_sync::schedule_layer_upload(
-                self.tenant_id,
-                self.timeline_id,
-                new_layer_paths,
-                None,
-            );
-            storage_sync::schedule_layer_delete(
-                self.tenant_id,
-                self.timeline_id,
-                layer_paths_do_delete,
-            );
+        // Also schedule the deletions in remote storage
+        if let Some(remote_client) = &self.remote_client {
+            remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?;
         }
 
         Ok(())
@@ -2088,25 +2387,43 @@ impl Timeline {
     /// within a layer file. We can only remove the whole file if it's fully
     /// obsolete.
     ///
-    pub(super) fn gc(&self) -> anyhow::Result<GcResult> {
-        let mut result: GcResult = GcResult::default();
-        let now = SystemTime::now();
-
+    pub(super) async fn gc(&self) -> anyhow::Result<GcResult> {
         fail_point!("before-timeline-gc");
 
-        let _layer_removal_cs = self.layer_removal_cs.lock().unwrap();
+        let _layer_removal_cs = self.layer_removal_cs.lock().await;
+        // Is the timeline being deleted?
+        let state = *self.state.borrow();
+        if state == TimelineState::Stopping {
+            anyhow::bail!("timeline is Stopping");
+        }
 
-        let gc_info = self.gc_info.read().unwrap();
+        let (horizon_cutoff, pitr_cutoff, retain_lsns) = {
+            let gc_info = self.gc_info.read().unwrap();
 
-        let horizon_cutoff = min(gc_info.horizon_cutoff, self.get_disk_consistent_lsn());
-        let pitr_cutoff = gc_info.pitr_cutoff;
-        let retain_lsns = &gc_info.retain_lsns;
+            let horizon_cutoff = min(gc_info.horizon_cutoff, self.get_disk_consistent_lsn());
+            let pitr_cutoff = gc_info.pitr_cutoff;
+            let retain_lsns = gc_info.retain_lsns.clone();
+            (horizon_cutoff, pitr_cutoff, retain_lsns)
+        };
 
         let new_gc_cutoff = Lsn::min(horizon_cutoff, pitr_cutoff);
 
-        let _enter =
-            info_span!("gc_timeline", timeline = %self.timeline_id, cutoff = %new_gc_cutoff)
-                .entered();
+        self.gc_timeline(horizon_cutoff, pitr_cutoff, retain_lsns, new_gc_cutoff)
+            .instrument(
+                info_span!("gc_timeline", timeline = %self.timeline_id, cutoff = %new_gc_cutoff),
+            )
+            .await
+    }
+
+    async fn gc_timeline(
+        &self,
+        horizon_cutoff: Lsn,
+        pitr_cutoff: Lsn,
+        retain_lsns: Vec<Lsn>,
+        new_gc_cutoff: Lsn,
+    ) -> anyhow::Result<GcResult> {
+        let now = SystemTime::now();
+        let mut result: GcResult = GcResult::default();
 
         // Nothing to GC. Return early.
         let latest_gc_cutoff = *self.get_latest_gc_cutoff_lsn();
@@ -2140,6 +2457,17 @@ impl Timeline {
 
         debug!("retain_lsns: {:?}", retain_lsns);
 
+        // Before deleting any layers, we need to wait for their upload ops to finish.
+        // See storage_sync module level comment on consistency.
+        // Do it here because we don't want to hold self.layers.write() while waiting.
+        if let Some(remote_client) = &self.remote_client {
+            info!("waiting for upload ops to complete");
+            remote_client
+                .wait_completion()
+                .await
+                .context("wait for layer upload ops to complete")?;
+        }
+
         let mut layers_to_remove = Vec::new();
 
         // Scan all on-disk layers in the timeline.
@@ -2191,7 +2519,7 @@ impl Timeline {
             // might be referenced by child branches forever.
             // We can track this in child timeline GC and delete parent layers when
             // they are no longer needed. This might be complicated with long inheritance chains.
-            for retain_lsn in retain_lsns {
+            for retain_lsn in &retain_lsns {
                 // start_lsn is inclusive
                 if &l.get_lsn_range().start <= retain_lsn {
                     debug!(
@@ -2247,13 +2575,13 @@ impl Timeline {
         // Actually delete the layers from disk and remove them from the map.
         // (couldn't do this in the loop above, because you cannot modify a collection
         // while iterating it. BTreeMap::retain() would be another option)
-        let mut layer_paths_to_delete = HashSet::with_capacity(layers_to_remove.len());
+        let mut layer_paths_to_delete = Vec::with_capacity(layers_to_remove.len());
         for doomed_layer in layers_to_remove {
             if let Some(path) = doomed_layer.local_path() {
                 self.metrics
                     .current_physical_size_gauge
                     .sub(path.metadata()?.len());
-                layer_paths_to_delete.insert(path);
+                layer_paths_to_delete.push(path);
             }
             doomed_layer.delete()?;
             layers.remove_historic(doomed_layer);
@@ -2269,12 +2597,8 @@ impl Timeline {
             fail_point!("after-timeline-gc-removed-layers");
         }
 
-        if self.can_upload_layers() {
-            storage_sync::schedule_layer_delete(
-                self.tenant_id,
-                self.timeline_id,
-                layer_paths_to_delete,
-            );
+        if let Some(remote_client) = &self.remote_client {
+            remote_client.schedule_layer_file_deletion(&layer_paths_to_delete)?;
         }
 
         result.elapsed = now.elapsed()?;
@@ -2355,11 +2679,6 @@ impl Timeline {
             }
         }
     }
-
-    fn can_upload_layers(&self) -> bool {
-        self.upload_layers.load(atomic::Ordering::Relaxed)
-            && self.current_state() != TimelineState::Broken
-    }
 }
 
 /// Helper function for get_reconstruct_data() to add the path of layers traversed
@@ -2437,12 +2756,12 @@ impl<'a> TimelineWriter<'a> {
 
 /// Add a suffix to a layer file's name: .{num}.old
 /// Uses the first available num (starts at 0)
-fn rename_to_backup(path: PathBuf) -> anyhow::Result<()> {
+fn rename_to_backup(path: &Path) -> anyhow::Result<()> {
     let filename = path
         .file_name()
         .ok_or_else(|| anyhow!("Path {} don't have a file name", path.display()))?
         .to_string_lossy();
-    let mut new_path = path.clone();
+    let mut new_path = path.to_owned();
 
     for i in 0u32.. {
         new_path.set_file_name(format!("{}.{}.old", filename, i));

pageserver/src/tenant_config.rs

@@ -51,6 +51,7 @@ pub struct TenantConf {
     // This parameter determines L1 layer file size.
     pub compaction_target_size: u64,
     // How often to check if there's compaction work to be done.
+    // Duration::ZERO means automatic compaction is disabled.
     #[serde(with = "humantime_serde")]
     pub compaction_period: Duration,
     // Level0 delta layer threshold for compaction.
@@ -61,6 +62,7 @@ pub struct TenantConf {
     // Page versions older than this are garbage collected away.
     pub gc_horizon: u64,
     // Interval at which garbage collection is triggered.
+    // Duration::ZERO means automatic GC is disabled
     #[serde(with = "humantime_serde")]
     pub gc_period: Duration,
     // Delta layer churn threshold to create L1 image layers.
@@ -82,6 +84,7 @@ pub struct TenantConf {
     /// A lagging safekeeper will be changed after `lagging_wal_timeout` time elapses since the last WAL update,
     /// to avoid eager reconnects.
     pub max_lsn_wal_lag: NonZeroU64,
+    pub trace_read_requests: bool,
 }
 
 /// Same as TenantConf, but this struct preserves the information about
@@ -105,6 +108,7 @@ pub struct TenantConfOpt {
     #[serde(with = "humantime_serde")]
     pub lagging_wal_timeout: Option<Duration>,
     pub max_lsn_wal_lag: Option<NonZeroU64>,
+    pub trace_read_requests: Option<bool>,
 }
 
 impl TenantConfOpt {
@@ -138,6 +142,9 @@ impl TenantConfOpt {
                 .lagging_wal_timeout
                 .unwrap_or(global_conf.lagging_wal_timeout),
             max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag),
+            trace_read_requests: self
+                .trace_read_requests
+                .unwrap_or(global_conf.trace_read_requests),
         }
     }
 
@@ -207,10 +214,10 @@ impl TenantConf {
                 .expect("cannot parse default walreceiver lagging wal timeout"),
             max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG)
                 .expect("cannot parse default max walreceiver Lsn wal lag"),
+            trace_read_requests: false,
         }
     }
 
-    #[cfg(test)]
     pub fn dummy_conf() -> Self {
         TenantConf {
             checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
@@ -232,6 +239,7 @@ impl TenantConf {
             .unwrap(),
             max_lsn_wal_lag: NonZeroU64::new(defaults::DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG)
                 .unwrap(),
+            trace_read_requests: false,
         }
     }
 }

pageserver/src/tenant_mgr.rs

@@ -1,10 +1,10 @@
 //! This module acts as a switchboard to access different repositories managed by this
 //! page server.
 
-use std::collections::{hash_map, HashMap};
+use std::collections::hash_map;
 use std::ffi::OsStr;
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::Arc;
 
 use anyhow::Context;
@@ -12,19 +12,12 @@ use tracing::*;
 
 use remote_storage::GenericRemoteStorage;
 
-use crate::config::{PageServerConf, METADATA_FILE_NAME, TIMELINE_UNINIT_MARK_SUFFIX};
-use crate::http::models::TenantInfo;
-use crate::storage_sync::index::{LayerFileMetadata, RemoteIndex, RemoteTimelineIndex};
-use crate::storage_sync::{self, LocalTimelineInitStatus, SyncStartupData, TimelineLocalFiles};
+use crate::config::PageServerConf;
 use crate::task_mgr::{self, TaskKind};
-use crate::tenant::{
-    ephemeral_file::is_ephemeral_file, metadata::TimelineMetadata, Tenant, TenantState,
-};
+use crate::tenant::{Tenant, TenantState};
 use crate::tenant_config::TenantConfOpt;
-use crate::walredo::PostgresRedoManager;
-use crate::TEMP_FILE_SUFFIX;
 
-use utils::crashsafe::{self, path_with_suffix_extension};
+use utils::fs_ext::PathExt;
 use utils::id::{TenantId, TimelineId};
 
 mod tenants_state {
@@ -59,153 +52,112 @@ mod tenants_state {
 pub fn init_tenant_mgr(
     conf: &'static PageServerConf,
     remote_storage: Option<GenericRemoteStorage>,
-) -> anyhow::Result<RemoteIndex> {
+) -> anyhow::Result<()> {
     let _entered = info_span!("init_tenant_mgr").entered();
 
-    let local_tenant_files = local_tenant_timeline_files(conf)
-        .context("Failed to collect local tenant timeline files")?;
-
-    let (remote_index, tenants_to_attach) = if let Some(storage) = remote_storage {
-        let storage_config = conf
-            .remote_storage_config
-            .as_ref()
-            .expect("remote storage without config");
-        let mut broken_tenants = HashMap::new();
-        let mut ready_tenants = HashMap::new();
-        for (tenant_id, tenant_attach_data) in local_tenant_files.into_iter() {
-            match tenant_attach_data {
-                TenantAttachData::Ready(t) => {
-                    ready_tenants.insert(tenant_id, t);
-                }
-                TenantAttachData::Broken(e) => {
-                    broken_tenants.insert(tenant_id, TenantAttachData::Broken(e));
-                }
-            }
-        }
-        let SyncStartupData {
-            remote_index,
-            local_timeline_init_statuses,
-        } = storage_sync::spawn_storage_sync_task(
-            conf,
-            ready_tenants,
-            storage,
-            storage_config.max_concurrent_syncs,
-            storage_config.max_sync_errors,
-        )
-        .context("Failed to spawn the storage sync thread")?;
-
-        let n = local_timeline_init_statuses.0.len();
-        let mut synced_timelines = local_timeline_init_statuses.0.into_iter().fold(
-            HashMap::<TenantId, TenantAttachData>::with_capacity(n),
-            |mut new_values, (tenant_id, old_values)| {
-                let new_timeline_values = new_values
-                    .entry(tenant_id)
-                    .or_insert_with(|| TenantAttachData::Ready(HashMap::new()));
-                if let TenantAttachData::Ready(t) = new_timeline_values {
-                    for (timeline_id, old_value) in old_values {
-                        if let LocalTimelineInitStatus::LocallyComplete(metadata) = old_value {
-                            t.insert(timeline_id, TimelineLocalFiles::ready(metadata));
+    // Scan local filesystem for attached tenants
+    let mut number_of_tenants = 0;
+    let tenants_dir = conf.tenants_path();
+    for dir_entry in std::fs::read_dir(&tenants_dir)
+        .with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))?
+    {
+        match &dir_entry {
+            Ok(dir_entry) => {
+                let tenant_dir_path = dir_entry.path();
+                if crate::is_temporary(&tenant_dir_path) {
+                    info!(
+                        "Found temporary tenant directory, removing: {}",
+                        tenant_dir_path.display()
+                    );
+                    if let Err(e) = std::fs::remove_dir_all(&tenant_dir_path) {
+                        error!(
+                            "Failed to remove temporary directory '{}': {:?}",
+                            tenant_dir_path.display(),
+                            e
+                        );
+                    }
+                } else {
+                    match load_local_tenant(conf, &tenant_dir_path, remote_storage.clone()) {
+                        Ok(Some(tenant)) => {
+                            tenants_state::write_tenants().insert(tenant.tenant_id(), tenant);
+                            number_of_tenants += 1;
+                        }
+                        Ok(None) => {
+                            // This case happens if we crash during attach before creating the attach marker file
+                            if let Err(e) = std::fs::remove_dir(&tenant_dir_path) {
+                                error!(
+                                    "Failed to remove empty tenant directory '{}': {e:#}",
+                                    tenant_dir_path.display()
+                                )
+                            }
+                        }
+                        Err(e) => {
+                            error!(
+                            "Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}",
+                            tenants_dir.display(),
+                            dir_entry,
+                            e
+                        );
                         }
                     }
                 }
-                new_values
-            },
-        );
-        synced_timelines.extend(broken_tenants);
-
-        (remote_index, synced_timelines)
-    } else {
-        info!("No remote storage configured, skipping storage sync, considering all local timelines with correct metadata files enabled");
-        (RemoteIndex::default(), local_tenant_files)
-    };
-    attach_local_tenants(conf, &remote_index, tenants_to_attach);
-
-    Ok(remote_index)
-}
-
-/// Reads local files to load tenants and their timelines given into pageserver's memory.
-/// Ignores other timelines that might be present for tenant, but were not passed as a parameter.
-/// Attempts to load as many entites as possible: if a certain timeline fails during the load, the tenant is marked as "Broken",
-/// and the load continues.
-///
-/// For successful tenant attach, it first has to have a `timelines/` subdirectory and a tenant config file that's loaded into memory successfully.
-/// If either of the conditions fails, the tenant will be added to memory with [`TenantState::Broken`] state, otherwise we start to load its timelines.
-/// Alternatively, tenant is considered loaded successfully, if it's already in pageserver's memory (i.e. was loaded already before).
-///
-/// Attach happens on startup and sucessful timeline downloads
-/// (some subset of timeline files, always including its metadata, after which the new one needs to be registered).
-pub fn attach_local_tenants(
-    conf: &'static PageServerConf,
-    remote_index: &RemoteIndex,
-    tenants_to_attach: HashMap<TenantId, TenantAttachData>,
-) {
-    let _entered = info_span!("attach_local_tenants").entered();
-    let number_of_tenants = tenants_to_attach.len();
-
-    for (tenant_id, local_timelines) in tenants_to_attach {
-        let mut tenants_accessor = tenants_state::write_tenants();
-        let tenant = match tenants_accessor.entry(tenant_id) {
-            hash_map::Entry::Occupied(o) => {
-                info!("Tenant {tenant_id} was found in pageserver's memory");
-                Arc::clone(o.get())
             }
-            hash_map::Entry::Vacant(v) => {
-                info!("Tenant {tenant_id} was not found in pageserver's memory, loading it");
-                let tenant = Arc::new(Tenant::new(
-                    conf,
-                    TenantConfOpt::default(),
-                    Arc::new(PostgresRedoManager::new(conf, tenant_id)),
-                    tenant_id,
-                    remote_index.clone(),
-                    conf.remote_storage_config.is_some(),
-                ));
-                match local_timelines {
-                    TenantAttachData::Broken(_) => {
-                        tenant.set_state(TenantState::Broken);
-                    }
-                    TenantAttachData::Ready(_) => {
-                        match Tenant::load_tenant_config(conf, tenant_id) {
-                            Ok(tenant_conf) => {
-                                tenant.update_tenant_config(tenant_conf);
-                                tenant.activate(false);
-                            }
-                            Err(e) => {
-                                error!("Failed to read config for tenant {tenant_id}, disabling tenant: {e:?}");
-                                tenant.set_state(TenantState::Broken);
-                            }
-                        };
-                    }
-                }
-                v.insert(Arc::clone(&tenant));
-                tenant
-            }
-        };
-        drop(tenants_accessor);
-        match local_timelines {
-            TenantAttachData::Broken(e) => warn!("{}", e),
-            TenantAttachData::Ready(ref timelines) => {
-                info!("Attaching {} timelines for {tenant_id}", timelines.len());
-                debug!("Timelines to attach: {local_timelines:?}");
-                let has_timelines = !timelines.is_empty();
-                let timelines_to_attach = timelines
-                    .iter()
-                    .map(|(&k, v)| (k, v.metadata().to_owned()))
-                    .collect();
-                match tenant.init_attach_timelines(timelines_to_attach) {
-                    Ok(()) => {
-                        info!("successfully loaded local timelines for tenant {tenant_id}");
-                        tenant.activate(has_timelines);
-                    }
-                    Err(e) => {
-                        error!("Failed to attach tenant timelines: {e:?}");
-                        tenant.set_state(TenantState::Broken);
-                    }
-                }
+            Err(e) => {
+                // On error, print it, but continue with the other tenants. If we error out
+                // here, the pageserver startup fails altogether, causing outage for *all*
+                // tenants. That seems worse.
+                error!(
+                    "Failed to list tenants dir entry {:?} in directory {}, reason: {:?}",
+                    dir_entry,
+                    tenants_dir.display(),
+                    e,
+                );
             }
         }
     }
 
-    info!("Processed {number_of_tenants} local tenants during attach")
+    info!("Processed {number_of_tenants} local tenants at startup");
+    Ok(())
+}
+
+fn load_local_tenant(
+    conf: &'static PageServerConf,
+    tenant_path: &Path,
+    remote_storage: Option<GenericRemoteStorage>,
+) -> anyhow::Result<Option<Arc<Tenant>>> {
+    if !tenant_path.is_dir() {
+        anyhow::bail!("tenant_path is not a directory: {tenant_path:?}")
+    }
+
+    let is_empty = tenant_path
+        .is_empty_dir()
+        .context("check whether tenant_path is an empty dir")?;
+    if is_empty {
+        info!("skipping empty tenant directory {tenant_path:?}");
+        return Ok(None);
+    }
+
+    let tenant_id = tenant_path
+        .file_name()
+        .and_then(OsStr::to_str)
+        .unwrap_or_default()
+        .parse::<TenantId>()
+        .context("Could not parse tenant id out of the tenant dir name")?;
+
+    let tenant = if conf.tenant_attaching_mark_file_path(&tenant_id).exists() {
+        info!("tenant {tenant_id} has attaching mark file, resuming its attach operation");
+        if let Some(remote_storage) = remote_storage {
+            Tenant::spawn_attach(conf, tenant_id, &remote_storage)
+        } else {
+            warn!("tenant {tenant_id} has attaching mark file, but pageserver has no remote storage configured");
+            Tenant::create_broken_tenant(conf, tenant_id)
+        }
+    } else {
+        info!("tenant {tenant_id} is assumed to be loadable, starting load operation");
+        // Start loading the tenant into memory. It will initially be in Loading state.
+        Tenant::spawn_load(conf, tenant_id, remote_storage)
+    };
+    Ok(Some(tenant))
 }
 
 ///
@@ -218,7 +170,7 @@ pub async fn shutdown_all_tenants() {
         for (_, tenant) in m.drain() {
             if tenant.is_active() {
                 // updates tenant state, forbidding new GC and compaction iterations from starting
-                tenant.set_state(TenantState::Paused);
+                tenant.set_stopping();
                 tenants_to_shut_down.push(tenant)
             }
         }
@@ -247,156 +199,41 @@ pub async fn shutdown_all_tenants() {
     }
 }
 
-fn create_tenant_files(
-    conf: &'static PageServerConf,
-    tenant_conf: TenantConfOpt,
-    tenant_id: TenantId,
-) -> anyhow::Result<()> {
-    let target_tenant_directory = conf.tenant_path(&tenant_id);
-    anyhow::ensure!(
-        !target_tenant_directory.exists(),
-        "cannot create new tenant repo: '{tenant_id}' directory already exists",
-    );
-
-    let temporary_tenant_dir =
-        path_with_suffix_extension(&target_tenant_directory, TEMP_FILE_SUFFIX);
-    debug!(
-        "Creating temporary directory structure in {}",
-        temporary_tenant_dir.display()
-    );
-
-    // top-level dir may exist if we are creating it through CLI
-    crashsafe::create_dir_all(&temporary_tenant_dir).with_context(|| {
-        format!(
-            "could not create temporary tenant directory {}",
-            temporary_tenant_dir.display()
-        )
-    })?;
-
-    let creation_result = try_create_target_tenant_dir(
-        conf,
-        tenant_conf,
-        tenant_id,
-        &temporary_tenant_dir,
-        &target_tenant_directory,
-    );
-
-    if creation_result.is_err() {
-        error!("Failed to create directory structure for tenant {tenant_id}, cleaning tmp data");
-        if let Err(e) = fs::remove_dir_all(&temporary_tenant_dir) {
-            error!("Failed to remove temporary tenant directory {temporary_tenant_dir:?}: {e}")
-        } else if let Err(e) = crashsafe::fsync(&temporary_tenant_dir) {
-            error!(
-                "Failed to fsync removed temporary tenant directory {temporary_tenant_dir:?}: {e}"
-            )
-        }
-    }
-
-    creation_result
-}
-
-fn try_create_target_tenant_dir(
-    conf: &'static PageServerConf,
-    tenant_conf: TenantConfOpt,
-    tenant_id: TenantId,
-    temporary_tenant_dir: &Path,
-    target_tenant_directory: &Path,
-) -> Result<(), anyhow::Error> {
-    let temporary_tenant_timelines_dir = rebase_directory(
-        &conf.timelines_path(&tenant_id),
-        target_tenant_directory,
-        temporary_tenant_dir,
-    )
-    .with_context(|| format!("Failed to resolve tenant {tenant_id} temporary timelines dir"))?;
-    let temporary_tenant_config_path = rebase_directory(
-        &conf.tenant_config_path(tenant_id),
-        target_tenant_directory,
-        temporary_tenant_dir,
-    )
-    .with_context(|| format!("Failed to resolve tenant {tenant_id} temporary config path"))?;
-
-    Tenant::persist_tenant_config(&temporary_tenant_config_path, tenant_conf, true).with_context(
-        || {
-            format!(
-                "Failed to write tenant {} config to {}",
-                tenant_id,
-                temporary_tenant_config_path.display()
-            )
-        },
-    )?;
-    crashsafe::create_dir(&temporary_tenant_timelines_dir).with_context(|| {
-        format!(
-            "could not create tenant {} temporary timelines directory {}",
-            tenant_id,
-            temporary_tenant_timelines_dir.display()
-        )
-    })?;
-    fail::fail_point!("tenant-creation-before-tmp-rename", |_| {
-        anyhow::bail!("failpoint tenant-creation-before-tmp-rename");
-    });
-
-    fs::rename(&temporary_tenant_dir, target_tenant_directory).with_context(|| {
-        format!(
-            "failed to move tenant {} temporary directory {} into the permanent one {}",
-            tenant_id,
-            temporary_tenant_dir.display(),
-            target_tenant_directory.display()
-        )
-    })?;
-    let target_dir_parent = target_tenant_directory.parent().with_context(|| {
-        format!(
-            "Failed to get tenant {} dir parent for {}",
-            tenant_id,
-            target_tenant_directory.display()
-        )
-    })?;
-    crashsafe::fsync(target_dir_parent).with_context(|| {
-        format!(
-            "Failed to fsync renamed directory's parent {} for tenant {}",
-            target_dir_parent.display(),
-            tenant_id,
-        )
-    })?;
-
-    Ok(())
-}
-
-fn rebase_directory(original_path: &Path, base: &Path, new_base: &Path) -> anyhow::Result<PathBuf> {
-    let relative_path = original_path.strip_prefix(base).with_context(|| {
-        format!(
-            "Failed to strip base prefix '{}' off path '{}'",
-            base.display(),
-            original_path.display()
-        )
-    })?;
-    Ok(new_base.join(relative_path))
-}
-
 pub fn create_tenant(
     conf: &'static PageServerConf,
     tenant_conf: TenantConfOpt,
     tenant_id: TenantId,
-    remote_index: RemoteIndex,
-) -> anyhow::Result<Option<TenantId>> {
+    remote_storage: Option<GenericRemoteStorage>,
+) -> anyhow::Result<Option<Arc<Tenant>>> {
     match tenants_state::write_tenants().entry(tenant_id) {
         hash_map::Entry::Occupied(_) => {
             debug!("tenant {tenant_id} already exists");
             Ok(None)
         }
         hash_map::Entry::Vacant(v) => {
-            let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id));
-            create_tenant_files(conf, tenant_conf, tenant_id)?;
-            let tenant = Arc::new(Tenant::new(
-                conf,
-                tenant_conf,
-                wal_redo_manager,
-                tenant_id,
-                remote_index,
-                conf.remote_storage_config.is_some(),
-            ));
-            tenant.activate(false);
-            v.insert(tenant);
-            Ok(Some(tenant_id))
+            // Hold the write_tenants() lock, since all of this is local IO.
+            // If this section ever becomes contentious, introduce a new `TenantState::Creating`.
+            let tenant_directory =
+                super::tenant::create_tenant_files(conf, tenant_conf, tenant_id)?;
+            let created_tenant = load_local_tenant(conf, &tenant_directory, remote_storage)?;
+            match created_tenant {
+                None => {
+                    // We get None in case the directory is empty.
+                    // This shouldn't happen here, because we just created the directory.
+                    // So, skip any cleanup work for now, we don't know how we reached this state.
+                    anyhow::bail!("we just created the tenant directory, it can't be empty");
+                }
+                Some(tenant) => {
+                    anyhow::ensure!(
+                        tenant_id == tenant.tenant_id(),
+                        "loaded created tenant has unexpected tenant id (expect {} != actual {})",
+                        tenant_id,
+                        tenant.tenant_id()
+                    );
+                    v.insert(Arc::clone(&tenant));
+                    Ok(Some(tenant))
+                }
+            }
         }
     }
 }
@@ -420,7 +257,10 @@ pub fn get_tenant(tenant_id: TenantId, active_only: bool) -> anyhow::Result<Arc<
         .get(&tenant_id)
         .with_context(|| format!("Tenant {tenant_id} not found in the local state"))?;
     if active_only && !tenant.is_active() {
-        anyhow::bail!("Tenant {tenant_id} is not active")
+        anyhow::bail!(
+            "Tenant {tenant_id} is not active. Current state: {:?}",
+            tenant.current_state()
+        )
     } else {
         Ok(Arc::clone(tenant))
     }
@@ -450,10 +290,7 @@ pub async fn delete_timeline(tenant_id: TenantId, timeline_id: TimelineId) -> an
     info!("timeline task shutdown completed");
     match get_tenant(tenant_id, true) {
         Ok(tenant) => {
-            tenant.delete_timeline(timeline_id)?;
-            if tenant.list_timelines().is_empty() {
-                tenant.activate(false);
-            }
+            tenant.delete_timeline(timeline_id).await?;
         }
         Err(e) => anyhow::bail!("Cannot access tenant {tenant_id} in local tenant state: {e:?}"),
     }
@@ -473,7 +310,7 @@ pub async fn detach_tenant(
         None => anyhow::bail!("Tenant not found for id {tenant_id}"),
     };
 
-    tenant.set_state(TenantState::Paused);
+    tenant.set_stopping();
     // shutdown all tenant and timeline tasks: gc, compaction, page service)
     task_mgr::shutdown_tasks(None, Some(tenant_id), None).await;
 
@@ -497,374 +334,94 @@ pub async fn detach_tenant(
 ///
 /// Get list of tenants, for the mgmt API
 ///
-pub fn list_tenant_info(remote_index: &RemoteTimelineIndex) -> Vec<TenantInfo> {
+pub fn list_tenants() -> Vec<(TenantId, TenantState)> {
     tenants_state::read_tenants()
         .iter()
-        .map(|(id, tenant)| {
-            let has_in_progress_downloads = remote_index
-            .tenant_entry(id)
-            .map(|entry| entry.has_in_progress_downloads());
-
-            // TODO this is not correct when we might have remote storage sync disabled:
-            // we keep `RemoteTimelineIndex` in memory anyway for simplicity and this error message is printed still
-            if has_in_progress_downloads.is_none() {
-                error!("timeline is not found in remote index while it is present in the tenants registry")
-            }
-
-            TenantInfo {
-                id: *id,
-                state: tenant.current_state(),
-                current_physical_size: None,
-                has_in_progress_downloads,
-            }
-        })
+        .map(|(id, tenant)| (*id, tenant.current_state()))
         .collect()
 }
 
-#[derive(Debug)]
-pub enum TenantAttachData {
-    Ready(HashMap<TimelineId, TimelineLocalFiles>),
-    Broken(anyhow::Error),
-}
-/// Attempts to collect information about all tenant and timelines, existing on the local FS.
-/// If finds any, deletes all temporary files and directories, created before. Also removes empty directories,
-/// that may appear due to such removals.
-/// Does not fail on particular timeline or tenant collection errors, rather logging them and ignoring the entities.
-fn local_tenant_timeline_files(
-    config: &'static PageServerConf,
-) -> anyhow::Result<HashMap<TenantId, TenantAttachData>> {
-    let _entered = info_span!("local_tenant_timeline_files").entered();
-
-    let mut local_tenant_timeline_files = HashMap::new();
-    let tenants_dir = config.tenants_path();
-    for tenants_dir_entry in fs::read_dir(&tenants_dir)
-        .with_context(|| format!("Failed to list tenants dir {}", tenants_dir.display()))?
-    {
-        match &tenants_dir_entry {
-            Ok(tenants_dir_entry) => {
-                let tenant_dir_path = tenants_dir_entry.path();
-                if is_temporary(&tenant_dir_path) {
-                    info!(
-                        "Found temporary tenant directory, removing: {}",
-                        tenant_dir_path.display()
-                    );
-                    if let Err(e) = fs::remove_dir_all(&tenant_dir_path) {
-                        error!(
-                            "Failed to remove temporary directory '{}': {:?}",
-                            tenant_dir_path.display(),
-                            e
-                        );
-                    }
-                } else {
-                    match collect_timelines_for_tenant(config, &tenant_dir_path) {
-                        Ok((tenant_id, TenantAttachData::Broken(e))) => {
-                            local_tenant_timeline_files.entry(tenant_id).or_insert(TenantAttachData::Broken(e));
-                        },
-                        Ok((tenant_id, TenantAttachData::Ready(collected_files))) => {
-                            if collected_files.is_empty() {
-                                match remove_if_empty(&tenant_dir_path) {
-                                    Ok(true) => info!("Removed empty tenant directory {}", tenant_dir_path.display()),
-                                    Ok(false) => {
-                                        // insert empty timeline entry: it has some non-temporary files inside that we cannot remove
-                                        // so make obvious for HTTP API callers, that something exists there and try to load the tenant
-                                        let _ = local_tenant_timeline_files.entry(tenant_id).or_insert_with(|| TenantAttachData::Ready(HashMap::new()));
-                                    },
-                                    Err(e) => error!("Failed to remove empty tenant directory: {e:?}"),
-                                }
-                            } else {
-                                match local_tenant_timeline_files.entry(tenant_id) {
-                                    hash_map::Entry::Vacant(entry) => {
-                                        entry.insert(TenantAttachData::Ready(collected_files));
-                                    }
-                                    hash_map::Entry::Occupied(entry) =>{
-                                        if let TenantAttachData::Ready(old_timelines) = entry.into_mut() {
-                                            old_timelines.extend(collected_files);
-                                        }
-                                    },
-                                }
-                            }
-                        },
-                        Err(e) => error!(
-                            "Failed to collect tenant files from dir '{}' for entry {:?}, reason: {:#}",
-                            tenants_dir.display(),
-                            tenants_dir_entry,
-                            e
-                        ),
-                    }
+/// Execute Attach mgmt API command.
+///
+/// Downloading all the tenant data is performed in the background, this merely
+/// spawns the background task and returns quickly.
+pub async fn attach_tenant(
+    conf: &'static PageServerConf,
+    tenant_id: TenantId,
+    remote_storage: &GenericRemoteStorage,
+) -> anyhow::Result<()> {
+    match tenants_state::write_tenants().entry(tenant_id) {
+        hash_map::Entry::Occupied(e) => {
+            // Cannot attach a tenant that already exists. The error message depends on
+            // the state it's in.
+            match e.get().current_state() {
+                TenantState::Attaching => {
+                    anyhow::bail!("tenant {tenant_id} attach is already in progress")
+                }
+                current_state => {
+                    anyhow::bail!("tenant already exists, current state: {current_state:?}")
                 }
             }
-            Err(e) => error!(
-                "Failed to list tenants dir entry {:?} in directory {}, reason: {:?}",
-                tenants_dir_entry,
-                tenants_dir.display(),
-                e
-            ),
+        }
+        hash_map::Entry::Vacant(v) => {
+            let tenant = Tenant::spawn_attach(conf, tenant_id, remote_storage);
+            v.insert(tenant);
+            Ok(())
         }
     }
-
-    info!(
-        "Collected files for {} tenants",
-        local_tenant_timeline_files.len(),
-    );
-    Ok(local_tenant_timeline_files)
 }
 
-fn remove_if_empty(tenant_dir_path: &Path) -> anyhow::Result<bool> {
-    let directory_is_empty = tenant_dir_path
-        .read_dir()
-        .with_context(|| {
-            format!(
-                "Failed to read directory '{}' contents",
-                tenant_dir_path.display()
-            )
-        })?
-        .next()
-        .is_none();
+#[cfg(feature = "testing")]
+use {
+    crate::repository::GcResult, pageserver_api::models::TimelineGcRequest,
+    utils::http::error::ApiError,
+};
 
-    if directory_is_empty {
-        fs::remove_dir_all(&tenant_dir_path).with_context(|| {
-            format!(
-                "Failed to remove empty directory '{}'",
-                tenant_dir_path.display(),
-            )
-        })?;
+#[cfg(feature = "testing")]
+pub fn immediate_gc(
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+    gc_req: TimelineGcRequest,
+) -> Result<tokio::sync::oneshot::Receiver<Result<GcResult, anyhow::Error>>, ApiError> {
+    let guard = tenants_state::read_tenants();
 
-        Ok(true)
-    } else {
-        Ok(false)
-    }
-}
+    let tenant = guard
+        .get(&tenant_id)
+        .map(Arc::clone)
+        .with_context(|| format!("Tenant {tenant_id} not found"))
+        .map_err(ApiError::NotFound)?;
 
-fn is_temporary(path: &Path) -> bool {
-    match path.file_name() {
-        Some(name) => name.to_string_lossy().ends_with(TEMP_FILE_SUFFIX),
-        None => false,
-    }
-}
+    let gc_horizon = gc_req.gc_horizon.unwrap_or_else(|| tenant.get_gc_horizon());
+    // Use tenant's pitr setting
+    let pitr = tenant.get_pitr_interval();
 
-fn is_uninit_mark(path: &Path) -> bool {
-    match path.file_name() {
-        Some(name) => name
-            .to_string_lossy()
-            .ends_with(TIMELINE_UNINIT_MARK_SUFFIX),
-        None => false,
-    }
-}
-
-fn collect_timelines_for_tenant(
-    config: &'static PageServerConf,
-    tenant_path: &Path,
-) -> anyhow::Result<(TenantId, TenantAttachData)> {
-    let tenant_id = tenant_path
-        .file_name()
-        .and_then(OsStr::to_str)
-        .unwrap_or_default()
-        .parse::<TenantId>()
-        .context("Could not parse tenant id out of the tenant dir name")?;
-    let timelines_dir = config.timelines_path(&tenant_id);
-
-    if !timelines_dir.as_path().is_dir() {
-        return Ok((
-            tenant_id,
-            TenantAttachData::Broken(anyhow::anyhow!(
-                "Tenant {} has no timelines directory at {}",
-                tenant_id,
-                timelines_dir.display()
-            )),
-        ));
-    }
-
-    let mut tenant_timelines = HashMap::new();
-    for timelines_dir_entry in fs::read_dir(&timelines_dir)
-        .with_context(|| format!("Failed to list timelines dir entry for tenant {tenant_id}"))?
-    {
-        match timelines_dir_entry {
-            Ok(timelines_dir_entry) => {
-                let timeline_dir = timelines_dir_entry.path();
-                if is_temporary(&timeline_dir) {
-                    info!(
-                        "Found temporary timeline directory, removing: {}",
-                        timeline_dir.display()
-                    );
-                    if let Err(e) = fs::remove_dir_all(&timeline_dir) {
-                        error!(
-                            "Failed to remove temporary directory '{}': {:?}",
-                            timeline_dir.display(),
-                            e
-                        );
-                    }
-                } else if is_uninit_mark(&timeline_dir) {
-                    let timeline_uninit_mark_file = &timeline_dir;
-                    info!(
-                        "Found an uninit mark file {}, removing the timeline and its uninit mark",
-                        timeline_uninit_mark_file.display()
-                    );
-                    let timeline_id = timeline_uninit_mark_file
-                        .file_stem()
-                        .and_then(OsStr::to_str)
-                        .unwrap_or_default()
-                        .parse::<TimelineId>()
-                        .with_context(|| {
-                            format!(
-                                "Could not parse timeline id out of the timeline uninit mark name {}",
-                                timeline_uninit_mark_file.display()
-                            )
-                        })?;
-                    let timeline_dir = config.timeline_path(&timeline_id, &tenant_id);
-                    if let Err(e) =
-                        remove_timeline_and_uninit_mark(&timeline_dir, timeline_uninit_mark_file)
-                    {
-                        error!("Failed to clean up uninit marked timeline: {e:?}");
-                    }
-                } else {
-                    let timeline_id = timeline_dir
-                        .file_name()
-                        .and_then(OsStr::to_str)
-                        .unwrap_or_default()
-                        .parse::<TimelineId>()
-                        .with_context(|| {
-                            format!(
-                                "Could not parse timeline id out of the timeline dir name {}",
-                                timeline_dir.display()
-                            )
-                        })?;
-                    let timeline_uninit_mark_file =
-                        config.timeline_uninit_mark_file_path(tenant_id, timeline_id);
-                    if timeline_uninit_mark_file.exists() {
-                        info!("Found an uninit mark file for timeline {tenant_id}/{timeline_id}, removing the timeline and its uninit mark");
-                        if let Err(e) = remove_timeline_and_uninit_mark(
-                            &timeline_dir,
-                            &timeline_uninit_mark_file,
-                        ) {
-                            error!("Failed to clean up uninit marked timeline: {e:?}");
-                        }
-                    } else {
-                        match collect_timeline_files(&timeline_dir) {
-                            Ok((metadata, timeline_files)) => {
-                                tenant_timelines.insert(
-                                    timeline_id,
-                                    TimelineLocalFiles::collected(metadata, timeline_files),
-                                );
-                            }
-                            Err(e) => {
-                                error!(
-                                    "Failed to process timeline dir contents at '{}', reason: {:?}",
-                                    timeline_dir.display(),
-                                    e
-                                );
-                                match remove_if_empty(&timeline_dir) {
-                                    Ok(true) => info!(
-                                        "Removed empty timeline directory {}",
-                                        timeline_dir.display()
-                                    ),
-                                    Ok(false) => (),
-                                    Err(e) => {
-                                        error!("Failed to remove empty timeline directory: {e:?}")
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            Err(e) => {
-                error!("Failed to list timelines for entry tenant {tenant_id}, reason: {e:?}")
+    // Run in task_mgr to avoid race with detach operation
+    let (task_done, wait_task_done) = tokio::sync::oneshot::channel();
+    task_mgr::spawn(
+        &tokio::runtime::Handle::current(),
+        TaskKind::GarbageCollector,
+        Some(tenant_id),
+        Some(timeline_id),
+        &format!("timeline_gc_handler garbage collection run for tenant {tenant_id} timeline {timeline_id}"),
+        false,
+        async move {
+            fail::fail_point!("immediate_gc_task_pre");
+            let result = tenant
+                .gc_iteration(Some(timeline_id), gc_horizon, pitr, true)
+                .instrument(info_span!("manual_gc", tenant = %tenant_id, timeline = %timeline_id))
+                .await;
+                // FIXME: `gc_iteration` can return an error for multiple reasons; we should handle it
+                // better once the types support it.
+            match task_done.send(result) {
+                Ok(_) => (),
+                Err(result) => error!("failed to send gc result: {result:?}"),
             }
+            Ok(())
         }
-    }
-
-    if tenant_timelines.is_empty() {
-        // this is normal, we've removed all broken, empty and temporary timeline dirs
-        // but should allow the tenant to stay functional and allow creating new timelines
-        // on a restart, we require tenants to have the timelines dir, so leave it on disk
-        debug!("Tenant {tenant_id} has no timelines loaded");
-    }
-
-    Ok((tenant_id, TenantAttachData::Ready(tenant_timelines)))
-}
-
-fn remove_timeline_and_uninit_mark(timeline_dir: &Path, uninit_mark: &Path) -> anyhow::Result<()> {
-    fs::remove_dir_all(&timeline_dir)
-        .or_else(|e| {
-            if e.kind() == std::io::ErrorKind::NotFound {
-                // we can leave the uninit mark without a timeline dir,
-                // just remove the mark then
-                Ok(())
-            } else {
-                Err(e)
-            }
-        })
-        .with_context(|| {
-            format!(
-                "Failed to remove unit marked timeline directory {}",
-                timeline_dir.display()
-            )
-        })?;
-    fs::remove_file(&uninit_mark).with_context(|| {
-        format!(
-            "Failed to remove timeline uninit mark file {}",
-            uninit_mark.display()
-        )
-    })?;
-
-    Ok(())
-}
-
-// discover timeline files and extract timeline metadata
-//  NOTE: ephemeral files are excluded from the list
-fn collect_timeline_files(
-    timeline_dir: &Path,
-) -> anyhow::Result<(TimelineMetadata, HashMap<PathBuf, LayerFileMetadata>)> {
-    let mut timeline_files = HashMap::new();
-    let mut timeline_metadata_path = None;
-
-    let timeline_dir_entries =
-        fs::read_dir(&timeline_dir).context("Failed to list timeline dir contents")?;
-    for entry in timeline_dir_entries {
-        let entry_path = entry.context("Failed to list timeline dir entry")?.path();
-        let metadata = entry_path.metadata()?;
-
-        if metadata.is_file() {
-            if entry_path.file_name().and_then(OsStr::to_str) == Some(METADATA_FILE_NAME) {
-                timeline_metadata_path = Some(entry_path);
-            } else if is_ephemeral_file(&entry_path.file_name().unwrap().to_string_lossy()) {
-                debug!("skipping ephemeral file {}", entry_path.display());
-                continue;
-            } else if is_temporary(&entry_path) {
-                info!("removing temp timeline file at {}", entry_path.display());
-                fs::remove_file(&entry_path).with_context(|| {
-                    format!(
-                        "failed to remove temp download file at {}",
-                        entry_path.display()
-                    )
-                })?;
-            } else {
-                let layer_metadata = LayerFileMetadata::new(metadata.len());
-                timeline_files.insert(entry_path, layer_metadata);
-            }
-        }
-    }
-
-    // FIXME (rodionov) if attach call succeeded, and then pageserver is restarted before download is completed
-    //   then attach is lost. There would be no retries for that,
-    //   initial collect will fail because there is no metadata.
-    //   We either need to start download if we see empty dir after restart or attach caller should
-    //   be aware of that and retry attach if awaits_download for timeline switched from true to false
-    //   but timelinne didn't appear locally.
-    //   Check what happens with remote index in that case.
-    let timeline_metadata_path = match timeline_metadata_path {
-        Some(path) => path,
-        None => anyhow::bail!("No metadata file found in the timeline directory"),
-    };
-    let metadata = TimelineMetadata::from_bytes(
-        &fs::read(&timeline_metadata_path).context("Failed to read timeline metadata file")?,
-    )
-    .context("Failed to parse timeline metadata file bytes")?;
-
-    anyhow::ensure!(
-        metadata.ancestor_timeline().is_some() || !timeline_files.is_empty(),
-        "Timeline has no ancestor and no layer files"
     );
 
-    Ok((metadata, timeline_files))
+    // drop the guard until after we've spawned the task so that timeline shutdown will wait for the task
+    drop(guard);
+
+    Ok(wait_task_done)
 }

pageserver/src/tenant_tasks.rs

@@ -6,7 +6,8 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use crate::metrics::TENANT_TASK_EVENTS;
-use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME};
+use crate::task_mgr;
+use crate::task_mgr::{TaskKind, BACKGROUND_RUNTIME};
 use crate::tenant::{Tenant, TenantState};
 use crate::tenant_mgr;
 use tracing::*;
@@ -65,15 +66,17 @@ async fn compaction_loop(tenant_id: TenantId) {
                 },
             };
 
-            // Run blocking part of the task
-
-            // Run compaction
             let mut sleep_duration = tenant.get_compaction_period();
-            if let Err(e) = tenant.compaction_iteration() {
-                sleep_duration = wait_duration;
-                error!("Compaction failed, retrying in {:?}: {e:#}", sleep_duration);
-                #[cfg(feature = "testing")]
-                std::process::abort();
+            if sleep_duration == Duration::ZERO {
+                info!("automatic compaction is disabled");
+                // check again in 10 seconds, in case it's been enabled again.
+                sleep_duration = Duration::from_secs(10);
+            } else {
+                // Run compaction
+                if let Err(e) = tenant.compaction_iteration().await {
+                    sleep_duration = wait_duration;
+                    error!("Compaction failed, retrying in {:?}: {e:?}", sleep_duration);
+                }
             }
 
             // Sleep
@@ -114,17 +117,21 @@ async fn gc_loop(tenant_id: TenantId) {
                 },
             };
 
-            // Run gc
             let gc_period = tenant.get_gc_period();
             let gc_horizon = tenant.get_gc_horizon();
             let mut sleep_duration = gc_period;
-            if gc_horizon > 0 {
-                if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), false).await
-                {
-                    sleep_duration = wait_duration;
-                    error!("Gc failed, retrying in {:?}: {e:#}", sleep_duration);
-                    #[cfg(feature = "testing")]
-                    std::process::abort();
+            if sleep_duration == Duration::ZERO {
+                info!("automatic GC is disabled");
+                // check again in 10 seconds, in case it's been enabled again.
+                sleep_duration = Duration::from_secs(10);
+            } else {
+                // Run gc
+                if gc_horizon > 0 {
+                    if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), false).await
+                    {
+                        sleep_duration = wait_duration;
+                        error!("Gc failed, retrying in {:?}: {e:?}", sleep_duration);
+                    }
                 }
             }
 
@@ -158,7 +165,7 @@ async fn wait_for_active_tenant(
     };
 
     // if the tenant has a proper status already, no need to wait for anything
-    if tenant.should_run_tasks() {
+    if tenant.current_state() == TenantState::Active {
         ControlFlow::Continue(tenant)
     } else {
         let mut tenant_state_updates = tenant.subscribe_for_state_updates();
@@ -167,14 +174,12 @@ async fn wait_for_active_tenant(
                 Ok(()) => {
                     let new_state = *tenant_state_updates.borrow();
                     match new_state {
-                        TenantState::Active {
-                            background_jobs_running: true,
-                        } => {
-                            debug!("Tenant state changed to active with background jobs enabled, continuing the task loop");
+                        TenantState::Active => {
+                            debug!("Tenant state changed to active, continuing the task loop");
                             return ControlFlow::Continue(tenant);
                         }
                         state => {
-                            debug!("Not running the task loop, tenant is not active with background jobs enabled: {state:?}");
+                            debug!("Not running the task loop, tenant is not active: {state:?}");
                             continue;
                         }
                     }

pageserver/src/trace.rs

@@ -0,0 +1,36 @@
+use bytes::Bytes;
+use std::{
+    fs::{create_dir_all, File},
+    io::{BufWriter, Write},
+    path::PathBuf,
+};
+
+pub struct Tracer {
+    writer: BufWriter<File>,
+}
+
+impl Drop for Tracer {
+    fn drop(&mut self) {
+        self.flush()
+    }
+}
+
+impl Tracer {
+    pub fn new(path: PathBuf) -> Self {
+        let parent = path.parent().expect("failed to parse parent path");
+        create_dir_all(parent).expect("failed to create trace dir");
+
+        let file = File::create(path).expect("failed to create trace file");
+        Tracer {
+            writer: BufWriter::new(file),
+        }
+    }
+
+    pub fn trace(&mut self, msg: &Bytes) {
+        self.writer.write_all(msg).expect("failed to write trace");
+    }
+
+    pub fn flush(&mut self) {
+        self.writer.flush().expect("failed to flush trace file");
+    }
+}

pageserver/src/walingest.rs

@@ -24,7 +24,7 @@
 use anyhow::Context;
 use postgres_ffi::v14::nonrelfile_utils::clogpage_precedes;
 use postgres_ffi::v14::nonrelfile_utils::slru_may_delete_clogsegment;
-use postgres_ffi::{page_is_new, page_set_lsn};
+use postgres_ffi::{fsm_logical_to_physical, page_is_new, page_set_lsn};
 
 use anyhow::Result;
 use bytes::{Buf, Bytes, BytesMut};
@@ -612,20 +612,19 @@ impl<'a> WalIngest<'a> {
                 forknum: FSM_FORKNUM,
             };
 
-            // FIXME: 'blkno' stored in the WAL record is the new size of the
-            // heap. The formula for calculating the new size of the FSM is
-            // pretty complicated (see FreeSpaceMapPrepareTruncateRel() in
-            // PostgreSQL), and we should also clear bits in the tail FSM block,
-            // and update the upper level FSM pages. None of that has been
-            // implemented. What we do instead, is always just truncate the FSM
-            // to zero blocks. That's bad for performance, but safe. (The FSM
-            // isn't needed for correctness, so we could also leave garbage in
-            // it. Seems more tidy to zap it away.)
-            if rec.blkno != 0 {
-                info!("Partial truncation of FSM is not supported");
+            let fsm_logical_page_no = rec.blkno / pg_constants::SLOTS_PER_FSM_PAGE;
+            let mut fsm_physical_page_no = fsm_logical_to_physical(fsm_logical_page_no);
+            if rec.blkno % pg_constants::SLOTS_PER_FSM_PAGE != 0 {
+                // Tail of last remaining FSM page has to be zeroed.
+                // We are not precise here and instead of digging in FSM bitmap format just clear the whole page.
+                modification.put_rel_page_image(rel, fsm_physical_page_no, ZERO_PAGE.clone())?;
+                fsm_physical_page_no += 1;
+            }
+            let nblocks = self.get_relsize(rel, modification.lsn)?;
+            if nblocks > fsm_physical_page_no {
+                // check if something to do: FSM is larger than truncate position
+                self.put_rel_truncation(modification, rel, fsm_physical_page_no)?;
             }
-            let num_fsm_blocks = 0;
-            self.put_rel_truncation(modification, rel, num_fsm_blocks)?;
         }
         if (rec.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
             let rel = RelTag {
@@ -635,16 +634,18 @@ impl<'a> WalIngest<'a> {
                 forknum: VISIBILITYMAP_FORKNUM,
             };
 
-            // FIXME: Like with the FSM above, the logic to truncate the VM
-            // correctly has not been implemented. Just zap it away completely,
-            // always. Unlike the FSM, the VM must never have bits incorrectly
-            // set. From a correctness point of view, it's always OK to clear
-            // bits or remove it altogether, though.
-            if rec.blkno != 0 {
-                info!("Partial truncation of VM is not supported");
+            let mut vm_page_no = rec.blkno / pg_constants::VM_HEAPBLOCKS_PER_PAGE;
+            if rec.blkno % pg_constants::VM_HEAPBLOCKS_PER_PAGE != 0 {
+                // Tail of last remaining vm page has to be zeroed.
+                // We are not precise here and instead of digging in VM bitmap format just clear the whole page.
+                modification.put_rel_page_image(rel, vm_page_no, ZERO_PAGE.clone())?;
+                vm_page_no += 1;
+            }
+            let nblocks = self.get_relsize(rel, modification.lsn)?;
+            if nblocks > vm_page_no {
+                // check if something to do: VM is larger than truncate position
+                self.put_rel_truncation(modification, rel, vm_page_no)?;
             }
-            let num_vm_blocks = 0;
-            self.put_rel_truncation(modification, rel, num_vm_blocks)?;
         }
         Ok(())
     }
@@ -1090,9 +1091,9 @@ mod tests {
         Ok(walingest)
     }
 
-    #[test]
-    fn test_relsize() -> Result<()> {
-        let tenant = TenantHarness::create("test_relsize")?.load();
+    #[tokio::test]
+    async fn test_relsize() -> Result<()> {
+        let tenant = TenantHarness::create("test_relsize")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 
@@ -1218,9 +1219,9 @@ mod tests {
 
     // Test what happens if we dropped a relation
     // and then created it again within the same layer.
-    #[test]
-    fn test_drop_extend() -> Result<()> {
-        let tenant = TenantHarness::create("test_drop_extend")?.load();
+    #[tokio::test]
+    async fn test_drop_extend() -> Result<()> {
+        let tenant = TenantHarness::create("test_drop_extend")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 
@@ -1258,9 +1259,9 @@ mod tests {
     // Test what happens if we truncated a relation
     // so that one of its segments was dropped
     // and then extended it again within the same layer.
-    #[test]
-    fn test_truncate_extend() -> Result<()> {
-        let tenant = TenantHarness::create("test_truncate_extend")?.load();
+    #[tokio::test]
+    async fn test_truncate_extend() -> Result<()> {
+        let tenant = TenantHarness::create("test_truncate_extend")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 
@@ -1346,9 +1347,9 @@ mod tests {
 
     /// Test get_relsize() and truncation with a file larger than 1 GB, so that it's
     /// split into multiple 1 GB segments in Postgres.
-    #[test]
-    fn test_large_rel() -> Result<()> {
-        let tenant = TenantHarness::create("test_large_rel")?.load();
+    #[tokio::test]
+    async fn test_large_rel() -> Result<()> {
+        let tenant = TenantHarness::create("test_large_rel")?.load().await;
         let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?;
         let mut walingest = init_walingest_test(&*tline)?;
 

pageserver/src/walreceiver/connection_manager.rs

@@ -35,6 +35,7 @@ use crate::{
     exponential_backoff, walreceiver::get_etcd_client, DEFAULT_BASE_BACKOFF_SECONDS,
     DEFAULT_MAX_BACKOFF_SECONDS,
 };
+use postgres_connection::{parse_host_port, PgConnectionConfig};
 use utils::{
     id::{NodeId, TenantTimelineId},
     lsn::Lsn,
@@ -49,6 +50,7 @@ pub fn spawn_connection_manager_task(
     wal_connect_timeout: Duration,
     lagging_wal_timeout: Duration,
     max_lsn_wal_lag: NonZeroU64,
+    auth_token: Option<Arc<String>>,
 ) {
     let mut etcd_client = get_etcd_client().clone();
 
@@ -69,6 +71,7 @@ pub fn spawn_connection_manager_task(
                 wal_connect_timeout,
                 lagging_wal_timeout,
                 max_lsn_wal_lag,
+                auth_token,
             );
             loop {
                 select! {
@@ -211,7 +214,7 @@ async fn connection_manager_loop_step(
                             match new_state {
                                 // we're already active as walreceiver, no need to reactivate
                                 TimelineState::Active => continue,
-                                TimelineState::Broken | TimelineState::Paused | TimelineState::Suspended => return ControlFlow::Continue(new_state),
+                                TimelineState::Broken | TimelineState::Stopping | TimelineState::Suspended => return ControlFlow::Continue(new_state),
                             }
                         }
                         Err(_sender_dropped_error) => return ControlFlow::Break(()),
@@ -247,7 +250,7 @@ async fn connection_manager_loop_step(
             walreceiver_state
                 .change_connection(
                     new_candidate.safekeeper_id,
-                    new_candidate.wal_source_connstr,
+                    new_candidate.wal_source_connconf,
                 )
                 .await
         }
@@ -359,6 +362,7 @@ struct WalreceiverState {
     wal_connection_retries: HashMap<NodeId, RetryInfo>,
     /// Data about all timelines, available for connection, fetched from etcd, grouped by their corresponding safekeeper node id.
     wal_stream_candidates: HashMap<NodeId, EtcdSkTimeline>,
+    auth_token: Option<Arc<String>>,
 }
 
 /// Current connection data.
@@ -407,6 +411,7 @@ impl WalreceiverState {
         wal_connect_timeout: Duration,
         lagging_wal_timeout: Duration,
         max_lsn_wal_lag: NonZeroU64,
+        auth_token: Option<Arc<String>>,
     ) -> Self {
         let id = TenantTimelineId {
             tenant_id: timeline.tenant_id,
@@ -421,11 +426,16 @@ impl WalreceiverState {
             wal_connection: None,
             wal_stream_candidates: HashMap::new(),
             wal_connection_retries: HashMap::new(),
+            auth_token,
         }
     }
 
     /// Shuts down the current connection (if any) and immediately starts another one with the given connection string.
-    async fn change_connection(&mut self, new_sk_id: NodeId, new_wal_source_connstr: String) {
+    async fn change_connection(
+        &mut self,
+        new_sk_id: NodeId,
+        new_wal_source_connconf: PgConnectionConfig,
+    ) {
         self.drop_old_connection(true).await;
 
         let id = self.id;
@@ -435,7 +445,7 @@ impl WalreceiverState {
             async move {
                 super::walreceiver_connection::handle_walreceiver_connection(
                     timeline,
-                    new_wal_source_connstr,
+                    new_wal_source_connconf,
                     events_sender,
                     cancellation,
                     connect_timeout,
@@ -575,7 +585,7 @@ impl WalreceiverState {
             Some(existing_wal_connection) => {
                 let connected_sk_node = existing_wal_connection.sk_id;
 
-                let (new_sk_id, new_safekeeper_etcd_data, new_wal_source_connstr) =
+                let (new_sk_id, new_safekeeper_etcd_data, new_wal_source_connconf) =
                     self.select_connection_candidate(Some(connected_sk_node))?;
 
                 let now = Utc::now().naive_utc();
@@ -586,7 +596,7 @@ impl WalreceiverState {
                     if latest_interaciton > self.wal_connect_timeout {
                         return Some(NewWalConnectionCandidate {
                             safekeeper_id: new_sk_id,
-                            wal_source_connstr: new_wal_source_connstr,
+                            wal_source_connconf: new_wal_source_connconf,
                             reason: ReconnectReason::NoKeepAlives {
                                 last_keep_alive: Some(
                                     existing_wal_connection.status.latest_connection_update,
@@ -611,7 +621,7 @@ impl WalreceiverState {
                             if new_sk_lsn_advantage >= self.max_lsn_wal_lag.get() {
                                 return Some(NewWalConnectionCandidate {
                                     safekeeper_id: new_sk_id,
-                                    wal_source_connstr: new_wal_source_connstr,
+                                    wal_source_connconf: new_wal_source_connconf,
                                     reason: ReconnectReason::LaggingWal {
                                         current_commit_lsn,
                                         new_commit_lsn,
@@ -685,7 +695,7 @@ impl WalreceiverState {
                         {
                             return Some(NewWalConnectionCandidate {
                                 safekeeper_id: new_sk_id,
-                                wal_source_connstr: new_wal_source_connstr,
+                                wal_source_connconf: new_wal_source_connconf,
                                 reason: ReconnectReason::NoWalTimeout {
                                     current_lsn,
                                     current_commit_lsn,
@@ -704,11 +714,11 @@ impl WalreceiverState {
                 self.wal_connection.as_mut().unwrap().discovered_new_wal = discovered_new_wal;
             }
             None => {
-                let (new_sk_id, _, new_wal_source_connstr) =
+                let (new_sk_id, _, new_wal_source_connconf) =
                     self.select_connection_candidate(None)?;
                 return Some(NewWalConnectionCandidate {
                     safekeeper_id: new_sk_id,
-                    wal_source_connstr: new_wal_source_connstr,
+                    wal_source_connconf: new_wal_source_connconf,
                     reason: ReconnectReason::NoExistingConnection,
                 });
             }
@@ -726,7 +736,7 @@ impl WalreceiverState {
     fn select_connection_candidate(
         &self,
         node_to_omit: Option<NodeId>,
-    ) -> Option<(NodeId, &SkTimelineInfo, String)> {
+    ) -> Option<(NodeId, &SkTimelineInfo, PgConnectionConfig)> {
         self.applicable_connection_candidates()
             .filter(|&(sk_id, _, _)| Some(sk_id) != node_to_omit)
             .max_by_key(|(_, info, _)| info.commit_lsn)
@@ -736,7 +746,7 @@ impl WalreceiverState {
     /// Some safekeepers are filtered by the retry cooldown.
     fn applicable_connection_candidates(
         &self,
-    ) -> impl Iterator<Item = (NodeId, &SkTimelineInfo, String)> {
+    ) -> impl Iterator<Item = (NodeId, &SkTimelineInfo, PgConnectionConfig)> {
         let now = Utc::now().naive_utc();
 
         self.wal_stream_candidates
@@ -754,9 +764,13 @@ impl WalreceiverState {
             })
             .filter_map(|(sk_id, etcd_info)| {
                 let info = &etcd_info.timeline;
-                match wal_stream_connection_string(
+                match wal_stream_connection_config(
                     self.id,
                     info.safekeeper_connstr.as_deref()?,
+                    match &self.auth_token {
+                        None => None,
+                        Some(x) => Some(x),
+                    },
                 ) {
                     Ok(connstr) => Some((*sk_id, info, connstr)),
                     Err(e) => {
@@ -797,10 +811,12 @@ impl WalreceiverState {
     }
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug)]
 struct NewWalConnectionCandidate {
     safekeeper_id: NodeId,
-    wal_source_connstr: String,
+    wal_source_connconf: PgConnectionConfig,
+    // This field is used in `derive(Debug)` only.
+    #[allow(dead_code)]
     reason: ReconnectReason,
 }
 
@@ -828,39 +844,36 @@ enum ReconnectReason {
     },
 }
 
-fn wal_stream_connection_string(
+fn wal_stream_connection_config(
     TenantTimelineId {
         tenant_id,
         timeline_id,
     }: TenantTimelineId,
     listen_pg_addr_str: &str,
-) -> anyhow::Result<String> {
-    let sk_connstr = format!("postgresql://no_user@{listen_pg_addr_str}/no_db");
-    sk_connstr
-        .parse()
-        .context("bad url")
-        .and_then(|url: url::Url| {
-            let host = url.host_str().context("host is missing")?;
-            let port = url.port().unwrap_or(5432); // default PG port
-
-            Ok(format!(
-                "host={host} \
-                 port={port} \
-                 options='-c timeline_id={timeline_id} tenant_id={tenant_id}'"
-            ))
-        })
-        .with_context(|| format!("Failed to parse pageserver connection URL '{sk_connstr}'"))
+    auth_token: Option<&str>,
+) -> anyhow::Result<PgConnectionConfig> {
+    let (host, port) =
+        parse_host_port(&listen_pg_addr_str).context("Unable to parse listen_pg_addr_str")?;
+    let port = port.unwrap_or(5432);
+    Ok(PgConnectionConfig::new_host_port(host, port)
+        .extend_options([
+            "-c".to_owned(),
+            format!("timeline_id={}", timeline_id),
+            format!("tenant_id={}", tenant_id),
+        ])
+        .set_password(auth_token.map(|s| s.to_owned())))
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::tenant::harness::{TenantHarness, TIMELINE_ID};
+    use url::Host;
 
-    #[test]
-    fn no_connection_no_candidate() -> anyhow::Result<()> {
+    #[tokio::test]
+    async fn no_connection_no_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("no_connection_no_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         let lagging_wal_timeout = chrono::Duration::from_std(state.lagging_wal_timeout)?;
@@ -951,7 +964,7 @@ mod tests {
     #[tokio::test]
     async fn connection_no_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("connection_no_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         let connected_sk_id = NodeId(0);
@@ -992,7 +1005,7 @@ mod tests {
                         peer_horizon_lsn: None,
                         local_start_lsn: None,
 
-                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                     },
                     etcd_version: 0,
                     latest_update: now,
@@ -1045,10 +1058,10 @@ mod tests {
         Ok(())
     }
 
-    #[test]
-    fn no_connection_candidate() -> anyhow::Result<()> {
+    #[tokio::test]
+    async fn no_connection_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("no_connection_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         state.wal_connection = None;
@@ -1064,7 +1077,7 @@ mod tests {
                     peer_horizon_lsn: None,
                     local_start_lsn: None,
 
-                    safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                    safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                 },
                 etcd_version: 0,
                 latest_update: now,
@@ -1080,9 +1093,10 @@ mod tests {
             ReconnectReason::NoExistingConnection,
             "Should select new safekeeper due to missing connection, even if there's also a lag in the wal over the threshold"
         );
-        assert!(only_candidate
-            .wal_source_connstr
-            .contains(DUMMY_SAFEKEEPER_CONNSTR));
+        assert_eq!(
+            only_candidate.wal_source_connconf.host(),
+            &Host::Domain(DUMMY_SAFEKEEPER_HOST.to_owned())
+        );
 
         let selected_lsn = 100_000;
         state.wal_stream_candidates = HashMap::from([
@@ -1116,7 +1130,7 @@ mod tests {
                         peer_horizon_lsn: None,
                         local_start_lsn: None,
 
-                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                     },
                     etcd_version: 0,
                     latest_update: now,
@@ -1151,9 +1165,10 @@ mod tests {
             ReconnectReason::NoExistingConnection,
             "Should select new safekeeper due to missing connection, even if there's also a lag in the wal over the threshold"
         );
-        assert!(biggest_wal_candidate
-            .wal_source_connstr
-            .contains(DUMMY_SAFEKEEPER_CONNSTR));
+        assert_eq!(
+            biggest_wal_candidate.wal_source_connconf.host(),
+            &Host::Domain(DUMMY_SAFEKEEPER_HOST.to_owned())
+        );
 
         Ok(())
     }
@@ -1161,7 +1176,7 @@ mod tests {
     #[tokio::test]
     async fn candidate_with_many_connection_failures() -> anyhow::Result<()> {
         let harness = TenantHarness::create("candidate_with_many_connection_failures")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let now = Utc::now().naive_utc();
 
         let current_lsn = Lsn(100_000).align();
@@ -1181,7 +1196,7 @@ mod tests {
                         peer_horizon_lsn: None,
                         local_start_lsn: None,
 
-                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                     },
                     etcd_version: 0,
                     latest_update: now,
@@ -1199,7 +1214,7 @@ mod tests {
                         peer_horizon_lsn: None,
                         local_start_lsn: None,
 
-                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                     },
                     etcd_version: 0,
                     latest_update: now,
@@ -1229,7 +1244,7 @@ mod tests {
     #[tokio::test]
     async fn lsn_wal_over_threshhold_current_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let current_lsn = Lsn(100_000).align();
         let now = Utc::now().naive_utc();
 
@@ -1270,7 +1285,7 @@ mod tests {
                         peer_horizon_lsn: None,
                         local_start_lsn: None,
 
-                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                        safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                     },
                     etcd_version: 0,
                     latest_update: now,
@@ -1310,9 +1325,10 @@ mod tests {
             },
             "Should select bigger WAL safekeeper if it starts to lag enough"
         );
-        assert!(over_threshcurrent_candidate
-            .wal_source_connstr
-            .contains("advanced_by_lsn_safekeeper"));
+        assert_eq!(
+            over_threshcurrent_candidate.wal_source_connconf.host(),
+            &Host::Domain("advanced_by_lsn_safekeeper".to_owned())
+        );
 
         Ok(())
     }
@@ -1320,7 +1336,7 @@ mod tests {
     #[tokio::test]
     async fn timeout_connection_threshhold_current_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("timeout_connection_threshhold_current_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let current_lsn = Lsn(100_000).align();
         let now = Utc::now().naive_utc();
 
@@ -1361,7 +1377,7 @@ mod tests {
                     peer_horizon_lsn: None,
                     local_start_lsn: None,
 
-                    safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                    safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                 },
                 etcd_version: 0,
                 latest_update: now,
@@ -1384,9 +1400,10 @@ mod tests {
             }
             unexpected => panic!("Unexpected reason: {unexpected:?}"),
         }
-        assert!(over_threshcurrent_candidate
-            .wal_source_connstr
-            .contains(DUMMY_SAFEKEEPER_CONNSTR));
+        assert_eq!(
+            over_threshcurrent_candidate.wal_source_connconf.host(),
+            &Host::Domain(DUMMY_SAFEKEEPER_HOST.to_owned())
+        );
 
         Ok(())
     }
@@ -1394,7 +1411,7 @@ mod tests {
     #[tokio::test]
     async fn timeout_wal_over_threshhold_current_candidate() -> anyhow::Result<()> {
         let harness = TenantHarness::create("timeout_wal_over_threshhold_current_candidate")?;
-        let mut state = dummy_state(&harness);
+        let mut state = dummy_state(&harness).await;
         let current_lsn = Lsn(100_000).align();
         let new_lsn = Lsn(100_100).align();
         let now = Utc::now().naive_utc();
@@ -1434,7 +1451,7 @@ mod tests {
                     peer_horizon_lsn: None,
                     local_start_lsn: None,
 
-                    safekeeper_connstr: Some(DUMMY_SAFEKEEPER_CONNSTR.to_string()),
+                    safekeeper_connstr: Some(DUMMY_SAFEKEEPER_HOST.to_string()),
                 },
                 etcd_version: 0,
                 latest_update: now,
@@ -1463,16 +1480,17 @@ mod tests {
             }
             unexpected => panic!("Unexpected reason: {unexpected:?}"),
         }
-        assert!(over_threshcurrent_candidate
-            .wal_source_connstr
-            .contains(DUMMY_SAFEKEEPER_CONNSTR));
+        assert_eq!(
+            over_threshcurrent_candidate.wal_source_connconf.host(),
+            &Host::Domain(DUMMY_SAFEKEEPER_HOST.to_owned())
+        );
 
         Ok(())
     }
 
-    const DUMMY_SAFEKEEPER_CONNSTR: &str = "safekeeper_connstr";
+    const DUMMY_SAFEKEEPER_HOST: &str = "safekeeper_connstr";
 
-    fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState {
+    async fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState {
         WalreceiverState {
             id: TenantTimelineId {
                 tenant_id: harness.tenant_id,
@@ -1480,6 +1498,7 @@ mod tests {
             },
             timeline: harness
                 .load()
+                .await
                 .create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION)
                 .expect("Failed to create an empty timeline for dummy wal connection manager")
                 .initialize()
@@ -1490,6 +1509,7 @@ mod tests {
             wal_connection: None,
             wal_stream_candidates: HashMap::new(),
             wal_connection_retries: HashMap::new(),
+            auth_token: None,
         }
     }
 }

pageserver/src/walreceiver/walreceiver_connection.rs

@@ -26,13 +26,13 @@ use crate::{
     task_mgr::TaskKind,
     task_mgr::WALRECEIVER_RUNTIME,
     tenant::{Timeline, WalReceiverInfo},
-    tenant_mgr,
     walingest::WalIngest,
     walrecord::DecodedWALRecord,
 };
+use postgres_connection::PgConnectionConfig;
 use postgres_ffi::waldecoder::WalStreamDecoder;
 use pq_proto::ReplicationFeedback;
-use utils::{id::TenantTimelineId, lsn::Lsn};
+use utils::lsn::Lsn;
 
 /// Status of the connection.
 #[derive(Debug, Clone)]
@@ -56,22 +56,23 @@ pub struct WalConnectionStatus {
 /// messages as we go.
 pub async fn handle_walreceiver_connection(
     timeline: Arc<Timeline>,
-    wal_source_connstr: String,
+    wal_source_connconf: PgConnectionConfig,
     events_sender: watch::Sender<TaskStateUpdate<WalConnectionStatus>>,
     mut cancellation: watch::Receiver<()>,
     connect_timeout: Duration,
 ) -> anyhow::Result<()> {
     // Connect to the database in replication mode.
-    info!("connecting to {wal_source_connstr}");
-    let connect_cfg = format!("{wal_source_connstr} application_name=pageserver replication=true");
+    info!("connecting to {wal_source_connconf:?}");
 
-    let (mut replication_client, connection) = time::timeout(
-        connect_timeout,
-        tokio_postgres::connect(&connect_cfg, postgres::NoTls),
-    )
-    .await
-    .context("Timed out while waiting for walreceiver connection to open")?
-    .context("Failed to open walreceiver connection")?;
+    let (mut replication_client, connection) = {
+        let mut config = wal_source_connconf.to_tokio_postgres_config();
+        config.application_name("pageserver");
+        config.replication_mode(tokio_postgres::config::ReplicationMode::Physical);
+        time::timeout(connect_timeout, config.connect(postgres::NoTls))
+            .await
+            .context("Timed out while waiting for walreceiver connection to open")?
+            .context("Failed to open walreceiver connection")?
+    };
 
     info!("connected!");
     let mut connection_status = WalConnectionStatus {
@@ -139,10 +140,6 @@ pub async fn handle_walreceiver_connection(
         return Ok(());
     }
 
-    let tenant_id = timeline.tenant_id;
-    let timeline_id = timeline.timeline_id;
-    let tenant = tenant_mgr::get_tenant(tenant_id, true)?;
-
     //
     // Start streaming the WAL, from where we left off previously.
     //
@@ -291,19 +288,8 @@ pub async fn handle_walreceiver_connection(
         })?;
 
         if let Some(last_lsn) = status_update {
-            let remote_index = tenant.get_remote_index();
-            let timeline_remote_consistent_lsn = remote_index
-                .read()
-                .await
-                // here we either do not have this timeline in remote index
-                // or there were no checkpoints for it yet
-                .timeline_entry(&TenantTimelineId {
-                    tenant_id,
-                    timeline_id,
-                })
-                .map(|remote_timeline| remote_timeline.metadata.disk_consistent_lsn())
-                // no checkpoint was uploaded
-                .unwrap_or(Lsn(0));
+            let timeline_remote_consistent_lsn =
+                timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));
 
             // The last LSN we processed. It is not guaranteed to survive pageserver crash.
             let write_lsn = u64::from(last_lsn);
@@ -316,7 +302,7 @@ pub async fn handle_walreceiver_connection(
 
             // Update the status about what we just received. This is shown in the mgmt API.
             let last_received_wal = WalReceiverInfo {
-                wal_source_connstr: wal_source_connstr.to_owned(),
+                wal_source_connconf: wal_source_connconf.clone(),
                 last_received_msg_lsn: last_lsn,
                 last_received_msg_ts: ts
                     .duration_since(SystemTime::UNIX_EPOCH)

pageserver/src/walredo.rs

@@ -21,21 +21,20 @@
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::{BufMut, Bytes, BytesMut};
 use nix::poll::*;
-use once_cell::sync::Lazy;
 use serde::Serialize;
-use std::fs;
 use std::fs::OpenOptions;
 use std::io::prelude::*;
 use std::io::{Error, ErrorKind};
+use std::ops::{Deref, DerefMut};
 use std::os::unix::io::AsRawFd;
 use std::os::unix::prelude::CommandExt;
 use std::path::PathBuf;
 use std::process::Stdio;
 use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command};
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::{Condvar, Mutex};
+use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
+use std::{fs, io};
 use tracing::*;
 use utils::crashsafe::path_with_suffix_extension;
 use utils::{bin_ser::BeSer, id::TenantId, lsn::Lsn, nonblock::set_nonblock};
@@ -46,6 +45,7 @@ use crate::metrics::{
 };
 use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block};
 use crate::repository::Key;
+use crate::task_mgr::BACKGROUND_RUNTIME;
 use crate::walrecord::NeonWalRecord;
 use crate::{config::PageServerConf, TEMP_FILE_SUFFIX};
 use pageserver_api::reltag::{RelTag, SlruKind};
@@ -57,9 +57,6 @@ use postgres_ffi::v14::nonrelfile_utils::{
 };
 use postgres_ffi::BLCKSZ;
 
-/// Maximum number of WAL redo processes to launch for a single tenant.
-const MAX_PROCESSES: usize = 4;
-
 ///
 /// `RelTag` + block number (`blknum`) gives us a unique id of the page in the cluster.
 ///
@@ -93,32 +90,18 @@ pub trait WalRedoManager: Send + Sync {
     ) -> Result<Bytes, WalRedoError>;
 }
 
-static WAL_REDO_PROCESS_COUNTER: Lazy<AtomicU64> = Lazy::new(|| { AtomicU64::new(0) });
-
 ///
-/// This is the real implementation that uses a special Postgres
-/// process to perform WAL replay. There is a pool of these processes.
+/// This is the real implementation that uses a Postgres process to
+/// perform WAL replay. Only one thread can use the process at a time,
+/// that is controlled by the Mutex. In the future, we might want to
+/// launch a pool of processes to allow concurrent replay of multiple
+/// records.
 ///
 pub struct PostgresRedoManager {
     tenant_id: TenantId,
     conf: &'static PageServerConf,
 
-    /// Pool of processes.
-    process_list: Mutex<ProcessList>,
-    /// Condition variable that can be used to sleep until a process
-    /// becomes available in the pool.
-    condvar: Condvar,
-}
-
-// A pool of WAL redo processes
-#[derive(Default)]
-struct ProcessList {
-    /// processes that are available for reuse
-    free_processes: Vec<PostgresRedoProcess>,
-
-    /// Total number of processes, including all the processes in
-    /// 'free_processes' list, and any processes that are in use.
-    num_processes: usize,
+    process: Mutex<Option<PostgresRedoProcess>>,
 }
 
 /// Can this request be served by neon redo functions
@@ -223,33 +206,18 @@ impl PostgresRedoManager {
         PostgresRedoManager {
             tenant_id,
             conf,
-            process_list: Mutex::new(ProcessList::default()),
-            condvar: Condvar::new(),
+            process: Mutex::new(None),
         }
     }
 
-    // Get a handle to a redo process from the pool.
-    fn get_process(&self, pg_version: u32) -> Result<PostgresRedoProcess, WalRedoError> {
-        let mut process_list = self.process_list.lock().unwrap();
-
-        loop {
-            // If there's a free process immediately available, take it.
-            if let Some(process) = process_list.free_processes.pop() {
-                return Ok(process);
-            }
-
-            // All processes are in use. If the pool is at its maximum size
-            // already, wait for a process to become free. Otherwise launch
-            // a new process.
-            if process_list.num_processes >= MAX_PROCESSES {
-                process_list = self.condvar.wait(process_list).unwrap();
-                continue;
-            } else {
-                let process = PostgresRedoProcess::launch(self.conf, &self.tenant_id, pg_version)?;
-                process_list.num_processes += 1;
-                return Ok(process);
-            }
+    /// Launch process pre-emptively. Should not be needed except for benchmarking.
+    pub fn launch_process(&mut self, pg_version: u32) -> anyhow::Result<()> {
+        let inner = self.process.get_mut().unwrap();
+        if inner.is_none() {
+            let p = PostgresRedoProcess::launch(self.conf, self.tenant_id, pg_version)?;
+            *inner = Some(p);
         }
+        Ok(())
     }
 
     ///
@@ -268,10 +236,16 @@ impl PostgresRedoManager {
 
         let start_time = Instant::now();
 
-        let mut process = self.get_process(pg_version)?;
-
+        let mut process_guard = self.process.lock().unwrap();
         let lock_time = Instant::now();
 
+        // launch the WAL redo process on first use
+        if process_guard.is_none() {
+            let p = PostgresRedoProcess::launch(self.conf, self.tenant_id, pg_version)?;
+            *process_guard = Some(p);
+        }
+        let process = process_guard.as_mut().unwrap();
+
         WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64());
 
         // Relational WAL records are applied using wal-redo-postgres
@@ -304,9 +278,8 @@ impl PostgresRedoManager {
             lsn
         );
 
-        // If something went wrong, don't try to reuse the
-        // process. Kill it, and next request will launch a new one.
-        // Otherwise return the process to the pool.
+        // If something went wrong, don't try to reuse the process. Kill it, and
+        // next request will launch a new one.
         if result.is_err() {
             error!(
                 "error applying {} WAL records ({} bytes) to reconstruct page image at LSN {}",
@@ -314,14 +287,8 @@ impl PostgresRedoManager {
                 nbytes,
                 lsn
             );
+            let process = process_guard.take().unwrap();
             process.kill();
-            let mut process_list = self.process_list.lock().unwrap();
-            process_list.num_processes -= 1;
-            self.condvar.notify_one();
-        } else {
-            let mut process_list = self.process_list.lock().unwrap();
-            process_list.free_processes.push(process);
-            self.condvar.notify_one();
         }
         result
     }
@@ -624,7 +591,8 @@ impl<C: CommandExt> CloseFileDescriptors for C {
 /// Handle to the Postgres WAL redo process
 ///
 struct PostgresRedoProcess {
-    child: Child,
+    tenant_id: TenantId,
+    child: NoLeakChild,
     stdin: ChildStdin,
     stdout: ChildStdout,
     stderr: ChildStderr,
@@ -634,15 +602,17 @@ impl PostgresRedoProcess {
     //
     // Start postgres binary in special WAL redo mode.
     //
+    #[instrument(skip_all,fields(tenant_id=%tenant_id, pg_version=pg_version))]
     fn launch(
         conf: &PageServerConf,
-        tenant_id: &TenantId,
+        tenant_id: TenantId,
         pg_version: u32,
     ) -> Result<PostgresRedoProcess, Error> {
-        // We need a dummy Postgres cluster to run the process in.
-        let processno = WAL_REDO_PROCESS_COUNTER.fetch_add(1, Ordering::Relaxed);
+        // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we
+        // just create one with constant name. That fails if you try to launch more than
+        // one WAL redo manager concurrently.
         let datadir = path_with_suffix_extension(
-            conf.tenant_path(tenant_id).join(format!("wal-redo-datadir-{}", processno)),
+            conf.tenant_path(&tenant_id).join("wal-redo-datadir"),
             TEMP_FILE_SUFFIX,
         );
 
@@ -697,7 +667,7 @@ impl PostgresRedoProcess {
         }
 
         // Start postgres itself
-        let mut child = Command::new(pg_bin_dir_path.join("postgres"))
+        let child = Command::new(pg_bin_dir_path.join("postgres"))
             .arg("--wal-redo")
             .stdin(Stdio::piped())
             .stderr(Stdio::piped())
@@ -716,7 +686,7 @@ impl PostgresRedoProcess {
             // as close-on-exec by default, but that's not enough, since we use
             // libraries that directly call libc open without setting that flag.
             .close_fds()
-            .spawn()
+            .spawn_no_leak_child()
             .map_err(|e| {
                 Error::new(
                     e.kind(),
@@ -724,20 +694,33 @@ impl PostgresRedoProcess {
                 )
             })?;
 
-        info!(
-            "launched WAL redo postgres process on {}",
-            datadir.display()
-        );
+        let mut child = scopeguard::guard(child, |child| {
+            error!("killing wal-redo-postgres process due to a problem during launch");
+            child.kill_and_wait();
+        });
 
         let stdin = child.stdin.take().unwrap();
         let stdout = child.stdout.take().unwrap();
         let stderr = child.stderr.take().unwrap();
 
-        set_nonblock(stdin.as_raw_fd())?;
-        set_nonblock(stdout.as_raw_fd())?;
-        set_nonblock(stderr.as_raw_fd())?;
+        macro_rules! set_nonblock_or_log_err {
+            ($file:ident) => {{
+                let res = set_nonblock($file.as_raw_fd());
+                if let Err(e) = &res {
+                    error!(error = %e, file = stringify!($file), pid = child.id(), "set_nonblock failed");
+                }
+                res
+            }};
+        }
+        set_nonblock_or_log_err!(stdin)?;
+        set_nonblock_or_log_err!(stdout)?;
+        set_nonblock_or_log_err!(stderr)?;
+
+        // all fallible operations post-spawn are complete, so get rid of the guard
+        let child = scopeguard::ScopeGuard::into_inner(child);
 
         Ok(PostgresRedoProcess {
+            tenant_id,
             child,
             stdin,
             stdout,
@@ -745,18 +728,16 @@ impl PostgresRedoProcess {
         })
     }
 
-    fn kill(mut self) {
-        let _ = self.child.kill();
-        if let Ok(exit_status) = self.child.wait() {
-            error!("wal-redo-postgres exited with code {}", exit_status);
-        }
-        drop(self);
+    #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%self.child.id()))]
+    fn kill(self) {
+        self.child.kill_and_wait();
     }
 
     //
     // Apply given WAL records ('records') over an old page image. Returns
     // new page image.
     //
+    #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%self.child.id()))]
     fn apply_wal_records(
         &mut self,
         tag: BufferTag,
@@ -886,6 +867,99 @@ impl PostgresRedoProcess {
     }
 }
 
+/// Wrapper type around `std::process::Child` which guarantees that the child
+/// will be killed and waited-for by this process before being dropped.
+struct NoLeakChild {
+    child: Option<Child>,
+}
+
+impl Deref for NoLeakChild {
+    type Target = Child;
+
+    fn deref(&self) -> &Self::Target {
+        self.child.as_ref().expect("must not use from drop")
+    }
+}
+
+impl DerefMut for NoLeakChild {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.child.as_mut().expect("must not use from drop")
+    }
+}
+
+impl NoLeakChild {
+    fn spawn(command: &mut Command) -> io::Result<Self> {
+        let child = command.spawn()?;
+        Ok(NoLeakChild { child: Some(child) })
+    }
+
+    fn kill_and_wait(mut self) {
+        let child = match self.child.take() {
+            Some(child) => child,
+            None => return,
+        };
+        Self::kill_and_wait_impl(child);
+    }
+
+    #[instrument(skip_all, fields(pid=child.id()))]
+    fn kill_and_wait_impl(mut child: Child) {
+        let res = child.kill();
+        if let Err(e) = res {
+            // This branch is very unlikely because:
+            // - We (= pageserver) spawned this process successfully, so, we're allowed to kill it.
+            // - This is the only place that calls .kill()
+            // - We consume `self`, so, .kill() can't be called twice.
+            // - If the process exited by itself or was killed by someone else,
+            //   .kill() will still succeed because we haven't wait()'ed yet.
+            //
+            // So, if we arrive here, we have really no idea what happened,
+            // whether the PID stored in self.child is still valid, etc.
+            // If this function were fallible, we'd return an error, but
+            // since it isn't, all we can do is log an error and proceed
+            // with the wait().
+            error!(error = %e, "failed to SIGKILL; subsequent wait() might fail or wait for wrong process");
+        }
+
+        match child.wait() {
+            Ok(exit_status) => {
+                // log at error level since .kill() is something we only do on errors ATM
+                error!(exit_status = %exit_status, "wait successful");
+            }
+            Err(e) => {
+                error!(error = %e, "wait error; might leak the child process; it will show as zombie (defunct)");
+            }
+        }
+    }
+}
+
+impl Drop for NoLeakChild {
+    fn drop(&mut self) {
+        let child = match self.child.take() {
+            Some(child) => child,
+            None => return,
+        };
+        // Offload the kill+wait of the child process into the background.
+        // If someone stops the runtime, we'll leak the child process.
+        // We can ignore that case because we only stop the runtime on pageserver exit.
+        BACKGROUND_RUNTIME.spawn(async move {
+            tokio::task::spawn_blocking(move || {
+                Self::kill_and_wait_impl(child);
+            })
+            .await
+        });
+    }
+}
+
+trait NoLeakChildCommandExt {
+    fn spawn_no_leak_child(&mut self) -> io::Result<NoLeakChild>;
+}
+
+impl NoLeakChildCommandExt for Command {
+    fn spawn_no_leak_child(&mut self) -> io::Result<NoLeakChild> {
+        NoLeakChild::spawn(self)
+    }
+}
+
 // Functions for constructing messages to send to the postgres WAL redo
 // process. See pgxn/neon_walredo/walredoproc.c for
 // explanation of the protocol.

pgxn/neon/libpagestore.c

@@ -32,11 +32,6 @@
 
 #define PageStoreTrace DEBUG5
 
-#define NEON_TAG "[NEON_SMGR] "
-#define neon_log(tag, fmt, ...) ereport(tag,                                  \
-										(errmsg(NEON_TAG fmt, ##__VA_ARGS__), \
-										 errhidestmt(true), errhidecontext(true)))
-
 bool		connected = false;
 PGconn	   *pageserver_conn = NULL;
 
@@ -49,8 +44,11 @@ PGconn	   *pageserver_conn = NULL;
 WaitEventSet *pageserver_conn_wes = NULL;
 
 char	   *page_server_connstring_raw;
+char	   *safekeeper_token_env;
 
+int			n_unflushed_requests = 0;
 int			flush_every_n_requests = 8;
+int			readahead_buffer_size = 128;
 
 static void pageserver_flush(void);
 
@@ -95,11 +93,10 @@ pageserver_connect()
 
 	while (PQisBusy(pageserver_conn))
 	{
-		int			wc;
 		WaitEvent	event;
 
 		/* Sleep until there's something to do */
-		wc = WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION);
+		(void) WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION);
 		ResetLatch(MyLatch);
 
 		CHECK_FOR_INTERRUPTS();
@@ -139,11 +136,10 @@ retry:
 
 	if (ret == 0)
 	{
-		int			wc;
 		WaitEvent	event;
 
 		/* Sleep until there's something to do */
-		wc = WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION);
+		(void) WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION);
 		ResetLatch(MyLatch);
 
 		CHECK_FOR_INTERRUPTS();
@@ -217,6 +213,11 @@ pageserver_send(NeonRequest * request)
 	}
 	pfree(req_buff.data);
 
+	n_unflushed_requests++;
+
+	if (flush_every_n_requests > 0 && n_unflushed_requests >= flush_every_n_requests)
+		pageserver_flush();
+
 	if (message_level_is_interesting(PageStoreTrace))
 	{
 		char	   *msg = nm_to_string((NeonMessage *) request);
@@ -232,6 +233,9 @@ pageserver_receive(void)
 	StringInfoData resp_buff;
 	NeonResponse *resp;
 
+	if (!connected)
+		return NULL;
+
 	PG_TRY();
 	{
 		/* read response */
@@ -241,7 +245,10 @@ pageserver_receive(void)
 		if (resp_buff.len < 0)
 		{
 			if (resp_buff.len == -1)
-				neon_log(ERROR, "end of COPY");
+			{
+				pageserver_disconnect();
+				return NULL;
+			}
 			else if (resp_buff.len == -2)
 				neon_log(ERROR, "could not read COPY data: %s", PQerrorMessage(pageserver_conn));
 		}
@@ -281,6 +288,7 @@ pageserver_flush(void)
 		pageserver_disconnect();
 		neon_log(ERROR, "failed to flush page requests: %s", msg);
 	}
+	n_unflushed_requests = 0;
 }
 
 page_server_api api = {
@@ -411,6 +419,15 @@ pg_init_libpagestore(void)
 							   0,	/* no flags required */
 							   NULL, NULL, NULL);
 
+    DefineCustomStringVariable("neon.safekeeper_token_env",
+                               "the environment variable containing JWT token for authentication with Safekeepers, the convention is to either unset or set to $ZENITH_AUTH_TOKEN",
+                               NULL,
+                               &safekeeper_token_env,
+                               NULL,
+                               PGC_POSTMASTER,
+                               0,	/* no flags required */
+                               NULL, NULL, NULL);
+
 	DefineCustomStringVariable("neon.timeline_id",
 							   "Neon timeline_id the server is running on",
 							   NULL,
@@ -445,6 +462,19 @@ pg_init_libpagestore(void)
 							PGC_USERSET,
 							0,	/* no flags required */
 							NULL, NULL, NULL);
+	DefineCustomIntVariable("neon.readahead_buffer_size",
+							"number of prefetches to buffer",
+							"This buffer is used to store prefetched data; so "
+							"it is important that this buffer is at least as "
+							"large as the configured value of all tablespaces' "
+							"effective_io_concurrency and maintenance_io_concurrency, "
+							"your sessions' values of these, and the value for "
+							"seqscan_prefetch_buffers.",
+							&readahead_buffer_size,
+							128, 16, 1024,
+							PGC_USERSET,
+							0,	/* no flags required */
+							NULL, (GucIntAssignHook) &readahead_buffer_resize, NULL);
 
 	relsize_hash_init();
 
@@ -461,6 +491,24 @@ pg_init_libpagestore(void)
 	neon_timeline_walproposer = neon_timeline;
 	neon_tenant_walproposer = neon_tenant;
 
+	/* retrieve the token for Safekeeper, if present */
+	if (safekeeper_token_env != NULL) {
+		if (safekeeper_token_env[0] != '$') {
+			ereport(ERROR,
+					(errcode(ERRCODE_CONNECTION_EXCEPTION),
+							errmsg("expected safekeeper auth token environment variable's name starting with $ but found: %s",
+								   safekeeper_token_env)));
+		}
+		neon_safekeeper_token_walproposer = getenv(&safekeeper_token_env[1]);
+		if (!neon_safekeeper_token_walproposer) {
+			ereport(ERROR,
+					(errcode(ERRCODE_CONNECTION_EXCEPTION),
+							errmsg("cannot get safekeeper auth token, environment variable %s is not set",
+								   &safekeeper_token_env[1])));
+		}
+		neon_log(LOG, "using safekeeper auth token from environment variable");
+	}
+
 	if (page_server_connstring && page_server_connstring[0])
 	{
 		neon_log(PageStoreTrace, "set neon_smgr hook");

pgxn/neon/pagestore_client.h

@@ -49,6 +49,11 @@ typedef struct
 
 #define messageTag(m) (((const NeonMessage *)(m))->tag)
 
+#define NEON_TAG "[NEON_SMGR] "
+#define neon_log(tag, fmt, ...) ereport(tag,                                  \
+										(errmsg(NEON_TAG fmt, ##__VA_ARGS__), \
+										 errhidestmt(true), errhidecontext(true)))
+
 /*
  * supertype of all the Neon*Request structs below
  *
@@ -151,6 +156,7 @@ extern page_server_api * page_server;
 
 extern char *page_server_connstring;
 extern int flush_every_n_requests;
+extern int readahead_buffer_size;
 extern bool seqscan_prefetch_enabled;
 extern int seqscan_prefetch_distance;
 extern char *neon_timeline;
@@ -160,6 +166,7 @@ extern int32 max_cluster_size;
 
 extern const f_smgr *smgr_neon(BackendId backend, RelFileNode rnode);
 extern void smgr_init_neon(void);
+extern void readahead_buffer_resize(int newsize, void *extra);
 
 /* Neon storage manager functionality */
 

pgxn/neon/pagestore_smgr.c

@@ -116,10 +116,10 @@ static UnloggedBuildPhase unlogged_build_phase = UNLOGGED_BUILD_NOT_IN_PROGRESS;
  * 
  * Prefetch is performed locally by each backend.
  *
- * There can be up to READ_BUFFER_SIZE active IO requests registered at any
- * time. Requests using smgr_prefetch are sent to the pageserver, but we don't
- * wait on the response. Requests using smgr_read are either read from the
- * buffer, or (if that's not possible) we wait on the response to arrive -
+ * There can be up to readahead_buffer_size active IO requests registered at
+ * any time. Requests using smgr_prefetch are sent to the pageserver, but we
+ * don't wait on the response. Requests using smgr_read are either read from
+ * the buffer, or (if that's not possible) we wait on the response to arrive -
  * this also will allow us to receive other prefetched pages. 
  * Each request is immediately written to the output buffer of the pageserver
  * connection, but may not be flushed if smgr_prefetch is used: pageserver
@@ -136,15 +136,25 @@ static UnloggedBuildPhase unlogged_build_phase = UNLOGGED_BUILD_NOT_IN_PROGRESS;
  * the connection; the responses are stored for later use.
  *
  * NOTE: The current implementation of the prefetch system implements a ring
- * buffer of up to READ_BUFFER_SIZE requests. If there are more _read and
+ * buffer of up to readahead_buffer_size requests. If there are more _read and
  * _prefetch requests between the initial _prefetch and the _read of a buffer,
  * the prefetch request will have been dropped from this prefetch buffer, and
  * your prefetch was wasted.
  */
 
-/* Max amount of tracked buffer reads */
-#define READ_BUFFER_SIZE 128
-
+/*
+ * State machine:
+ *        
+ * not in hash : in hash
+ *             :
+ * UNUSED ------> REQUESTED --> RECEIVED
+ *   ^         :      |            |
+ *   |         :      v            |
+ *   |         : TAG_UNUSED        |
+ *   |         :      |            |
+ *   +----------------+------------+
+ *             :
+ */
 typedef enum PrefetchStatus {
 	PRFS_UNUSED = 0,	/* unused slot */
 	PRFS_REQUESTED,		/* request was written to the sendbuffer to PS, but not
@@ -219,11 +229,19 @@ typedef struct PrefetchState {
 
 	/* the buffers */
 	prfh_hash *prf_hash;
-	PrefetchRequest prf_buffer[READ_BUFFER_SIZE]; /* prefetch buffers */
+	PrefetchRequest prf_buffer[]; /* prefetch buffers */
 } PrefetchState;
 
 PrefetchState *MyPState;
 
+#define GetPrfSlot(ring_index) ( \
+	( \
+		AssertMacro((ring_index) < MyPState->ring_unused && \
+					(ring_index) >= MyPState->ring_last), \
+		&MyPState->prf_buffer[((ring_index) % readahead_buffer_size)] \
+	) \
+)
+
 int			n_prefetch_hits = 0;
 int			n_prefetch_misses = 0;
 int			n_prefetch_missed_caches = 0;
@@ -233,18 +251,116 @@ XLogRecPtr	prefetch_lsn = 0;
 
 static void consume_prefetch_responses(void);
 static uint64 prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_lsn);
-static void prefetch_read(PrefetchRequest *slot);
+static bool prefetch_read(PrefetchRequest *slot);
 static void prefetch_do_request(PrefetchRequest *slot, bool *force_latest, XLogRecPtr *force_lsn);
-static void prefetch_wait_for(uint64 ring_index);
+static bool prefetch_wait_for(uint64 ring_index);
 static void prefetch_cleanup(void);
-static inline void prefetch_set_unused(uint64 ring_index, bool hash_cleanup);
+static inline void prefetch_set_unused(uint64 ring_index);
 
 static XLogRecPtr neon_get_request_lsn(bool *latest, RelFileNode rnode,
 									   ForkNumber forknum, BlockNumber blkno);
 
+void
+readahead_buffer_resize(int newsize, void *extra)
+{
+	uint64		end,
+				nfree = newsize;
+	PrefetchState *newPState;
+	Size 		newprfs_size = offsetof(PrefetchState, prf_buffer) + (
+		sizeof(PrefetchRequest) * newsize
+	);
+	
+	/* don't try to re-initialize if we haven't initialized yet */
+	if (MyPState == NULL)
+		return;
+
+	/*
+	 * Make sure that we don't lose track of active prefetch requests by
+	 * ensuring we have received all but the last n requests (n = newsize).
+	 */
+	if (MyPState->n_requests_inflight > newsize)
+		prefetch_wait_for(MyPState->ring_unused - newsize);
+
+	/* construct the new PrefetchState, and copy over the memory contexts */
+	newPState = MemoryContextAllocZero(TopMemoryContext, newprfs_size);
+
+	newPState->bufctx = MyPState->bufctx;
+	newPState->errctx = MyPState->errctx;
+	newPState->hashctx = MyPState->hashctx;
+	newPState->prf_hash = prfh_create(MyPState->hashctx, newsize, NULL);
+	newPState->n_unused = newsize;
+	newPState->n_requests_inflight = 0;
+	newPState->n_responses_buffered = 0;
+	newPState->ring_last = newsize;
+	newPState->ring_unused = newsize;
+	newPState->ring_receive = newsize;
+	newPState->ring_flush = newsize;
+
+	/* 
+	 * Copy over the prefetches.
+	 * 
+	 * We populate the prefetch array from the end; to retain the most recent
+	 * prefetches, but this has the benefit of only needing to do one iteration
+	 * on the dataset, and trivial compaction.
+	 */
+	for (end = MyPState->ring_unused - 1;
+		 end >= MyPState->ring_last && end != UINT64_MAX && nfree != 0;
+		 end -= 1)
+	{
+		PrefetchRequest *slot = GetPrfSlot(end);
+		PrefetchRequest *newslot;
+		bool	found;
+
+		if (slot->status == PRFS_UNUSED)
+			continue;
+
+		nfree -= 1;
+
+		newslot = &newPState->prf_buffer[nfree];
+		*newslot = *slot;
+		newslot->my_ring_index = nfree;
+
+		prfh_insert(newPState->prf_hash, newslot, &found);
+
+		Assert(!found);
+		
+		switch (newslot->status)
+		{
+			case PRFS_UNUSED:
+				pg_unreachable();
+			case PRFS_REQUESTED:
+				newPState->n_requests_inflight += 1;
+				newPState->ring_receive -= 1;
+				newPState->ring_last -= 1;
+				break;
+			case PRFS_RECEIVED:
+				newPState->n_responses_buffered += 1;
+				newPState->ring_last -= 1;
+				break;
+			case PRFS_TAG_REMAINS:
+				newPState->ring_last -= 1;
+				break;
+		}
+		newPState->n_unused -= 1;
+	}
+
+	for (; end >= MyPState->ring_last && end != UINT64_MAX; end -= 1)
+	{
+		prefetch_set_unused(end);
+	}
+
+	prfh_destroy(MyPState->prf_hash);
+	pfree(MyPState);
+	MyPState = newPState;
+}
+
+
 
 /*
  * Make sure that there are no responses still in the buffer.
+ *
+ * NOTE: this function may indirectly update MyPState->pfs_hash; which
+ * invalidates any active pointers into the hash table.
  */
 static void
 consume_prefetch_responses(void)
@@ -256,14 +372,12 @@ consume_prefetch_responses(void)
 static void
 prefetch_cleanup(void)
 {
-	int		index;
 	uint64	ring_index;
 	PrefetchRequest *slot;
 
 	while (MyPState->ring_last < MyPState->ring_receive) {
 		ring_index = MyPState->ring_last;
-		index = (ring_index % READ_BUFFER_SIZE);
-		slot = &MyPState->prf_buffer[index];
+		slot = GetPrfSlot(ring_index);
 
 		if (slot->status == PRFS_UNUSED)
 			MyPState->ring_last += 1;
@@ -275,23 +389,33 @@ prefetch_cleanup(void)
 /*
  * Wait for slot of ring_index to have received its response.
  * The caller is responsible for making sure the request buffer is flushed.
+ * 
+ * NOTE: this function may indirectly update MyPState->pfs_hash; which
+ * invalidates any active pointers into the hash table.
  */
-static void
+static bool
 prefetch_wait_for(uint64 ring_index)
 {
-	int index;
 	PrefetchRequest *entry;
 
+	if (MyPState->ring_flush <= ring_index &&
+		MyPState->ring_unused > MyPState->ring_flush)
+	{
+		page_server->flush();
+		MyPState->ring_flush = MyPState->ring_unused;
+	}
+
 	Assert(MyPState->ring_unused > ring_index);
 
 	while (MyPState->ring_receive <= ring_index)
 	{
-		index = (MyPState->ring_receive % READ_BUFFER_SIZE);
-		entry = &MyPState->prf_buffer[index];
+		entry = GetPrfSlot(MyPState->ring_receive);
 
 		Assert(entry->status == PRFS_REQUESTED);
-		prefetch_read(entry);
+		if (!prefetch_read(entry))
+			return false;
 	}
+	return true;
 }
 
 /*
@@ -299,8 +423,11 @@ prefetch_wait_for(uint64 ring_index)
  * 
  * The caller is responsible for making sure that the request for this buffer
  * was flushed to the PageServer.
+ *
+ * NOTE: this function may indirectly update MyPState->pfs_hash; which
+ * invalidates any active pointers into the hash table.
  */
-static void
+static bool
 prefetch_read(PrefetchRequest *slot)
 {
 	NeonResponse *response;
@@ -313,15 +440,22 @@ prefetch_read(PrefetchRequest *slot)
 	old = MemoryContextSwitchTo(MyPState->errctx);
 	response = (NeonResponse *) page_server->receive();
 	MemoryContextSwitchTo(old);
+	if (response)
+	{
+		/* update prefetch state */
+		MyPState->n_responses_buffered += 1;
+		MyPState->n_requests_inflight -= 1;
+		MyPState->ring_receive += 1;
 
-	/* update prefetch state */
-	MyPState->n_responses_buffered += 1;
-	MyPState->n_requests_inflight -= 1;
-	MyPState->ring_receive += 1;
-
-	/* update slot state */
-	slot->status = PRFS_RECEIVED;
-	slot->response = response;
+		/* update slot state */
+		slot->status = PRFS_RECEIVED;
+		slot->response = response;
+		return true;
+	}
+	else
+	{
+		return false;
+	}
 }
 
 /*
@@ -333,19 +467,22 @@ prefetch_read(PrefetchRequest *slot)
 void
 prefetch_on_ps_disconnect(void)
 {
-	for (; MyPState->ring_receive < MyPState->ring_unused; MyPState->ring_receive++)
+	MyPState->ring_flush = MyPState->ring_unused;
+	while (MyPState->ring_receive < MyPState->ring_unused)
 	{
 		PrefetchRequest *slot;
-		int		index = MyPState->ring_receive % READ_BUFFER_SIZE;
+		uint64 ring_index = MyPState->ring_receive;
+
+		slot = GetPrfSlot(ring_index);
 
-		slot = &MyPState->prf_buffer[index];
 		Assert(slot->status == PRFS_REQUESTED);
-		Assert(slot->my_ring_index == MyPState->ring_receive);
+		Assert(slot->my_ring_index == ring_index);
 
 		/* clean up the request */
 		slot->status = PRFS_TAG_REMAINS;
-		MyPState->n_requests_inflight--;
-		prefetch_set_unused(MyPState->ring_receive, true);
+		MyPState->n_requests_inflight -= 1;
+		MyPState->ring_receive += 1;
+		prefetch_set_unused(ring_index);
 	}
 }
 
@@ -354,21 +491,24 @@ prefetch_on_ps_disconnect(void)
  *
  * The slot at ring_index must be a current member of the ring buffer,
  * and may not be in the PRFS_REQUESTED state.
+ *
+ * NOTE: this function will update MyPState->pfs_hash; which invalidates any
+ * active pointers into the hash table.
  */
 static inline void
-prefetch_set_unused(uint64 ring_index, bool hash_cleanup)
+prefetch_set_unused(uint64 ring_index)
 {
-	PrefetchRequest *slot = &MyPState->prf_buffer[ring_index % READ_BUFFER_SIZE];
+	PrefetchRequest *slot = GetPrfSlot(ring_index);
 
-	Assert(MyPState->ring_last <= ring_index &&
-		   MyPState->ring_unused > ring_index);
+	if (ring_index < MyPState->ring_last)
+		return; /* Should already be unused */
+
+	Assert(MyPState->ring_unused > ring_index);
 
 	if (slot->status == PRFS_UNUSED)
 		return;
 
 	Assert(slot->status == PRFS_RECEIVED || slot->status == PRFS_TAG_REMAINS);
-	Assert(ring_index >= MyPState->ring_last &&
-		   ring_index < MyPState->ring_unused);
 
 	if (slot->status == PRFS_RECEIVED)
 	{
@@ -383,8 +523,7 @@ prefetch_set_unused(uint64 ring_index, bool hash_cleanup)
 		Assert(slot->response == NULL);
 	}
 
-	if (hash_cleanup)
-		prfh_delete(MyPState->prf_hash, slot);
+	prfh_delete(MyPState->prf_hash, slot);
 
 	/* clear all fields */
 	MemSet(slot, 0, sizeof(PrefetchRequest));
@@ -398,6 +537,7 @@ prefetch_set_unused(uint64 ring_index, bool hash_cleanup)
 static void
 prefetch_do_request(PrefetchRequest *slot, bool *force_latest, XLogRecPtr *force_lsn)
 {
+	bool found;
 	NeonGetPageRequest request = {
 		.req.tag = T_NeonGetPageRequest,
 		.req.latest = false,
@@ -455,6 +595,9 @@ prefetch_do_request(PrefetchRequest *slot, bool *force_latest, XLogRecPtr *force
 
 	/* update slot state */
 	slot->status = PRFS_REQUESTED;
+
+	prfh_insert(MyPState->prf_hash, slot, &found);
+	Assert(!found);
 }
 
 /*
@@ -465,13 +608,14 @@ prefetch_do_request(PrefetchRequest *slot, bool *force_latest, XLogRecPtr *force
  * If force_latest and force_lsn are not NULL, those values are sent to the
  * pageserver. If they are NULL, we utilize the lastWrittenLsn -infrastructure
  * to fill in these values manually.
+ *
+ * NOTE: this function may indirectly update MyPState->pfs_hash; which
+ * invalidates any active pointers into the hash table.
  */
 
 static uint64
 prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_lsn)
 {
-	int		index;
-	bool	found;
 	uint64	ring_index;
 	PrefetchRequest req;
 	PrefetchRequest *slot;
@@ -486,28 +630,49 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls
 	{
 		slot = entry->slot;
 		ring_index = slot->my_ring_index;
-		index = (ring_index % READ_BUFFER_SIZE);
-		Assert(slot == &MyPState->prf_buffer[index]);
+		Assert(slot == GetPrfSlot(ring_index));
 
 		Assert(slot->status != PRFS_UNUSED);
+		Assert(MyPState->ring_last <= ring_index &&
+			   ring_index < MyPState->ring_unused);
 		Assert(BUFFERTAGS_EQUAL(slot->buftag, tag));
-		
+
 		/*
 		 * If we want a specific lsn, we do not accept requests that were made
 		 * with a potentially different LSN.
 		 */
-		if (force_lsn && slot->effective_request_lsn != *force_lsn)
+		if (force_latest && force_lsn)
 		{
-			prefetch_wait_for(ring_index);
-			prefetch_set_unused(ring_index, true);
+			/* if we want the latest version, any effective_request_lsn < request lsn is OK */
+			if (*force_latest)
+			{
+				if (*force_lsn > slot->effective_request_lsn)
+				{
+					prefetch_wait_for(ring_index);
+					prefetch_set_unused(ring_index);
+					entry = NULL;
+				}
+			}
+			/* if we don't want the latest version, only accept requests with the exact same LSN */
+			else
+			{
+				if (*force_lsn != slot->effective_request_lsn)
+				{
+					prefetch_wait_for(ring_index);
+					prefetch_set_unused(ring_index);
+					entry = NULL;
+				}
+			}
 		}
+
 		/*
 		 * We received a prefetch for a page that was recently read and
 		 * removed from the buffers. Remove that request from the buffers.
 		 */
 		else if (slot->status == PRFS_TAG_REMAINS)
 		{
-			prefetch_set_unused(ring_index, true);
+			prefetch_set_unused(ring_index);
+			entry = NULL;
 		}
 		else
 		{
@@ -530,9 +695,10 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls
 	 * output buffer, and 'not sending' a prefetch request kind of goes
 	 * against the principles of prefetching)
 	 */
-	if (MyPState->ring_last + READ_BUFFER_SIZE - 1 == MyPState->ring_unused)
+	if (MyPState->ring_last + readahead_buffer_size - 1 == MyPState->ring_unused)
 	{
-		slot = &MyPState->prf_buffer[(MyPState->ring_last % READ_BUFFER_SIZE)];
+		uint64 cleanup_index = MyPState->ring_last;
+		slot = GetPrfSlot(cleanup_index);
 
 		Assert(slot->status != PRFS_UNUSED);
 
@@ -540,13 +706,13 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls
 		switch (slot->status)
 		{
 			case PRFS_REQUESTED:
-				Assert(MyPState->ring_receive == MyPState->ring_last);
-				prefetch_wait_for(MyPState->ring_last);
-				prefetch_set_unused(MyPState->ring_last, true);
+				Assert(MyPState->ring_receive == cleanup_index);
+				prefetch_wait_for(cleanup_index);
+				prefetch_set_unused(cleanup_index);
 				break;
 			case PRFS_RECEIVED:
 			case PRFS_TAG_REMAINS:
-				prefetch_set_unused(MyPState->ring_last, true);
+				prefetch_set_unused(cleanup_index);
 				break;
 			default:
 				pg_unreachable();
@@ -554,12 +720,11 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls
 	}
 
 	/*
-	 * The next buffer pointed to by `ring_unused` is now unused, so we can insert
-	 * the new request to it.
+	 * The next buffer pointed to by `ring_unused` is now definitely empty,
+	 * so we can insert the new request to it.
 	 */
 	ring_index = MyPState->ring_unused;
-	index = (ring_index % READ_BUFFER_SIZE);
-	slot = &MyPState->prf_buffer[index];
+	slot = &MyPState->prf_buffer[((ring_index) % readahead_buffer_size)];
 
 	Assert(MyPState->ring_last <= ring_index);
 
@@ -572,14 +737,13 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls
 	slot->buftag = tag;
 	slot->my_ring_index = ring_index;
 
-	prfh_insert(MyPState->prf_hash, slot, &found);
-	Assert(!found);
-
 	prefetch_do_request(slot, force_latest, force_lsn);
 	Assert(slot->status == PRFS_REQUESTED);
-	Assert(ring_index < MyPState->ring_unused);
+	Assert(MyPState->ring_last <= ring_index &&
+		   ring_index < MyPState->ring_unused);
 
-	if (flush_every_n_requests > 0 && MyPState->ring_unused - MyPState->ring_flush >= flush_every_n_requests)
+	if (flush_every_n_requests > 0 &&
+		MyPState->ring_unused - MyPState->ring_flush >= flush_every_n_requests)
 	{
 		page_server->flush();
 		MyPState->ring_flush = MyPState->ring_unused;
@@ -591,11 +755,16 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls
 static NeonResponse *
 page_server_request(void const *req)
 {
-	page_server->send((NeonRequest *) req);
-	page_server->flush();
-	MyPState->ring_flush = MyPState->ring_unused;
-	consume_prefetch_responses();
-	return page_server->receive();
+	NeonResponse* resp;
+	do {
+		page_server->send((NeonRequest *) req);
+		page_server->flush();
+		MyPState->ring_flush = MyPState->ring_unused;
+		consume_prefetch_responses();
+		resp = page_server->receive();
+	} while (resp == NULL);
+	return resp;
+
 }
 
 
@@ -1061,14 +1230,18 @@ neon_wallog_page(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, ch
 void
 neon_init(void)
 {
-	HASHCTL info;
+	Size prfs_size;
 
 	if (MyPState != NULL)
 		return;
 
-	MyPState = MemoryContextAllocZero(TopMemoryContext, sizeof(PrefetchState));
+	prfs_size = offsetof(PrefetchState, prf_buffer) + (
+		sizeof(PrefetchRequest) * readahead_buffer_size
+	);
+
+	MyPState = MemoryContextAllocZero(TopMemoryContext, prfs_size);
 	
-	MyPState->n_unused = READ_BUFFER_SIZE;
+	MyPState->n_unused = readahead_buffer_size;
 
 	MyPState->bufctx = SlabContextCreate(TopMemoryContext,
 										 "NeonSMGR/prefetch",
@@ -1081,11 +1254,8 @@ neon_init(void)
 											  "NeonSMGR/prefetch",
 											  ALLOCSET_DEFAULT_SIZES);
 
-	info.keysize = sizeof(BufferTag);
-	info.entrysize = sizeof(uint64);
-
 	MyPState->prf_hash = prfh_create(MyPState->hashctx,
-									 READ_BUFFER_SIZE, NULL);
+									 readahead_buffer_size, NULL);
 
 #ifdef DEBUG_COMPARE_LOCAL
 	mdinit();
@@ -1479,7 +1649,8 @@ neon_close(SMgrRelation reln, ForkNumber forknum)
 bool
 neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum)
 {
-	uint64 ring_index;
+	BufferTag	tag;
+	uint64		ring_index PG_USED_FOR_ASSERTS_ONLY;
 
 	switch (reln->smgr_relpersistence)
 	{
@@ -1495,7 +1666,7 @@ neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum)
 			elog(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence);
 	}
 
-	BufferTag tag = (BufferTag) {
+	tag = (BufferTag) {
 		.rnode = reln->smgr_rnode.node,
 		.forkNum = forknum,
 		.blockNum = blocknum
@@ -1574,9 +1745,9 @@ neon_read_at_lsn(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 
 	if (entry != NULL)
 	{
-		if (entry->slot->effective_request_lsn >= prefetch_lsn)
+		slot = entry->slot;
+		if (slot->effective_request_lsn >= request_lsn)
 		{
-			slot = entry->slot;
 			ring_index = slot->my_ring_index;
 			n_prefetch_hits += 1;
 		}
@@ -1587,41 +1758,36 @@ neon_read_at_lsn(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 			 * unlikely this happens, but it can happen if prefetch distance is
 			 * large enough and a backend didn't consume all prefetch requests.
 			 */
-			if (entry->slot->status == PRFS_REQUESTED)
+			if (slot->status == PRFS_REQUESTED)
 			{
-				page_server->flush();
-				MyPState->ring_flush = MyPState->ring_unused;
-				prefetch_wait_for(entry->slot->my_ring_index);
+				prefetch_wait_for(slot->my_ring_index);
 			}
 			/* drop caches */
-			prefetch_set_unused(entry->slot->my_ring_index, true);
+			prefetch_set_unused(slot->my_ring_index);
 			n_prefetch_missed_caches += 1;
 			/* make it look like a prefetch cache miss */
 			entry = NULL;
 		}
 	}
 
-	if (entry == NULL)
+	do
 	{
-		n_prefetch_misses += 1;
+		if (entry == NULL)
+		{
+			n_prefetch_misses += 1;
 
-		ring_index = prefetch_register_buffer(buftag, &request_latest,
-											  &request_lsn);
-		slot = &MyPState->prf_buffer[(ring_index % READ_BUFFER_SIZE)];
-	}
+			ring_index = prefetch_register_buffer(buftag, &request_latest,
+												  &request_lsn);
+			slot = GetPrfSlot(ring_index);
+		}
 
-	Assert(MyPState->ring_last <= ring_index &&
-		   MyPState->ring_unused > ring_index);
-	Assert(slot->my_ring_index == ring_index);
-	Assert(slot->status != PRFS_UNUSED);
-	Assert(&MyPState->prf_buffer[(ring_index % READ_BUFFER_SIZE)] == slot);
+		Assert(slot->my_ring_index == ring_index);
+		Assert(MyPState->ring_last <= ring_index &&
+			   MyPState->ring_unused > ring_index);
+		Assert(slot->status != PRFS_UNUSED);
+		Assert(GetPrfSlot(ring_index) == slot);
 
-	if (ring_index >= MyPState->ring_flush)
-	{
-		page_server->flush();
-		MyPState->ring_flush = MyPState->ring_unused;
-	}
-	prefetch_wait_for(ring_index);
+	} while (!prefetch_wait_for(ring_index));
 
 	Assert(slot->status == PRFS_RECEIVED);
 
@@ -1651,7 +1817,7 @@ neon_read_at_lsn(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	}
 
 	/* buffer was used, clean up for later reuse */
-	prefetch_set_unused(ring_index, true);
+	prefetch_set_unused(ring_index);
 	prefetch_cleanup();
 }