Add missing semicolon to finalize_drop_subscriptions.sql

Signed-off-by: Tristan Partin <tristan@neon.tech>

.github/PULL_REQUEST_TEMPLATE/release-pr.md

@@ -0,0 +1,21 @@
+## Release 202Y-MM-DD
+
+**NB: this PR must be merged only by 'Create a merge commit'!**
+
+### Checklist when preparing for release
+- [ ] Read or refresh [the release flow guide](https://www.notion.so/neondatabase/Release-general-flow-61f2e39fd45d4d14a70c7749604bd70b)
+- [ ] Ask in the [cloud Slack channel](https://neondb.slack.com/archives/C033A2WE6BZ) that you are going to rollout the release. Any blockers?
+- [ ] Does this release contain any db migrations? Destructive ones? What is the rollback plan?
+
+<!-- List everything that should be done **before** release, any issues / setting changes / etc -->
+
+### Checklist after release
+- [ ] Make sure instructions from PRs included in this release and labeled `manual_release_instructions` are executed (either by you or by people who wrote them).
+- [ ] Based on the merged commits write release notes and open a PR into `website` repo ([example](https://github.com/neondatabase/website/pull/219/files))
+- [ ] Check [#dev-production-stream](https://neondb.slack.com/archives/C03F5SM1N02) Slack channel
+- [ ] Check [stuck projects page](https://console.neon.tech/admin/projects?sort=last_active&order=desc&stuck=true)
+- [ ] Check [recent operation failures](https://console.neon.tech/admin/operations?action=create_timeline%2Cstart_compute%2Cstop_compute%2Csuspend_compute%2Capply_config%2Cdelete_timeline%2Cdelete_tenant%2Ccreate_branch%2Ccheck_availability&sort=updated_at&order=desc&had_retries=some)
+- [ ] Check [cloud SLO dashboard](https://neonprod.grafana.net/d/_oWcBMJ7k/cloud-slos?orgId=1)
+- [ ] Check [compute startup metrics dashboard](https://neonprod.grafana.net/d/5OkYJEmVz/compute-startup-time)
+
+<!-- List everything that should be done **after** release, any admin UI configuration / Grafana dashboard / alert changes / setting changes / etc -->

.github/actionlint.yml

@@ -33,5 +33,3 @@ config-variables:
   - NEON_PROD_AWS_ACCOUNT_ID
   - AWS_ECR_REGION
   - BENCHMARK_LARGE_OLTP_PROJECTID
-  - SLACK_ON_CALL_DEVPROD_STREAM
-  - SLACK_RUST_CHANNEL_ID

.github/scripts/generate_image_maps.py

@@ -1,16 +1,14 @@
 import itertools
 import json
 import os
-import sys
 
-source_tag = os.getenv("SOURCE_TAG")
-target_tag = os.getenv("TARGET_TAG")
-branch = os.getenv("BRANCH")
-dev_acr = os.getenv("DEV_ACR")
-prod_acr = os.getenv("PROD_ACR")
-dev_aws = os.getenv("DEV_AWS")
-prod_aws = os.getenv("PROD_AWS")
-aws_region = os.getenv("AWS_REGION")
+build_tag = os.environ["BUILD_TAG"]
+branch = os.environ["BRANCH"]
+dev_acr = os.environ["DEV_ACR"]
+prod_acr = os.environ["PROD_ACR"]
+dev_aws = os.environ["DEV_AWS"]
+prod_aws = os.environ["PROD_AWS"]
+aws_region = os.environ["AWS_REGION"]
 
 components = {
     "neon": ["neon"],
@@ -41,23 +39,24 @@ registries = {
 
 outputs: dict[str, dict[str, list[str]]] = {}
 
-target_tags = [target_tag, "latest"] if branch == "main" else [target_tag]
-target_stages = (
-    ["dev", "prod"] if branch in ["release", "release-proxy", "release-compute"] else ["dev"]
-)
+target_tags = [build_tag, "latest"] if branch == "main" else [build_tag]
+target_stages = ["dev", "prod"] if branch.startswith("release") else ["dev"]
 
 for component_name, component_images in components.items():
     for stage in target_stages:
-        outputs[f"{component_name}-{stage}"] = {
-            f"docker.io/neondatabase/{component_image}:{source_tag}": [
-                f"{registry}/{component_image}:{tag}"
-                for registry, tag in itertools.product(registries[stage], target_tags)
-                if not (registry == "docker.io/neondatabase" and tag == source_tag)
+        outputs[f"{component_name}-{stage}"] = dict(
+            [
+                (
+                    f"docker.io/neondatabase/{component_image}:{build_tag}",
+                    [
+                        f"{combo[0]}/{component_image}:{combo[1]}"
+                        for combo in itertools.product(registries[stage], target_tags)
+                    ],
+                )
+                for component_image in component_images
             ]
-            for component_image in component_images
-        }
+        )
 
-with open(os.getenv("GITHUB_OUTPUT", "/dev/null"), "a") as f:
+with open(os.environ["GITHUB_OUTPUT"], "a") as f:
     for key, value in outputs.items():
         f.write(f"{key}={json.dumps(value)}\n")
-        print(f"Image map for {key}:\n{json.dumps(value, indent=2)}\n\n", file=sys.stderr)

.github/scripts/lint-release-pr.sh

@@ -1,110 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-DOCS_URL="https://docs.neon.build/overview/repositories/neon.html"
-
-message() {
-  if [[ -n "${GITHUB_PR_NUMBER:-}" ]]; then
-    gh pr comment --repo "${GITHUB_REPOSITORY}" "${GITHUB_PR_NUMBER}" --edit-last --body "$1" \
-      || gh pr comment --repo "${GITHUB_REPOSITORY}" "${GITHUB_PR_NUMBER}" --body "$1"
-  fi
-  echo "$1"
-}
-
-report_error() {
-  message "❌ $1
-  For more details, see the documentation: ${DOCS_URL}"
-
-  exit 1
-}
-
-case "$RELEASE_BRANCH" in
-  "release") COMPONENT="Storage" ;;
-  "release-proxy") COMPONENT="Proxy" ;;
-  "release-compute") COMPONENT="Compute" ;;
-  *)
-    report_error "Unknown release branch: ${RELEASE_BRANCH}"
-    ;;
-esac
-
-
-# Identify main and release branches
-MAIN_BRANCH="origin/main"
-REMOTE_RELEASE_BRANCH="origin/${RELEASE_BRANCH}"
-
-# Find merge base
-MERGE_BASE=$(git merge-base "${MAIN_BRANCH}" "${REMOTE_RELEASE_BRANCH}")
-echo "Merge base of ${MAIN_BRANCH} and ${RELEASE_BRANCH}: ${MERGE_BASE}"
-
-# Get the HEAD commit (last commit in PR, expected to be the merge commit)
-LAST_COMMIT=$(git rev-parse HEAD)
-
-MERGE_COMMIT_MESSAGE=$(git log -1 --format=%s "${LAST_COMMIT}")
-EXPECTED_MESSAGE_REGEX="^$COMPONENT release [0-9]{4}-[0-9]{2}-[0-9]{2}$"
-
-if ! [[ "${MERGE_COMMIT_MESSAGE}" =~ ${EXPECTED_MESSAGE_REGEX} ]]; then
-  report_error "Merge commit message does not match expected pattern: '<component> release YYYY-MM-DD'
-  Expected component: ${COMPONENT}
-  Found: '${MERGE_COMMIT_MESSAGE}'"
-fi
-echo "✅ Merge commit message is correctly formatted: '${MERGE_COMMIT_MESSAGE}'"
-
-LAST_COMMIT_PARENTS=$(git cat-file -p "${LAST_COMMIT}" | jq -sR '[capture("parent (?<parent>[0-9a-f]{40})"; "g") | .parent]')
-
-if [[ "$(echo "${LAST_COMMIT_PARENTS}" | jq 'length')" -ne 2 ]]; then
-  report_error "Last commit must be a merge commit with exactly two parents"
-fi
-
-EXPECTED_RELEASE_HEAD=$(git rev-parse "${REMOTE_RELEASE_BRANCH}")
-if echo "${LAST_COMMIT_PARENTS}" | jq -e --arg rel "${EXPECTED_RELEASE_HEAD}" 'index($rel) != null' > /dev/null; then
-  LINEAR_HEAD=$(echo "${LAST_COMMIT_PARENTS}" | jq -r '[.[] | select(. != $rel)][0]' --arg rel "${EXPECTED_RELEASE_HEAD}")
-else
-  report_error "Last commit must merge the release branch (${RELEASE_BRANCH})"
-fi
-echo "✅ Last commit correctly merges the previous commit and the release branch"
-echo "Top commit of linear history: ${LINEAR_HEAD}"
-
-MERGE_COMMIT_TREE=$(git rev-parse "${LAST_COMMIT}^{tree}")
-LINEAR_HEAD_TREE=$(git rev-parse "${LINEAR_HEAD}^{tree}")
-
-if [[ "${MERGE_COMMIT_TREE}" != "${LINEAR_HEAD_TREE}" ]]; then
-  report_error "Tree of merge commit (${MERGE_COMMIT_TREE}) does not match tree of linear history head (${LINEAR_HEAD_TREE})
-  This indicates that the merge of ${RELEASE_BRANCH} into this branch was not performed using the merge strategy 'ours'"
-fi
-echo "✅ Merge commit tree matches the linear history head"
-
-EXPECTED_PREVIOUS_COMMIT="${LINEAR_HEAD}"
-
-# Now traverse down the history, ensuring each commit has exactly one parent
-CURRENT_COMMIT="${EXPECTED_PREVIOUS_COMMIT}"
-while [[ "${CURRENT_COMMIT}" != "${MERGE_BASE}" && "${CURRENT_COMMIT}" != "${EXPECTED_RELEASE_HEAD}" ]]; do
-  CURRENT_COMMIT_PARENTS=$(git cat-file -p "${CURRENT_COMMIT}" | jq -sR '[capture("parent (?<parent>[0-9a-f]{40})"; "g") | .parent]')
-
-  if [[ "$(echo "${CURRENT_COMMIT_PARENTS}" | jq 'length')" -ne 1 ]]; then
-    report_error "Commit ${CURRENT_COMMIT} must have exactly one parent"
-  fi
-
-  NEXT_COMMIT=$(echo "${CURRENT_COMMIT_PARENTS}" | jq -r '.[0]')
-
-  if [[ "${NEXT_COMMIT}" == "${MERGE_BASE}" ]]; then
-    echo "✅ Reached merge base (${MERGE_BASE})"
-    PR_BASE="${MERGE_BASE}"
-  if [[ "${NEXT_COMMIT}" == "${EXPECTED_RELEASE_HEAD}" ]]; then
-    echo "✅ Reached release branch (${EXPECTED_RELEASE_HEAD})"
-    PR_BASE="${EXPECTED_RELEASE_HEAD}"
-  elif [[ -z "${NEXT_COMMIT}" ]]; then
-    report_error "Unexpected end of commit history before reaching merge base"
-  fi
-
-  # Move to the next commit in the chain
-  CURRENT_COMMIT="${NEXT_COMMIT}"
-done
-
-echo "✅ All commits are properly ordered and linear"
-echo "✅ Release PR structure is valid"
-
-echo
-
-message "Commits that are part of this release:
-$(git log --oneline "${PR_BASE}..${LINEAR_HEAD}")"

.github/scripts/previous-releases.jq

@@ -17,12 +17,6 @@
 ({};
  .[$entry.component] |= (if . == null or $entry.version > .version then $entry else . end))
 
-# Ensure that each component exists, or fail
-| (["storage", "compute", "proxy"] - (keys)) as $missing
-| if ($missing | length) > 0 then
-    "Error: Found no release for \($missing | join(", "))!\n" | halt_error(1)
-  else . end
-
 # Convert the resulting object into an array of formatted strings
 | to_entries
 | map("\(.key)=\(.value.full)")

.github/workflows/_create-release-pr.yml

@@ -7,8 +7,8 @@ on:
         description: 'Component name'
         required: true
         type: string
-      source-branch:
-        description: 'Source branch'
+      release-branch:
+        description: 'Release branch'
         required: true
         type: string
     secrets:
@@ -30,25 +30,17 @@ jobs:
     steps:
     - uses: actions/checkout@v4
       with:
-        ref: ${{ inputs.source-branch }}
-        fetch-depth: 0
+        ref: main
 
     - name: Set variables
       id: vars
       env:
         COMPONENT_NAME: ${{ inputs.component-name }}
-        RELEASE_BRANCH: >-
-          ${{
-            false
-            || inputs.component-name == 'Storage' && 'release'
-            || inputs.component-name == 'Proxy' && 'release-proxy'
-            || inputs.component-name == 'Compute' && 'release-compute'
-          }}
+        RELEASE_BRANCH: ${{ inputs.release-branch }}
       run: |
         today=$(date +'%Y-%m-%d')
         echo "title=${COMPONENT_NAME} release ${today}" | tee -a ${GITHUB_OUTPUT}
         echo "rc-branch=rc/${RELEASE_BRANCH}/${today}"  | tee -a ${GITHUB_OUTPUT}
-        echo "release-branch=${RELEASE_BRANCH}"         | tee -a ${GITHUB_OUTPUT}
 
     - name: Configure git
       run: |
@@ -57,35 +49,31 @@ jobs:
 
     - name: Create RC branch
       env:
-        RELEASE_BRANCH: ${{ steps.vars.outputs.release-branch }}
         RC_BRANCH: ${{ steps.vars.outputs.rc-branch }}
         TITLE: ${{ steps.vars.outputs.title }}
       run: |
-        git switch -c "${RC_BRANCH}"
+        git checkout -b "${RC_BRANCH}"
 
-        # Manually create a merge commit on the current branch, keeping the
-        # tree and setting the parents to the current HEAD and the HEAD of the
-        # release branch. This commit is what we'll fast-forward the release
-        # branch to when merging the release branch.
-        # For details on why, look at
-        # https://docs.neon.build/overview/repositories/neon.html#background-on-commit-history-of-release-prs
-        current_tree=$(git rev-parse 'HEAD^{tree}')
-        release_head=$(git rev-parse "origin/${RELEASE_BRANCH}")
-        current_head=$(git rev-parse HEAD)
-        merge_commit=$(git commit-tree -p "${current_head}" -p "${release_head}" -m "${TITLE}" "${current_tree}")
-
-        # Fast-forward the current branch to the newly created merge_commit
-        git merge --ff-only ${merge_commit}
+        # create an empty commit to distinguish workflow runs
+        # from other possible releases from the same commit
+        git commit --allow-empty -m "${TITLE}"
 
         git push origin "${RC_BRANCH}"
 
-    - name: Create a PR into ${{ steps.vars.outputs.release-branch }}
+    - name: Create a PR into ${{ inputs.release-branch }}
       env:
         GH_TOKEN: ${{ secrets.ci-access-token }}
         RC_BRANCH: ${{ steps.vars.outputs.rc-branch }}
-        RELEASE_BRANCH: ${{ steps.vars.outputs.release-branch }}
+        RELEASE_BRANCH: ${{ inputs.release-branch }}
         TITLE: ${{ steps.vars.outputs.title }}
       run: |
+        cat << EOF > body.md
+          ## ${TITLE}
+
+          **Please merge this Pull Request using 'Create a merge commit' button**
+        EOF
+
         gh pr create --title "${TITLE}" \
+                     --body-file "body.md" \
                      --head "${RC_BRANCH}" \
                      --base "${RELEASE_BRANCH}"

.github/workflows/_meta.yml

@@ -19,18 +19,11 @@ on:
         description: "Tag of the last compute release"
         value: ${{ jobs.tags.outputs.compute }}
       run-kind:
-        description: "The kind of run we're currently in. Will be one of `push-main`, `storage-release`, `compute-release`, `proxy-release`, `storage-rc-pr`, `compute-rc-pr`,  `proxy-rc-pr`, `pr`, or `workflow-dispatch`"
+        description: "The kind of run we're currently in. Will be one of `pr`, `push-main`, `storage-rc`, `storage-release`, `proxy-rc`, `proxy-release`, `compute-rc`, `compute-release` or `merge_queue`"
         value: ${{ jobs.tags.outputs.run-kind }}
-      release-pr-run-id:
-        description: "Only available if `run-kind in [storage-release, proxy-release, compute-release]`. Contains the run ID of the `Build and Test` workflow, assuming one with the current commit can be found."
-        value: ${{ jobs.tags.outputs.release-pr-run-id }}
 
 permissions: {}
 
-defaults:
-  run:
-    shell: bash -euo pipefail {0}
-
 jobs:
   tags:
     runs-on: ubuntu-22.04
@@ -40,7 +33,6 @@ jobs:
       proxy: ${{ steps.previous-releases.outputs.proxy }}
       storage: ${{ steps.previous-releases.outputs.storage }}
       run-kind: ${{ steps.run-kind.outputs.run-kind }}
-      release-pr-run-id: ${{ steps.release-pr-run-id.outputs.release-pr-run-id }}
     permissions:
       contents: read
     steps:
@@ -63,7 +55,6 @@ jobs:
               || (inputs.github-event-name == 'pull_request' && github.base_ref == 'release-compute') && 'compute-rc-pr'
               || (inputs.github-event-name == 'pull_request' && github.base_ref == 'release-proxy')   && 'proxy-rc-pr'
               || (inputs.github-event-name == 'pull_request')                                         && 'pr'
-              || (inputs.github-event-name == 'workflow_dispatch')                                    && 'workflow-dispatch'
               || 'unknown'
             }}
         run: |
@@ -91,16 +82,9 @@ jobs:
             echo "tag=release-compute-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
             ;;
           pr|storage-rc-pr|compute-rc-pr|proxy-rc-pr)
-            BUILD_AND_TEST_RUN_ID=$(gh api --paginate \
-              -H "Accept: application/vnd.github+json" \
-              -H "X-GitHub-Api-Version: 2022-11-28" \
-              "/repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${CURRENT_SHA}&branch=${CURRENT_BRANCH}" \
-              | jq '[.workflow_runs[] | select(.name == "Build and Test")][0].id // ("Error: No matching workflow run found." | halt_error(1))')
+            BUILD_AND_TEST_RUN_ID=$(gh run list -b $CURRENT_BRANCH -c $CURRENT_SHA -w 'Build and Test' -L 1 --json databaseId --jq '.[].databaseId')
             echo "tag=$BUILD_AND_TEST_RUN_ID" | tee -a $GITHUB_OUTPUT
             ;;
-          workflow-dispatch)
-            echo "tag=$GITHUB_RUN_ID" | tee -a $GITHUB_OUTPUT
-            ;;
           *)
             echo "Unexpected RUN_KIND ('${RUN_KIND}'), failing to assign build-tag!"
             exit 1
@@ -117,13 +101,3 @@ jobs:
             "/repos/${GITHUB_REPOSITORY}/releases" \
           | jq -f .github/scripts/previous-releases.jq -r \
           | tee -a "${GITHUB_OUTPUT}"
-
-      - name: Get the release PR run ID
-        id: release-pr-run-id
-        if: ${{ contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), steps.run-kind.outputs.run-kind) }}
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-        run: |
-          RELEASE_PR_RUN_ID=$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=$CURRENT_SHA" | jq '[.workflow_runs[] | select(.name == "Build and Test") | select(.head_branch | test("^rc/release(-(proxy)|(compute))?/[0-9]{4}-[0-9]{2}-[0-9]{2}$"; "s"))] | first | .id // "Faied to find Build and Test run from  RC PR!" | halt_error(1)')
-          echo "release-pr-run-id=$RELEASE_PR_RUN_ID" | tee -a $GITHUB_OUTPUT

.github/workflows/build_and_test.yml

@@ -476,7 +476,7 @@ jobs:
         (
           !github.event.pull_request.draft
           || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft')
-          || needs.meta.outputs.run-kind == 'push-main'
+          || contains(fromJSON('["push-main", "storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind)
         ) && !failure() && !cancelled()
       }}
     needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, meta ]
@@ -487,7 +487,7 @@ jobs:
 
   neon-image-arch:
     needs: [ check-permissions, build-build-tools-image, meta ]
-    if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
     strategy:
       matrix:
         arch: [ x64, arm64 ]
@@ -537,7 +537,7 @@ jobs:
 
   neon-image:
     needs: [ neon-image-arch, meta ]
-    if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
     runs-on: ubuntu-22.04
     permissions:
       id-token: write # aws-actions/configure-aws-credentials
@@ -559,7 +559,7 @@ jobs:
 
   compute-node-image-arch:
     needs: [ check-permissions, build-build-tools-image, meta ]
-    if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
     permissions:
       id-token: write # aws-actions/configure-aws-credentials
       statuses: write
@@ -651,7 +651,7 @@ jobs:
 
   compute-node-image:
     needs: [ compute-node-image-arch, meta ]
-    if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
     permissions:
       id-token: write # aws-actions/configure-aws-credentials
       statuses: write
@@ -694,7 +694,7 @@ jobs:
 
   vm-compute-node-image-arch:
     needs: [ check-permissions, meta, compute-node-image ]
-    if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
     runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
     strategy:
       fail-fast: false
@@ -747,7 +747,7 @@ jobs:
 
   vm-compute-node-image:
     needs: [ vm-compute-node-image-arch, meta ]
-    if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
     runs-on: ubuntu-22.04
     strategy:
       matrix:
@@ -773,12 +773,7 @@ jobs:
   test-images:
     needs: [ check-permissions, meta, neon-image, compute-node-image ]
     # Depends on jobs that can get skipped
-    if: >-
-      ${{
-        !failure()
-        && !cancelled()
-        && contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind)
-      }}
+    if: "!failure() && !cancelled()"
     strategy:
       fail-fast: false
       matrix:
@@ -805,7 +800,7 @@ jobs:
       # Ensure that we don't have bad versions.
       - name: Verify image versions
         shell: bash # ensure no set -e for better error messages
-        if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
+        if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
         run: |
           pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
 
@@ -826,19 +821,19 @@ jobs:
         env:
           TAG: >-
             ${{
-              needs.meta.outputs.run-kind == 'compute-rc-pr'
+              contains(fromJSON('["compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)
               && needs.meta.outputs.previous-storage-release
               || needs.meta.outputs.build-tag
             }}
           COMPUTE_TAG: >-
             ${{
-              contains(fromJSON('["storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)
+              contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)
               && needs.meta.outputs.previous-compute-release
               || needs.meta.outputs.build-tag
             }}
           TEST_EXTENSIONS_TAG: >-
             ${{
-              contains(fromJSON('["storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)
+              contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)
               && 'latest'
               || needs.meta.outputs.build-tag
             }}
@@ -890,13 +885,7 @@ jobs:
         id: generate
         run: python3 .github/scripts/generate_image_maps.py
         env:
-          SOURCE_TAG: >-
-            ${{
-              contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), needs.meta.outputs.run-kind)
-              && needs.meta.outputs.release-pr-run-id
-              || needs.meta.outputs.build-tag
-            }}
-          TARGET_TAG: ${{ needs.meta.outputs.build-tag }}
+          BUILD_TAG: "${{ needs.meta.outputs.build-tag }}"
           BRANCH: "${{ github.ref_name }}"
           DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}"
           PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}"
@@ -906,7 +895,7 @@ jobs:
 
   push-neon-image-dev:
     needs: [ meta, generate-image-maps, neon-image ]
-    if: ${{ !failure() && !cancelled() && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
     uses: ./.github/workflows/_push-to-container-registry.yml
     permissions:
       id-token: write  # Required for aws/azure login
@@ -924,7 +913,7 @@ jobs:
 
   push-compute-image-dev:
     needs: [ meta, generate-image-maps, vm-compute-node-image ]
-    if: ${{ !failure() && !cancelled() && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
     uses: ./.github/workflows/_push-to-container-registry.yml
     permissions:
       id-token: write  # Required for aws/azure login
@@ -1186,7 +1175,7 @@ jobs:
               -f deployPgSniRouter=false \
               -f deployProxy=false \
               -f deployStorage=true \
-              -f deployStorageBroker=false \
+              -f deployStorageBroker=true \
               -f deployStorageController=true \
               -f branch=main \
               -f dockerTag=${{needs.meta.outputs.build-tag}} \
@@ -1194,7 +1183,7 @@ jobs:
 
             gh workflow --repo neondatabase/infra run deploy-prod.yml --ref main \
               -f deployStorage=true \
-              -f deployStorageBroker=false \
+              -f deployStorageBroker=true \
               -f deployStorageController=true \
               -f branch=main \
               -f dockerTag=${{needs.meta.outputs.build-tag}}
@@ -1242,11 +1231,11 @@ jobs:
           payload: |
             channel: ${{ vars.SLACK_STORAGE_CHANNEL_ID }}
             text: |
-              🔴 <!subteam^S06CJ87UMNY|@oncall-storage>: deploy job on release branch had unexpected status "${{ needs.deploy.result }}" <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>.
+              🔴 @oncall-storage: deploy job on release branch had unexpected status "${{ needs.deploy.result }}" <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>.
 
   # The job runs on `release` branch and copies compatibility data and Neon artifact from the last *release PR* to the latest directory
   promote-compatibility-data:
-    needs: [ meta, deploy ]
+    needs: [ deploy ]
     permissions:
       id-token: write # aws-actions/configure-aws-credentials
       statuses: write
@@ -1256,6 +1245,37 @@ jobs:
 
     runs-on: ubuntu-22.04
     steps:
+      - name: Fetch GITHUB_RUN_ID and COMMIT_SHA for the last merged release PR
+        id: fetch-last-release-pr-info
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          branch_name_and_pr_number=$(gh pr list \
+            --repo "${GITHUB_REPOSITORY}" \
+            --base release \
+            --state merged \
+            --limit 10 \
+            --json mergeCommit,headRefName,number \
+            --jq ".[] | select(.mergeCommit.oid==\"${GITHUB_SHA}\") | { branch_name: .headRefName, pr_number: .number }")
+          branch_name=$(echo "${branch_name_and_pr_number}" | jq -r '.branch_name')
+          pr_number=$(echo "${branch_name_and_pr_number}" | jq -r '.pr_number')
+
+          run_id=$(gh run list \
+            --repo "${GITHUB_REPOSITORY}" \
+            --workflow build_and_test.yml \
+            --branch "${branch_name}" \
+            --json databaseId \
+            --limit 1 \
+            --jq '.[].databaseId')
+
+          last_commit_sha=$(gh pr view "${pr_number}" \
+            --repo "${GITHUB_REPOSITORY}" \
+            --json commits \
+            --jq '.commits[-1].oid')
+
+          echo "run-id=${run_id}" | tee -a ${GITHUB_OUTPUT}
+          echo "commit-sha=${last_commit_sha}" | tee -a ${GITHUB_OUTPUT}
+
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: eu-central-1
@@ -1266,8 +1286,8 @@ jobs:
         env:
           BUCKET: neon-github-public-dev
           AWS_REGION: eu-central-1
-          COMMIT_SHA: ${{ github.sha }}
-          RUN_ID: ${{ needs.meta.outputs.release-pr-run-id }}
+          COMMIT_SHA: ${{ steps.fetch-last-release-pr-info.outputs.commit-sha }}
+          RUN_ID: ${{ steps.fetch-last-release-pr-info.outputs.run-id }}
         run: |
           old_prefix="artifacts/${COMMIT_SHA}/${RUN_ID}"
           new_prefix="artifacts/latest"
@@ -1356,5 +1376,5 @@ jobs:
           || needs.files-changed.result == 'skipped'
           || (needs.push-compute-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind))
           || (needs.push-neon-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind))
-          || (needs.test-images.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind))
+          || needs.test-images.result == 'skipped'
           || (needs.trigger-custom-extensions-build-and-wait.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind))

.github/workflows/cargo-deny.yml

@@ -7,7 +7,7 @@ on:
         required: false
         type: string
   schedule:
-    - cron: '0 10 * * *'
+    - cron: '0 0 * * *'
 
 jobs:
   cargo-deny:
@@ -50,9 +50,8 @@ jobs:
           method: chat.postMessage
           token: ${{ secrets.SLACK_BOT_TOKEN }}
           payload: |
-            channel: ${{ vars.SLACK_ON_CALL_DEVPROD_STREAM }}
+            channel: ${{ vars.SLACK_CICD_CHANNEL_ID }}
             text: |
               Periodic cargo-deny on ${{ matrix.ref }}: ${{ job.status }}
               <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
-              Fixing the problem should be fairly straight forward from the logs. If not, <#${{ vars.SLACK_RUST_CHANNEL_ID }}> is there to help.
-              Pinging <!subteam^S0838JPSH32|@oncall-devprod>.
+              Pinging @oncall-devprod.

.github/workflows/fast-forward.yml

@@ -1,36 +0,0 @@
-name: Fast forward merge
-on:
-  pull_request:
-    types: [labeled]
-    branches:
-      - release
-      - release-proxy
-      - release-compute
-
-jobs:
-  fast-forward:
-    if: ${{ github.event.label.name == 'fast-forward' }}
-    runs-on: ubuntu-22.04
-
-    steps:
-      - name: Remove fast-forward label to PR
-        env:
-          GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
-        run: |
-          gh pr edit ${{ github.event.pull_request.number }} --repo "${GITHUB_REPOSITORY}" --remove-label "fast-forward"
-
-      - name: Fast forwarding
-        uses: sequoia-pgp/fast-forward@ea7628bedcb0b0b96e94383ada458d812fca4979
-        # See https://docs.github.com/en/graphql/reference/enums#mergestatestatus
-        if: ${{ github.event.pull_request.mergeable_state  == 'clean' }}
-        with:
-          merge: true
-          comment: on-error
-          github_token: ${{ secrets.CI_ACCESS_TOKEN }}
-
-      - name: Comment if mergeable_state is not clean
-        if: ${{ github.event.pull_request.mergeable_state  != 'clean' }}
-        run: |
-          gh pr comment ${{ github.event.pull_request.number }} \
-            --repo "${GITHUB_REPOSITORY}" \
-            --body "Not trying to forward pull-request, because \`mergeable_state\` is \`${{ github.event.pull_request.mergeable_state }}\`, not \`clean\`."

.github/workflows/lint-release-pr.yml

@@ -1,23 +0,0 @@
-name: Lint Release PR
-
-on:
-  pull_request:
-    branches:
-      - release
-      - release-proxy
-      - release-compute
-
-jobs:
-  lint-release-pr:
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout PR branch
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0  # Fetch full history for git operations
-
-      - name: Run lint script
-        env:
-          RELEASE_BRANCH: ${{ github.base_ref }}
-        run: |
-          ./.github/scripts/lint-release-pr.sh

.github/workflows/pre-merge-checks.yml

@@ -8,6 +8,8 @@ on:
       - .github/workflows/build-build-tools-image.yml
       - .github/workflows/pre-merge-checks.yml
   merge_group:
+    branches:
+      - main
 
 defaults:
   run:
@@ -17,13 +19,11 @@ defaults:
 permissions: {}
 
 jobs:
-  meta:
+  get-changed-files:
     runs-on: ubuntu-22.04
     outputs:
       python-changed: ${{ steps.python-src.outputs.any_changed }}
       rust-changed: ${{ steps.rust-src.outputs.any_changed }}
-      branch: ${{ steps.group-metadata.outputs.branch }}
-      pr-number: ${{ steps.group-metadata.outputs.pr-number }}
     steps:
       - uses: actions/checkout@v4
 
@@ -58,20 +58,12 @@ jobs:
           echo "${PYTHON_CHANGED_FILES}"
           echo "${RUST_CHANGED_FILES}"
 
-      - name: Merge group metadata
-        if: ${{ github.event_name == 'merge_group' }}
-        id: group-metadata
-        env:
-          MERGE_QUEUE_REF: ${{ github.event.merge_group.head_ref }}
-        run: |
-          echo $MERGE_QUEUE_REF | jq -Rr 'capture("refs/heads/gh-readonly-queue/(?<branch>.*)/pr-(?<pr_number>[0-9]+)-[0-9a-f]{40}") | ["branch=" + .branch, "pr-number=" + .pr_number] | .[]' | tee -a "${GITHUB_OUTPUT}"
-
   build-build-tools-image:
     if: |
       false
-      || needs.meta.outputs.python-changed == 'true'
-      || needs.meta.outputs.rust-changed == 'true'
-    needs: [ meta ]
+      || needs.get-changed-files.outputs.python-changed == 'true'
+      || needs.get-changed-files.outputs.rust-changed == 'true'
+    needs: [ get-changed-files ]
     uses: ./.github/workflows/build-build-tools-image.yml
     with:
       # Build only one combination to save time
@@ -80,8 +72,8 @@ jobs:
     secrets: inherit
 
   check-codestyle-python:
-    if: needs.meta.outputs.python-changed == 'true'
-    needs: [ meta, build-build-tools-image ]
+    if: needs.get-changed-files.outputs.python-changed == 'true'
+    needs: [ get-changed-files, build-build-tools-image ]
     uses: ./.github/workflows/_check-codestyle-python.yml
     with:
       # `-bookworm-x64` suffix should match the combination in `build-build-tools-image`
@@ -89,8 +81,8 @@ jobs:
     secrets: inherit
 
   check-codestyle-rust:
-    if: needs.meta.outputs.rust-changed == 'true'
-    needs: [ meta, build-build-tools-image ]
+    if: needs.get-changed-files.outputs.rust-changed == 'true'
+    needs: [ get-changed-files, build-build-tools-image ]
     uses: ./.github/workflows/_check-codestyle-rust.yml
     with:
       # `-bookworm-x64` suffix should match the combination in `build-build-tools-image`
@@ -109,7 +101,7 @@ jobs:
       statuses: write # for `github.repos.createCommitStatus(...)`
       contents: write
     needs:
-      - meta
+      - get-changed-files
       - check-codestyle-python
       - check-codestyle-rust
     runs-on: ubuntu-22.04
@@ -137,20 +129,7 @@ jobs:
         run: exit 1
         if: |
           false
-          || (github.event_name == 'merge_group' && needs.meta.outputs.branch != 'main')
-          || (needs.check-codestyle-python.result == 'skipped' && needs.meta.outputs.python-changed == 'true')
-          || (needs.check-codestyle-rust.result   == 'skipped' && needs.meta.outputs.rust-changed   == 'true')
+          || (needs.check-codestyle-python.result == 'skipped' && needs.get-changed-files.outputs.python-changed == 'true')
+          || (needs.check-codestyle-rust.result   == 'skipped' && needs.get-changed-files.outputs.rust-changed   == 'true')
           || contains(needs.*.result, 'failure')
           || contains(needs.*.result, 'cancelled')
-
-      - name: Add fast-forward label to PR to trigger fast-forward merge
-        if: >-
-          ${{
-            always()
-            && github.event_name == 'merge_group'
-            && contains(fromJson('["release", "release-proxy", "release-compute"]'), github.base_ref)
-          }}
-        env:
-          GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
-        run: >-
-          gh pr edit ${{ needs.meta.outputs.pr-number }} --repo "${GITHUB_REPOSITORY}" --add-label "fast-forward"

.github/workflows/release.yml

@@ -38,7 +38,7 @@ jobs:
     uses: ./.github/workflows/_create-release-pr.yml
     with:
       component-name: 'Storage'
-      source-branch: ${{ github.ref_name }}
+      release-branch: 'release'
     secrets:
       ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }}
 
@@ -51,7 +51,7 @@ jobs:
     uses: ./.github/workflows/_create-release-pr.yml
     with:
       component-name: 'Proxy'
-      source-branch: ${{ github.ref_name }}
+      release-branch: 'release-proxy'
     secrets:
       ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }}
 
@@ -64,6 +64,6 @@ jobs:
     uses: ./.github/workflows/_create-release-pr.yml
     with:
       component-name: 'Compute'
-      source-branch: ${{ github.ref_name }}
+      release-branch: 'release-compute'
     secrets:
       ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }}

CODEOWNERS

@@ -3,6 +3,7 @@
 
 # DevProd & PerfCorr
 /.github/ @neondatabase/developer-productivity @neondatabase/performance-correctness
+/test_runner/	@neondatabase/performance-correctness
 
 # Compute
 /pgxn/ @neondatabase/compute

Cargo.lock

@@ -191,7 +191,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
  "synstructure",
 ]
 
@@ -203,7 +203,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -272,7 +272,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -283,7 +283,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1021,7 +1021,7 @@ dependencies = [
  "regex",
  "rustc-hash 2.1.1",
  "shlex",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1127,9 +1127,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
-version = "1.2.16"
+version = "1.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
+checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945"
 dependencies = [
  "jobserver",
  "libc",
@@ -1248,7 +1248,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1703,7 +1703,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1727,7 +1727,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.10.0",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1738,7 +1738,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1888,7 +1888,7 @@ dependencies = [
  "dsl_auto_type",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1908,7 +1908,7 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25"
 dependencies = [
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1937,7 +1937,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -1960,7 +1960,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -2105,7 +2105,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -2115,19 +2115,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
 dependencies = [
  "log",
- "regex",
 ]
 
 [[package]]
 name = "env_logger"
-version = "0.11.7"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697"
+checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580"
+dependencies = [
+ "humantime",
+ "is-terminal",
+ "log",
+ "regex",
+ "termcolor",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d"
 dependencies = [
- "anstream",
- "anstyle",
  "env_filter",
- "jiff",
  "log",
 ]
 
@@ -2148,7 +2157,7 @@ checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -2408,7 +2417,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -2521,7 +2530,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -2839,7 +2848,6 @@ dependencies = [
  "anyhow",
  "bytes",
  "fail",
- "futures",
  "hyper 0.14.30",
  "itertools 0.10.5",
  "jemalloc_pprof",
@@ -2853,7 +2861,6 @@ dependencies = [
  "serde_path_to_error",
  "thiserror 1.0.69",
  "tokio",
- "tokio-rustls 0.26.0",
  "tokio-stream",
  "tokio-util",
  "tracing",
@@ -3139,7 +3146,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -3232,7 +3239,7 @@ dependencies = [
  "crossbeam-channel",
  "crossbeam-utils",
  "dashmap 6.1.0",
- "env_logger",
+ "env_logger 0.11.2",
  "indexmap 2.0.1",
  "itoa",
  "log",
@@ -3245,11 +3252,11 @@ dependencies = [
 
 [[package]]
 name = "inotify"
-version = "0.9.6"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff"
+checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.8.0",
  "inotify-sys",
  "libc",
 ]
@@ -3355,30 +3362,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "jiff"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e"
-dependencies = [
- "jiff-static",
- "log",
- "portable-atomic",
- "portable-atomic-util",
- "serde",
-]
-
-[[package]]
-name = "jiff-static"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
 [[package]]
 name = "jobserver"
 version = "0.1.32"
@@ -3550,9 +3533,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.26"
+version = "0.4.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
+checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 
 [[package]]
 name = "lru"
@@ -3633,7 +3616,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -3747,18 +3730,6 @@ dependencies = [
  "adler2",
 ]
 
-[[package]]
-name = "mio"
-version = "0.8.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
-dependencies = [
- "libc",
- "log",
- "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.48.0",
-]
-
 [[package]]
 name = "mio"
 version = "1.0.3"
@@ -3766,6 +3737,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
  "libc",
+ "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
  "windows-sys 0.52.0",
 ]
@@ -3843,23 +3815,29 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
 
 [[package]]
 name = "notify"
-version = "6.1.1"
+version = "8.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
+checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943"
 dependencies = [
  "bitflags 2.8.0",
- "crossbeam-channel",
  "filetime",
  "fsevent-sys",
  "inotify",
  "kqueue",
  "libc",
  "log",
- "mio 0.8.11",
+ "mio",
+ "notify-types",
  "walkdir",
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "notify-types"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d"
+
 [[package]]
 name = "ntapi"
 version = "0.4.1"
@@ -4084,7 +4062,7 @@ dependencies = [
  "opentelemetry-http",
  "opentelemetry-proto",
  "opentelemetry_sdk",
- "prost 0.13.3",
+ "prost",
  "reqwest",
  "thiserror 1.0.69",
 ]
@@ -4097,7 +4075,7 @@ checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6"
 dependencies = [
  "opentelemetry",
  "opentelemetry_sdk",
- "prost 0.13.3",
+ "prost",
  "tonic",
 ]
 
@@ -4211,7 +4189,6 @@ dependencies = [
  "pageserver_api",
  "pageserver_client",
  "rand 0.8.5",
- "reqwest",
  "serde",
  "serde_json",
  "tokio",
@@ -4301,9 +4278,6 @@ dependencies = [
  "remote_storage",
  "reqwest",
  "rpds",
- "rustls 0.23.18",
- "rustls-pemfile 2.1.1",
- "rustls-pki-types",
  "scopeguard",
  "send-future",
  "serde",
@@ -4322,13 +4296,11 @@ dependencies = [
  "tokio-epoll-uring",
  "tokio-io-timeout",
  "tokio-postgres",
- "tokio-rustls 0.26.0",
  "tokio-stream",
  "tokio-tar",
  "tokio-util",
  "toml_edit",
  "tracing",
- "tracing-utils",
  "url",
  "utils",
  "uuid",
@@ -4506,7 +4478,7 @@ dependencies = [
  "parquet",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -4608,7 +4580,7 @@ checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -4704,15 +4676,6 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6"
 
-[[package]]
-name = "portable-atomic-util"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
-dependencies = [
- "portable-atomic",
-]
-
 [[package]]
 name = "postgres"
 version = "0.19.7"
@@ -4820,7 +4783,7 @@ dependencies = [
  "bytes",
  "crc32c",
  "criterion",
- "env_logger",
+ "env_logger 0.10.2",
  "log",
  "memoffset 0.9.0",
  "once_cell",
@@ -4867,10 +4830,8 @@ dependencies = [
  "nix 0.26.4",
  "once_cell",
  "parking_lot 0.12.1",
- "prost 0.12.6",
- "prost-build 0.12.6",
- "prost-derive 0.12.6",
- "sha2",
+ "protobuf",
+ "protobuf-codegen-pure",
  "smallvec",
  "symbolic-demangle",
  "tempfile",
@@ -4889,7 +4850,7 @@ dependencies = [
  "inferno 0.12.0",
  "num",
  "paste",
- "prost 0.13.3",
+ "prost",
 ]
 
 [[package]]
@@ -4919,7 +4880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7"
 dependencies = [
  "proc-macro2",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -4933,9 +4894,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.94"
+version = "1.0.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84"
+checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
 dependencies = [
  "unicode-ident",
 ]
@@ -4982,16 +4943,6 @@ dependencies = [
  "thiserror 1.0.69",
 ]
 
-[[package]]
-name = "prost"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29"
-dependencies = [
- "bytes",
- "prost-derive 0.12.6",
-]
-
 [[package]]
 name = "prost"
 version = "0.13.3"
@@ -4999,28 +4950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f"
 dependencies = [
  "bytes",
- "prost-derive 0.13.3",
-]
-
-[[package]]
-name = "prost-build"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
-dependencies = [
- "bytes",
- "heck",
- "itertools 0.12.1",
- "log",
- "multimap",
- "once_cell",
- "petgraph",
- "prettyplease",
- "prost 0.12.6",
- "prost-types 0.12.6",
- "regex",
- "syn 2.0.100",
- "tempfile",
+ "prost-derive",
 ]
 
 [[package]]
@@ -5037,26 +4967,13 @@ dependencies = [
  "once_cell",
  "petgraph",
  "prettyplease",
- "prost 0.13.3",
- "prost-types 0.13.3",
+ "prost",
+ "prost-types",
  "regex",
- "syn 2.0.100",
+ "syn 2.0.90",
  "tempfile",
 ]
 
-[[package]]
-name = "prost-derive"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
-dependencies = [
- "anyhow",
- "itertools 0.12.1",
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
 [[package]]
 name = "prost-derive"
 version = "0.13.3"
@@ -5067,16 +4984,7 @@ dependencies = [
  "itertools 0.12.1",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "prost-types"
-version = "0.12.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0"
-dependencies = [
- "prost 0.12.6",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -5085,7 +4993,32 @@ version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670"
 dependencies = [
- "prost 0.13.3",
+ "prost",
+]
+
+[[package]]
+name = "protobuf"
+version = "2.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
+
+[[package]]
+name = "protobuf-codegen"
+version = "2.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6"
+dependencies = [
+ "protobuf",
+]
+
+[[package]]
+name = "protobuf-codegen-pure"
+version = "2.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865"
+dependencies = [
+ "protobuf",
+ "protobuf-codegen",
 ]
 
 [[package]]
@@ -5114,7 +5047,7 @@ dependencies = [
  "consumption_metrics",
  "ecdsa 0.16.9",
  "ed25519-dalek",
- "env_logger",
+ "env_logger 0.10.2",
  "fallible-iterator",
  "flate2",
  "framed-websockets",
@@ -5251,9 +5184,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.39"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -5694,16 +5627,16 @@ dependencies = [
 
 [[package]]
 name = "ring"
-version = "0.17.13"
+version = "0.17.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee"
+checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866"
 dependencies = [
  "cc",
- "cfg-if",
  "getrandom 0.2.11",
  "libc",
+ "spin",
  "untrusted",
- "windows-sys 0.52.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -5782,7 +5715,7 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version",
- "syn 2.0.100",
+ "syn 2.0.90",
  "unicode-ident",
 ]
 
@@ -5945,9 +5878,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.11.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
+checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b"
 
 [[package]]
 name = "rustls-webpki"
@@ -5997,7 +5930,7 @@ dependencies = [
  "crc32c",
  "criterion",
  "desim",
- "env_logger",
+ "env_logger 0.10.2",
  "fail",
  "futures",
  "hex",
@@ -6328,7 +6261,7 @@ checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -6410,7 +6343,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -6633,7 +6566,7 @@ dependencies = [
  "metrics",
  "once_cell",
  "parking_lot 0.12.1",
- "prost 0.13.3",
+ "prost",
  "rustls 0.23.18",
  "tokio",
  "tonic",
@@ -6651,7 +6584,6 @@ dependencies = [
  "bytes",
  "chrono",
  "clap",
- "clashmap",
  "control_plane",
  "cron",
  "diesel",
@@ -6812,7 +6744,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -6863,9 +6795,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.100"
+version = "2.0.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
+checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6895,7 +6827,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -6946,6 +6878,15 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "termcolor"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "test-context"
 version = "0.3.0"
@@ -6964,7 +6905,7 @@ checksum = "78ea17a2dc368aeca6f554343ced1b1e31f76d63683fa8016e5844bd7a5144a1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -6993,7 +6934,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -7004,7 +6945,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -7144,7 +7085,7 @@ dependencies = [
  "backtrace",
  "bytes",
  "libc",
- "mio 1.0.3",
+ "mio",
  "parking_lot 0.12.1",
  "pin-project-lite",
  "signal-hook-registry",
@@ -7187,7 +7128,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -7397,7 +7338,7 @@ dependencies = [
  "hyper-util",
  "percent-encoding",
  "pin-project",
- "prost 0.13.3",
+ "prost",
  "rustls-native-certs 0.8.0",
  "rustls-pemfile 2.1.1",
  "tokio",
@@ -7417,10 +7358,10 @@ checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11"
 dependencies = [
  "prettyplease",
  "proc-macro2",
- "prost-build 0.13.3",
- "prost-types 0.13.3",
+ "prost-build",
+ "prost-types",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -7535,7 +7476,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -7866,7 +7807,6 @@ dependencies = [
  "tracing",
  "tracing-error",
  "tracing-subscriber",
- "tracing-utils",
  "walkdir",
 ]
 
@@ -7930,7 +7870,7 @@ dependencies = [
  "anyhow",
  "camino-tempfile",
  "clap",
- "env_logger",
+ "env_logger 0.10.2",
  "log",
  "postgres",
  "postgres_ffi",
@@ -7952,7 +7892,7 @@ dependencies = [
  "pageserver_api",
  "postgres_ffi",
  "pprof",
- "prost 0.13.3",
+ "prost",
  "remote_storage",
  "serde",
  "serde_json",
@@ -8035,7 +7975,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
  "wasm-bindgen-shared",
 ]
 
@@ -8069,7 +8009,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -8376,7 +8316,6 @@ name = "workspace_hack"
 version = "0.1.0"
 dependencies = [
  "ahash",
- "anstream",
  "anyhow",
  "base64 0.13.1",
  "base64 0.21.7",
@@ -8393,8 +8332,6 @@ dependencies = [
  "digest",
  "displaydoc",
  "either",
- "env_filter",
- "env_logger",
  "fail",
  "form_urlencoded",
  "futures-channel",
@@ -8430,7 +8367,7 @@ dependencies = [
  "parquet",
  "prettyplease",
  "proc-macro2",
- "prost 0.13.3",
+ "prost",
  "quote",
  "rand 0.8.5",
  "regex",
@@ -8447,7 +8384,7 @@ dependencies = [
  "spki 0.7.3",
  "stable_deref_trait",
  "subtle",
- "syn 2.0.100",
+ "syn 2.0.90",
  "sync_wrapper 0.1.2",
  "tikv-jemalloc-ctl",
  "tikv-jemalloc-sys",
@@ -8564,7 +8501,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
  "synstructure",
 ]
 
@@ -8586,7 +8523,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -8606,7 +8543,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
  "synstructure",
 ]
 
@@ -8628,7 +8565,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]
@@ -8650,7 +8587,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.100",
+ "syn 2.0.90",
 ]
 
 [[package]]

Cargo.toml

@@ -126,9 +126,7 @@ measured = { version = "0.0.22", features=["lasso"] }
 measured-process = { version = "0.0.22" }
 memoffset = "0.9"
 nix = { version = "0.27", features = ["dir", "fs", "process", "socket", "signal", "poll"] }
-# Do not update to >= 7.0.0, at least. The update will have a significant impact
-# on compute startup metrics (start_postgres_ms), >= 25% degradation.
-notify = "6.0.0"
+notify = "8.0.0"
 num_cpus = "1.15"
 num-traits = "0.2.15"
 once_cell = "1.13"
@@ -141,7 +139,7 @@ parquet = { version = "53", default-features = false, features = ["zstd"] }
 parquet_derive = "53"
 pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
 pin-project-lite = "0.2"
-pprof = { version = "0.14", features = ["criterion", "flamegraph", "frame-pointer", "prost-codec"] }
+pprof = { version = "0.14", features = ["criterion", "flamegraph", "frame-pointer", "protobuf", "protobuf-codec"] }
 procfs = "0.16"
 prometheus = {version = "0.13", default-features=false, features = ["process"]} # removes protobuf dependency
 prost = "0.13"
@@ -157,7 +155,6 @@ rpds = "0.13"
 rustc-hash = "1.1.0"
 rustls = { version = "0.23.16", default-features = false }
 rustls-pemfile = "2"
-rustls-pki-types = "1.11"
 scopeguard = "1.1"
 sysinfo = "0.29.2"
 sd-notify = "0.4.1"
@@ -221,7 +218,7 @@ zerocopy = { version = "0.7", features = ["derive"] }
 json-structural-diff = { version = "0.2.0" }
 
 ## TODO replace this with tracing
-env_logger = "0.11"
+env_logger = "0.10"
 log = "0.4"
 
 ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed

compute_tools/src/compute.rs

@@ -645,9 +645,9 @@ impl ComputeNode {
         if pspec.spec.mode == ComputeMode::Primary {
             self.configure_as_primary(&compute_state)?;
 
-            let conf = self.get_tokio_conn_conf(None);
-            tokio::task::spawn(async {
-                let res = get_installed_extensions(conf).await;
+            let conf = self.get_conn_conf(None);
+            tokio::task::spawn_blocking(|| {
+                let res = get_installed_extensions(conf);
                 match res {
                     Ok(extensions) => {
                         info!(

compute_tools/src/extension_server.rs

@@ -202,24 +202,8 @@ pub async fn download_extension(
     // move contents of the libdir / sharedir in unzipped archive to the correct local paths
     for paths in [sharedir_paths, libdir_paths] {
         let (zip_dir, real_dir) = paths;
-
-        let dir = match std::fs::read_dir(&zip_dir) {
-            Ok(dir) => dir,
-            Err(e) => match e.kind() {
-                // In the event of a SQL-only extension, there would be nothing
-                // to move from the lib/ directory, so note that in the log and
-                // move on.
-                std::io::ErrorKind::NotFound => {
-                    info!("nothing to move from {}", zip_dir);
-                    continue;
-                }
-                _ => return Err(anyhow::anyhow!(e)),
-            },
-        };
-
         info!("mv {zip_dir:?}/*  {real_dir:?}");
-
-        for file in dir {
+        for file in std::fs::read_dir(zip_dir)? {
             let old_file = file?.path();
             let new_file =
                 Path::new(&real_dir).join(old_file.file_name().context("error parsing file")?);

compute_tools/src/http/middleware/mod.rs

@@ -1,2 +1 @@
 pub(in crate::http) mod authorize;
-pub(in crate::http) mod request_id;

compute_tools/src/http/middleware/request_id.rs

@@ -1,16 +0,0 @@
-use axum::{extract::Request, middleware::Next, response::Response};
-use uuid::Uuid;
-
-use crate::http::headers::X_REQUEST_ID;
-
-/// This middleware function allows compute_ctl to generate its own request ID
-/// if one isn't supplied. The control plane will always send one as a UUID. The
-/// neon Postgres extension on the other hand does not send one.
-pub async fn maybe_add_request_id_header(mut request: Request, next: Next) -> Response {
-    let headers = request.headers_mut();
-    if !headers.contains_key(X_REQUEST_ID) {
-        headers.append(X_REQUEST_ID, Uuid::new_v4().to_string().parse().unwrap());
-    }
-
-    next.run(request).await
-}

compute_tools/src/http/server.rs

@@ -5,8 +5,9 @@ use std::time::Duration;
 
 use anyhow::Result;
 use axum::Router;
-use axum::middleware::{self};
-use axum::response::IntoResponse;
+use axum::extract::Request;
+use axum::middleware::{self, Next};
+use axum::response::{IntoResponse, Response};
 use axum::routing::{get, post};
 use http::StatusCode;
 use jsonwebtoken::jwk::JwkSet;
@@ -16,8 +17,8 @@ use tower_http::{
     auth::AsyncRequireAuthorizationLayer, request_id::PropagateRequestIdLayer, trace::TraceLayer,
 };
 use tracing::{Span, error, info};
+use uuid::Uuid;
 
-use super::middleware::request_id::maybe_add_request_id_header;
 use super::{
     headers::X_REQUEST_ID,
     middleware::authorize::Authorize,
@@ -218,3 +219,15 @@ impl Server {
         tokio::spawn(self.serve(state));
     }
 }
+
+/// This middleware function allows compute_ctl to generate its own request ID
+/// if one isn't supplied. The control plane will always send one as a UUID. The
+/// neon Postgres extension on the other hand does not send one.
+async fn maybe_add_request_id_header(mut request: Request, next: Next) -> Response {
+    let headers = request.headers_mut();
+    if headers.get(X_REQUEST_ID).is_none() {
+        headers.append(X_REQUEST_ID, Uuid::new_v4().to_string().parse().unwrap());
+    }
+
+    next.run(request).await
+}

compute_tools/src/installed_extensions.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use anyhow::Result;
 use compute_api::responses::{InstalledExtension, InstalledExtensions};
-use tokio_postgres::{Client, Config, NoTls};
+use postgres::{Client, NoTls};
 
 use crate::metrics::INSTALLED_EXTENSIONS;
 
@@ -10,7 +10,7 @@ use crate::metrics::INSTALLED_EXTENSIONS;
 /// and to make database listing query here more explicit.
 ///
 /// Limit the number of databases to 500 to avoid excessive load.
-async fn list_dbs(client: &mut Client) -> Result<Vec<String>> {
+fn list_dbs(client: &mut Client) -> Result<Vec<String>> {
     // `pg_database.datconnlimit = -2` means that the database is in the
     // invalid state
     let databases = client
@@ -20,8 +20,7 @@ async fn list_dbs(client: &mut Client) -> Result<Vec<String>> {
                 AND datconnlimit <> - 2
                 LIMIT 500",
             &[],
-        )
-        .await?
+        )?
         .iter()
         .map(|row| {
             let db: String = row.get("datname");
@@ -37,36 +36,20 @@ async fn list_dbs(client: &mut Client) -> Result<Vec<String>> {
 /// Same extension can be installed in multiple databases with different versions,
 /// so we report a separate metric (number of databases where it is installed)
 /// for each extension version.
-pub async fn get_installed_extensions(mut conf: Config) -> Result<InstalledExtensions> {
+pub fn get_installed_extensions(mut conf: postgres::config::Config) -> Result<InstalledExtensions> {
     conf.application_name("compute_ctl:get_installed_extensions");
-    let databases: Vec<String> = {
-        let (mut client, connection) = conf.connect(NoTls).await?;
-        tokio::spawn(async move {
-            if let Err(e) = connection.await {
-                eprintln!("connection error: {}", e);
-            }
-        });
-
-        list_dbs(&mut client).await?
-    };
+    let mut client = conf.connect(NoTls)?;
+    let databases: Vec<String> = list_dbs(&mut client)?;
 
     let mut extensions_map: HashMap<(String, String, String), InstalledExtension> = HashMap::new();
     for db in databases.iter() {
         conf.dbname(db);
-
-        let (client, connection) = conf.connect(NoTls).await?;
-        tokio::spawn(async move {
-            if let Err(e) = connection.await {
-                eprintln!("connection error: {}", e);
-            }
-        });
-
-        let extensions: Vec<(String, String, i32)> = client
+        let mut db_client = conf.connect(NoTls)?;
+        let extensions: Vec<(String, String, i32)> = db_client
             .query(
                 "SELECT extname, extversion, extowner::integer FROM pg_catalog.pg_extension",
                 &[],
-            )
-            .await?
+            )?
             .iter()
             .map(|row| {
                 (

compute_tools/src/logger.rs

@@ -24,8 +24,7 @@ pub async fn init_tracing_and_logging(default_log_level: &str) -> anyhow::Result
         .with_writer(std::io::stderr);
 
     // Initialize OpenTelemetry
-    let otlp_layer =
-        tracing_utils::init_tracing("compute_ctl", tracing_utils::ExportConfig::default()).await;
+    let otlp_layer = tracing_utils::init_tracing("compute_ctl").await;
 
     // Put it all together
     tracing_subscriber::registry()

compute_tools/src/sql/finalize_drop_subscriptions.sql

@@ -17,5 +17,5 @@ BEGIN
     INSERT INTO neon.drop_subscriptions_done VALUES (1, current_setting('neon.timeline_id'))
     ON CONFLICT (id) DO UPDATE
     SET timeline_id = current_setting('neon.timeline_id');
-END
-$$
+END;
+$$;

compute_tools/src/sql/pre_drop_role_revoke_privileges.sql

@@ -1,7 +1,8 @@
+SET SESSION ROLE neon_superuser;
+
 DO ${outer_tag}$
 DECLARE
     schema TEXT;
-    grantor TEXT;
     revoke_query TEXT;
 BEGIN
     FOR schema IN
@@ -14,25 +15,16 @@ BEGIN
         -- ii) it's easy to add more schemas to the list if needed.
         WHERE schema_name IN ('public')
     LOOP
-        FOR grantor IN EXECUTE
-            format(
-                'SELECT DISTINCT rtg.grantor FROM information_schema.role_table_grants AS rtg WHERE grantee = %s',
-                -- N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`
-                quote_literal({role_name})
-            )
-        LOOP
-            EXECUTE format('SET LOCAL ROLE %I', grantor);
+        revoke_query := format(
+            'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM %I GRANTED BY neon_superuser;',
+            schema,
+            -- N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`
+            {role_name}
+        );
 
-            revoke_query := format(
-                'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM %I GRANTED BY %I',
-                schema,
-                -- N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`
-                {role_name},
-                grantor
-            );
-
-            EXECUTE revoke_query;
-        END LOOP;
+        EXECUTE revoke_query;
     END LOOP;
 END;
 ${outer_tag}$;
+
+RESET ROLE;

control_plane/src/bin/neon_local.rs

@@ -36,9 +36,7 @@ use pageserver_api::config::{
 use pageserver_api::controller_api::{
     NodeAvailabilityWrapper, PlacementPolicy, TenantCreateRequest,
 };
-use pageserver_api::models::{
-    ShardParameters, TenantConfigRequest, TimelineCreateRequest, TimelineInfo,
-};
+use pageserver_api::models::{ShardParameters, TimelineCreateRequest, TimelineInfo};
 use pageserver_api::shard::{ShardCount, ShardStripeSize, TenantShardId};
 use postgres_backend::AuthType;
 use postgres_connection::parse_host_port;
@@ -965,7 +963,6 @@ fn handle_init(args: &InitCmdArgs) -> anyhow::Result<LocalEnv> {
                         id: pageserver_id,
                         listen_pg_addr: format!("127.0.0.1:{pg_port}"),
                         listen_http_addr: format!("127.0.0.1:{http_port}"),
-                        listen_https_addr: None,
                         pg_auth_type: AuthType::Trust,
                         http_auth_type: AuthType::Trust,
                         other: Default::default(),
@@ -980,7 +977,6 @@ fn handle_init(args: &InitCmdArgs) -> anyhow::Result<LocalEnv> {
             default_tenant_id: TenantId::from_array(std::array::from_fn(|_| 0)),
             storage_controller: None,
             control_plane_compute_hook_api: None,
-            generate_local_ssl_certs: false,
         }
     };
 
@@ -1131,16 +1127,12 @@ async fn handle_tenant(subcmd: &TenantCmd, env: &mut local_env::LocalEnv) -> any
             let tenant_id = get_tenant_id(args.tenant_id, env)?;
             let tenant_conf: HashMap<_, _> =
                 args.config.iter().flat_map(|c| c.split_once(':')).collect();
-            let config = PageServerNode::parse_config(tenant_conf)?;
 
-            let req = TenantConfigRequest { tenant_id, config };
-
-            let storage_controller = StorageController::from_env(env);
-            storage_controller
-                .set_tenant_config(&req)
+            pageserver
+                .tenant_config(tenant_id, tenant_conf)
                 .await
                 .with_context(|| format!("Tenant config failed for tenant with id {tenant_id}"))?;
-            println!("tenant {tenant_id} successfully configured via storcon");
+            println!("tenant {tenant_id} successfully configured on the pageserver");
         }
     }
     Ok(())

control_plane/src/local_env.rs

@@ -81,10 +81,6 @@ pub struct LocalEnv {
     // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error.
     // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table".
     pub branch_name_mappings: HashMap<String, Vec<(TenantId, TimelineId)>>,
-
-    /// Flag to generate SSL certificates for components that need it.
-    /// Also generates root CA certificate that is used to sign all other certificates.
-    pub generate_local_ssl_certs: bool,
 }
 
 /// On-disk state stored in `.neon/config`.
@@ -106,10 +102,6 @@ pub struct OnDiskConfig {
     pub control_plane_api: Option<Url>,
     pub control_plane_compute_hook_api: Option<Url>,
     branch_name_mappings: HashMap<String, Vec<(TenantId, TimelineId)>>,
-    // Note: skip serializing because in compat tests old storage controller fails
-    // to load new config file. May be removed after this field is in release branch.
-    #[serde(skip_serializing_if = "std::ops::Not::not")]
-    pub generate_local_ssl_certs: bool,
 }
 
 fn fail_if_pageservers_field_specified<'de, D>(_: D) -> Result<Vec<PageServerConf>, D::Error>
@@ -137,7 +129,6 @@ pub struct NeonLocalInitConf {
     pub safekeepers: Vec<SafekeeperConf>,
     pub control_plane_api: Option<Url>,
     pub control_plane_compute_hook_api: Option<Option<Url>>,
-    pub generate_local_ssl_certs: bool,
 }
 
 /// Broker config for cluster internal communication.
@@ -174,11 +165,6 @@ pub struct NeonStorageControllerConf {
 
     #[serde(with = "humantime_serde")]
     pub long_reconcile_threshold: Option<Duration>,
-
-    #[serde(default)]
-    pub use_https_pageserver_api: bool,
-
-    pub timelines_onto_safekeepers: bool,
 }
 
 impl NeonStorageControllerConf {
@@ -202,8 +188,6 @@ impl Default for NeonStorageControllerConf {
             max_secondary_lag_bytes: None,
             heartbeat_interval: Self::DEFAULT_HEARTBEAT_INTERVAL,
             long_reconcile_threshold: None,
-            use_https_pageserver_api: false,
-            timelines_onto_safekeepers: false,
         }
     }
 }
@@ -233,7 +217,6 @@ pub struct PageServerConf {
     pub id: NodeId,
     pub listen_pg_addr: String,
     pub listen_http_addr: String,
-    pub listen_https_addr: Option<String>,
     pub pg_auth_type: AuthType,
     pub http_auth_type: AuthType,
     pub no_sync: bool,
@@ -245,7 +228,6 @@ impl Default for PageServerConf {
             id: NodeId(0),
             listen_pg_addr: String::new(),
             listen_http_addr: String::new(),
-            listen_https_addr: None,
             pg_auth_type: AuthType::Trust,
             http_auth_type: AuthType::Trust,
             no_sync: false,
@@ -261,7 +243,6 @@ pub struct NeonLocalInitPageserverConf {
     pub id: NodeId,
     pub listen_pg_addr: String,
     pub listen_http_addr: String,
-    pub listen_https_addr: Option<String>,
     pub pg_auth_type: AuthType,
     pub http_auth_type: AuthType,
     #[serde(default, skip_serializing_if = "std::ops::Not::not")]
@@ -276,7 +257,6 @@ impl From<&NeonLocalInitPageserverConf> for PageServerConf {
             id,
             listen_pg_addr,
             listen_http_addr,
-            listen_https_addr,
             pg_auth_type,
             http_auth_type,
             no_sync,
@@ -286,7 +266,6 @@ impl From<&NeonLocalInitPageserverConf> for PageServerConf {
             id: *id,
             listen_pg_addr: listen_pg_addr.clone(),
             listen_http_addr: listen_http_addr.clone(),
-            listen_https_addr: listen_https_addr.clone(),
             pg_auth_type: *pg_auth_type,
             http_auth_type: *http_auth_type,
             no_sync: *no_sync,
@@ -431,41 +410,6 @@ impl LocalEnv {
         }
     }
 
-    pub fn ssl_ca_cert_path(&self) -> Option<PathBuf> {
-        if self.generate_local_ssl_certs {
-            Some(self.base_data_dir.join("rootCA.crt"))
-        } else {
-            None
-        }
-    }
-
-    pub fn ssl_ca_key_path(&self) -> Option<PathBuf> {
-        if self.generate_local_ssl_certs {
-            Some(self.base_data_dir.join("rootCA.key"))
-        } else {
-            None
-        }
-    }
-
-    pub fn generate_ssl_ca_cert(&self) -> anyhow::Result<()> {
-        let cert_path = self.ssl_ca_cert_path().unwrap();
-        let key_path = self.ssl_ca_key_path().unwrap();
-        if !fs::exists(cert_path.as_path())? {
-            generate_ssl_ca_cert(cert_path.as_path(), key_path.as_path())?;
-        }
-        Ok(())
-    }
-
-    pub fn generate_ssl_cert(&self, cert_path: &Path, key_path: &Path) -> anyhow::Result<()> {
-        self.generate_ssl_ca_cert()?;
-        generate_ssl_cert(
-            cert_path,
-            key_path,
-            self.ssl_ca_cert_path().unwrap().as_path(),
-            self.ssl_ca_key_path().unwrap().as_path(),
-        )
-    }
-
     /// Inspect the base data directory and extract the instance id and instance directory path
     /// for all storage controller instances
     pub async fn storage_controller_instances(&self) -> std::io::Result<Vec<(u8, PathBuf)>> {
@@ -575,7 +519,6 @@ impl LocalEnv {
                 control_plane_api,
                 control_plane_compute_hook_api,
                 branch_name_mappings,
-                generate_local_ssl_certs,
             } = on_disk_config;
             LocalEnv {
                 base_data_dir: repopath.to_owned(),
@@ -590,7 +533,6 @@ impl LocalEnv {
                 control_plane_api: control_plane_api.unwrap(),
                 control_plane_compute_hook_api,
                 branch_name_mappings,
-                generate_local_ssl_certs,
             }
         };
 
@@ -626,7 +568,6 @@ impl LocalEnv {
                 struct PageserverConfigTomlSubset {
                     listen_pg_addr: String,
                     listen_http_addr: String,
-                    listen_https_addr: Option<String>,
                     pg_auth_type: AuthType,
                     http_auth_type: AuthType,
                     #[serde(default)]
@@ -651,7 +592,6 @@ impl LocalEnv {
                 let PageserverConfigTomlSubset {
                     listen_pg_addr,
                     listen_http_addr,
-                    listen_https_addr,
                     pg_auth_type,
                     http_auth_type,
                     no_sync,
@@ -669,7 +609,6 @@ impl LocalEnv {
                     },
                     listen_pg_addr,
                     listen_http_addr,
-                    listen_https_addr,
                     pg_auth_type,
                     http_auth_type,
                     no_sync,
@@ -697,7 +636,6 @@ impl LocalEnv {
                 control_plane_api: Some(self.control_plane_api.clone()),
                 control_plane_compute_hook_api: self.control_plane_compute_hook_api.clone(),
                 branch_name_mappings: self.branch_name_mappings.clone(),
-                generate_local_ssl_certs: self.generate_local_ssl_certs,
             },
         )
     }
@@ -780,7 +718,6 @@ impl LocalEnv {
             safekeepers,
             control_plane_api,
             control_plane_compute_hook_api,
-            generate_local_ssl_certs,
         } = conf;
 
         // Find postgres binaries.
@@ -829,13 +766,8 @@ impl LocalEnv {
             control_plane_api: control_plane_api.unwrap(),
             control_plane_compute_hook_api: control_plane_compute_hook_api.unwrap_or_default(),
             branch_name_mappings: Default::default(),
-            generate_local_ssl_certs,
         };
 
-        if generate_local_ssl_certs {
-            env.generate_ssl_ca_cert()?;
-        }
-
         // create endpoints dir
         fs::create_dir_all(env.endpoints_path())?;
 
@@ -919,80 +851,3 @@ fn generate_auth_keys(private_key_path: &Path, public_key_path: &Path) -> anyhow
     }
     Ok(())
 }
-
-fn generate_ssl_ca_cert(cert_path: &Path, key_path: &Path) -> anyhow::Result<()> {
-    // openssl req -x509 -newkey rsa:2048 -nodes -subj "/CN=Neon Local CA" -days 36500 \
-    // -out rootCA.crt -keyout rootCA.key
-    let keygen_output = Command::new("openssl")
-        .args([
-            "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "36500",
-        ])
-        .args(["-subj", "/CN=Neon Local CA"])
-        .args(["-out", cert_path.to_str().unwrap()])
-        .args(["-keyout", key_path.to_str().unwrap()])
-        .output()
-        .context("failed to generate CA certificate")?;
-    if !keygen_output.status.success() {
-        bail!(
-            "openssl failed: '{}'",
-            String::from_utf8_lossy(&keygen_output.stderr)
-        );
-    }
-    Ok(())
-}
-
-fn generate_ssl_cert(
-    cert_path: &Path,
-    key_path: &Path,
-    ca_cert_path: &Path,
-    ca_key_path: &Path,
-) -> anyhow::Result<()> {
-    // Generate Certificate Signing Request (CSR).
-    let mut csr_path = cert_path.to_path_buf();
-    csr_path.set_extension(".csr");
-
-    // openssl req -new -nodes -newkey rsa:2048 -keyout server.key -out server.csr \
-    // -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
-    let keygen_output = Command::new("openssl")
-        .args(["req", "-new", "-nodes"])
-        .args(["-newkey", "rsa:2048"])
-        .args(["-subj", "/CN=localhost"])
-        .args(["-addext", "subjectAltName=DNS:localhost,IP:127.0.0.1"])
-        .args(["-keyout", key_path.to_str().unwrap()])
-        .args(["-out", csr_path.to_str().unwrap()])
-        .output()
-        .context("failed to generate CSR")?;
-    if !keygen_output.status.success() {
-        bail!(
-            "openssl failed: '{}'",
-            String::from_utf8_lossy(&keygen_output.stderr)
-        );
-    }
-
-    // Sign CSR with CA key.
-    //
-    // openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial \
-    // -out server.crt -days 36500 -copy_extensions copyall
-    let keygen_output = Command::new("openssl")
-        .args(["x509", "-req"])
-        .args(["-in", csr_path.to_str().unwrap()])
-        .args(["-CA", ca_cert_path.to_str().unwrap()])
-        .args(["-CAkey", ca_key_path.to_str().unwrap()])
-        .arg("-CAcreateserial")
-        .args(["-out", cert_path.to_str().unwrap()])
-        .args(["-days", "36500"])
-        .args(["-copy_extensions", "copyall"])
-        .output()
-        .context("failed to sign CSR")?;
-    if !keygen_output.status.success() {
-        bail!(
-            "openssl failed: '{}'",
-            String::from_utf8_lossy(&keygen_output.stderr)
-        );
-    }
-
-    // Remove CSR file as it's not needed anymore.
-    fs::remove_file(csr_path)?;
-
-    Ok(())
-}

control_plane/src/pageserver.rs

@@ -21,7 +21,6 @@ use pageserver_api::shard::TenantShardId;
 use pageserver_client::mgmt_api;
 use postgres_backend::AuthType;
 use postgres_connection::{PgConnectionConfig, parse_host_port};
-use reqwest::Certificate;
 use utils::auth::{Claims, Scope};
 use utils::id::{NodeId, TenantId, TimelineId};
 use utils::lsn::Lsn;
@@ -50,29 +49,12 @@ impl PageServerNode {
         let (host, port) =
             parse_host_port(&conf.listen_pg_addr).expect("Unable to parse listen_pg_addr");
         let port = port.unwrap_or(5432);
-
-        let ssl_ca_cert = env.ssl_ca_cert_path().map(|ssl_ca_file| {
-            let buf = std::fs::read(ssl_ca_file).expect("SSL root CA file should exist");
-            Certificate::from_pem(&buf).expect("CA certificate should be valid")
-        });
-
-        let endpoint = if env.storage_controller.use_https_pageserver_api {
-            format!(
-                "https://{}",
-                conf.listen_https_addr.as_ref().expect(
-                    "listen https address should be specified if use_https_pageserver_api is on"
-                )
-            )
-        } else {
-            format!("http://{}", conf.listen_http_addr)
-        };
-
         Self {
             pg_connection_config: PgConnectionConfig::new_host_port(host, port),
             conf: conf.clone(),
             env: env.clone(),
             http_client: mgmt_api::Client::new(
-                endpoint,
+                format!("http://{}", conf.listen_http_addr),
                 {
                     match conf.http_auth_type {
                         AuthType::Trust => None,
@@ -83,9 +65,7 @@ impl PageServerNode {
                     }
                 }
                 .as_deref(),
-                ssl_ca_cert,
-            )
-            .expect("Client constructs with no errors"),
+            ),
         }
     }
 
@@ -240,13 +220,6 @@ impl PageServerNode {
             .context("write identity toml")?;
         drop(identity_toml);
 
-        if self.env.generate_local_ssl_certs {
-            self.env.generate_ssl_cert(
-                datadir.join("server.crt").as_path(),
-                datadir.join("server.key").as_path(),
-            )?;
-        }
-
         // TODO: invoke a TBD config-check command to validate that pageserver will start with the written config
 
         // Write metadata file, used by pageserver on startup to register itself with
@@ -257,15 +230,6 @@ impl PageServerNode {
             parse_host_port(&self.conf.listen_http_addr).expect("Unable to parse listen_http_addr");
         let http_port = http_port.unwrap_or(9898);
 
-        let https_port = match self.conf.listen_https_addr.as_ref() {
-            Some(https_addr) => {
-                let (_https_host, https_port) =
-                    parse_host_port(https_addr).expect("Unable to parse listen_https_addr");
-                Some(https_port.unwrap_or(9899))
-            }
-            None => None,
-        };
-
         // Intentionally hand-craft JSON: this acts as an implicit format compat test
         // in case the pageserver-side structure is edited, and reflects the real life
         // situation: the metadata is written by some other script.
@@ -276,7 +240,6 @@ impl PageServerNode {
                 postgres_port: self.pg_connection_config.port(),
                 http_host: "localhost".to_string(),
                 http_port,
-                https_port,
                 other: HashMap::from([(
                     "availability_zone_id".to_string(),
                     serde_json::json!(az_id),

control_plane/src/storage_controller.rs

@@ -12,10 +12,13 @@ use hyper0::Uri;
 use nix::unistd::Pid;
 use pageserver_api::controller_api::{
     NodeConfigureRequest, NodeDescribeResponse, NodeRegisterRequest, TenantCreateRequest,
-    TenantCreateResponse, TenantLocateResponse,
+    TenantCreateResponse, TenantLocateResponse, TenantShardMigrateRequest,
+    TenantShardMigrateResponse,
 };
-use pageserver_api::models::{TenantConfigRequest, TimelineCreateRequest, TimelineInfo};
-use pageserver_api::shard::TenantShardId;
+use pageserver_api::models::{
+    TenantShardSplitRequest, TenantShardSplitResponse, TimelineCreateRequest, TimelineInfo,
+};
+use pageserver_api::shard::{ShardStripeSize, TenantShardId};
 use pageserver_client::mgmt_api::ResponseErrorMessageExt;
 use postgres_backend::AuthType;
 use reqwest::Method;
@@ -534,14 +537,6 @@ impl StorageController {
             args.push("--start-as-candidate".to_string());
         }
 
-        if self.config.use_https_pageserver_api {
-            args.push("--use-https-pageserver-api".to_string());
-        }
-
-        if let Some(ssl_ca_file) = self.env.ssl_ca_cert_path() {
-            args.push(format!("--ssl-ca-file={}", ssl_ca_file.to_str().unwrap()));
-        }
-
         if let Some(private_key) = &self.private_key {
             let claims = Claims::new(None, Scope::PageServerApi);
             let jwt_token =
@@ -584,10 +579,6 @@ impl StorageController {
             self.env.base_data_dir.display()
         ));
 
-        if self.config.timelines_onto_safekeepers {
-            args.push("--timelines-onto-safekeepers".to_string());
-        }
-
         background_process::start_process(
             COMMAND,
             &instance_dir,
@@ -834,6 +825,41 @@ impl StorageController {
         .await
     }
 
+    #[instrument(skip(self))]
+    pub async fn tenant_migrate(
+        &self,
+        tenant_shard_id: TenantShardId,
+        node_id: NodeId,
+    ) -> anyhow::Result<TenantShardMigrateResponse> {
+        self.dispatch(
+            Method::PUT,
+            format!("control/v1/tenant/{tenant_shard_id}/migrate"),
+            Some(TenantShardMigrateRequest {
+                node_id,
+                migration_config: None,
+            }),
+        )
+        .await
+    }
+
+    #[instrument(skip(self), fields(%tenant_id, %new_shard_count))]
+    pub async fn tenant_split(
+        &self,
+        tenant_id: TenantId,
+        new_shard_count: u8,
+        new_stripe_size: Option<ShardStripeSize>,
+    ) -> anyhow::Result<TenantShardSplitResponse> {
+        self.dispatch(
+            Method::PUT,
+            format!("control/v1/tenant/{tenant_id}/shard_split"),
+            Some(TenantShardSplitRequest {
+                new_shard_count,
+                new_stripe_size,
+            }),
+        )
+        .await
+    }
+
     #[instrument(skip_all, fields(node_id=%req.node_id))]
     pub async fn node_register(&self, req: NodeRegisterRequest) -> anyhow::Result<()> {
         self.dispatch::<_, ()>(Method::POST, "control/v1/node".to_string(), Some(req))
@@ -878,9 +904,4 @@ impl StorageController {
         )
         .await
     }
-
-    pub async fn set_tenant_config(&self, req: &TenantConfigRequest) -> anyhow::Result<()> {
-        self.dispatch(Method::PUT, "v1/tenant/config".to_string(), Some(req))
-            .await
-    }
 }

control_plane/storcon_cli/src/main.rs

@@ -1,21 +1,20 @@
 use std::collections::{HashMap, HashSet};
-use std::path::PathBuf;
 use std::str::FromStr;
 use std::time::Duration;
 
 use clap::{Parser, Subcommand};
 use futures::StreamExt;
 use pageserver_api::controller_api::{
-    AvailabilityZone, MigrationConfig, NodeAvailabilityWrapper, NodeConfigureRequest,
-    NodeDescribeResponse, NodeRegisterRequest, NodeSchedulingPolicy, NodeShardResponse,
-    PlacementPolicy, SafekeeperDescribeResponse, SafekeeperSchedulingPolicyRequest,
-    ShardSchedulingPolicy, ShardsPreferredAzsRequest, ShardsPreferredAzsResponse,
-    SkSchedulingPolicy, TenantCreateRequest, TenantDescribeResponse, TenantPolicyRequest,
-    TenantShardMigrateRequest, TenantShardMigrateResponse,
+    AvailabilityZone, NodeAvailabilityWrapper, NodeConfigureRequest, NodeDescribeResponse,
+    NodeRegisterRequest, NodeSchedulingPolicy, NodeShardResponse, PlacementPolicy,
+    SafekeeperDescribeResponse, SafekeeperSchedulingPolicyRequest, ShardSchedulingPolicy,
+    ShardsPreferredAzsRequest, ShardsPreferredAzsResponse, SkSchedulingPolicy, TenantCreateRequest,
+    TenantDescribeResponse, TenantPolicyRequest, TenantShardMigrateRequest,
+    TenantShardMigrateResponse,
 };
 use pageserver_api::models::{
-    EvictionPolicy, EvictionPolicyLayerAccessThreshold, ShardParameters, TenantConfig,
-    TenantConfigPatchRequest, TenantConfigRequest, TenantShardSplitRequest,
+    EvictionPolicy, EvictionPolicyLayerAccessThreshold, LocationConfigSecondary, ShardParameters,
+    TenantConfig, TenantConfigPatchRequest, TenantConfigRequest, TenantShardSplitRequest,
     TenantShardSplitResponse,
 };
 use pageserver_api::shard::{ShardStripeSize, TenantShardId};
@@ -113,15 +112,6 @@ enum Command {
         tenant_shard_id: TenantShardId,
         #[arg(long)]
         node: NodeId,
-        #[arg(long, default_value_t = true, action = clap::ArgAction::Set)]
-        prewarm: bool,
-        #[arg(long, default_value_t = false, action = clap::ArgAction::Set)]
-        override_scheduler: bool,
-    },
-    /// Watch the location of a tenant shard evolve, e.g. while expecting it to migrate
-    TenantShardWatch {
-        #[arg(long)]
-        tenant_shard_id: TenantShardId,
     },
     /// Migrate the secondary location for a tenant shard to a specific pageserver.
     TenantShardMigrateSecondary {
@@ -158,6 +148,12 @@ enum Command {
         #[arg(long)]
         tenant_id: TenantId,
     },
+    /// For a tenant which hasn't been onboarded to the storage controller yet, add it in secondary
+    /// mode so that it can warm up content on a pageserver.
+    TenantWarmup {
+        #[arg(long)]
+        tenant_id: TenantId,
+    },
     TenantSetPreferredAz {
         #[arg(long)]
         tenant_id: TenantId,
@@ -273,10 +269,6 @@ struct Cli {
     /// a token with both scopes to use with this tool.
     jwt: Option<String>,
 
-    #[arg(long)]
-    /// Trusted root CA certificate to use in https APIs.
-    ssl_ca_file: Option<PathBuf>,
-
     #[command(subcommand)]
     command: Command,
 }
@@ -387,17 +379,9 @@ async fn main() -> anyhow::Result<()> {
 
     let storcon_client = Client::new(cli.api.clone(), cli.jwt.clone());
 
-    let ssl_ca_cert = match &cli.ssl_ca_file {
-        Some(ssl_ca_file) => {
-            let buf = tokio::fs::read(ssl_ca_file).await?;
-            Some(reqwest::Certificate::from_pem(&buf)?)
-        }
-        None => None,
-    };
-
     let mut trimmed = cli.api.to_string();
     trimmed.pop();
-    let vps_client = mgmt_api::Client::new(trimmed, cli.jwt.as_deref(), ssl_ca_cert)?;
+    let vps_client = mgmt_api::Client::new(trimmed, cli.jwt.as_deref());
 
     match cli.command {
         Command::NodeRegister {
@@ -635,43 +619,19 @@ async fn main() -> anyhow::Result<()> {
         Command::TenantShardMigrate {
             tenant_shard_id,
             node,
-            prewarm,
-            override_scheduler,
         } => {
-            let migration_config = MigrationConfig {
-                prewarm,
-                override_scheduler,
-                ..Default::default()
-            };
-
             let req = TenantShardMigrateRequest {
                 node_id: node,
-                origin_node_id: None,
-                migration_config,
+                migration_config: None,
             };
 
-            match storcon_client
+            storcon_client
                 .dispatch::<TenantShardMigrateRequest, TenantShardMigrateResponse>(
                     Method::PUT,
                     format!("control/v1/tenant/{tenant_shard_id}/migrate"),
                     Some(req),
                 )
-                .await
-            {
-                Err(mgmt_api::Error::ApiError(StatusCode::PRECONDITION_FAILED, msg)) => {
-                    anyhow::bail!(
-                        "Migration to {node} rejected, may require `--force` ({}) ",
-                        msg
-                    );
-                }
-                Err(e) => return Err(e.into()),
-                Ok(_) => {}
-            }
-
-            watch_tenant_shard(storcon_client, tenant_shard_id, Some(node)).await?;
-        }
-        Command::TenantShardWatch { tenant_shard_id } => {
-            watch_tenant_shard(storcon_client, tenant_shard_id, None).await?;
+                .await?;
         }
         Command::TenantShardMigrateSecondary {
             tenant_shard_id,
@@ -679,8 +639,7 @@ async fn main() -> anyhow::Result<()> {
         } => {
             let req = TenantShardMigrateRequest {
                 node_id: node,
-                origin_node_id: None,
-                migration_config: MigrationConfig::default(),
+                migration_config: None,
             };
 
             storcon_client
@@ -865,6 +824,94 @@ async fn main() -> anyhow::Result<()> {
                 )
                 .await?;
         }
+        Command::TenantWarmup { tenant_id } => {
+            let describe_response = storcon_client
+                .dispatch::<(), TenantDescribeResponse>(
+                    Method::GET,
+                    format!("control/v1/tenant/{tenant_id}"),
+                    None,
+                )
+                .await;
+            match describe_response {
+                Ok(describe) => {
+                    if matches!(describe.policy, PlacementPolicy::Secondary) {
+                        // Fine: it's already known to controller in secondary mode: calling
+                        // again to put it into secondary mode won't cause problems.
+                    } else {
+                        anyhow::bail!("Tenant already present with policy {:?}", describe.policy);
+                    }
+                }
+                Err(mgmt_api::Error::ApiError(StatusCode::NOT_FOUND, _)) => {
+                    // Fine: this tenant isn't know to the storage controller yet.
+                }
+                Err(e) => {
+                    // Unexpected API error
+                    return Err(e.into());
+                }
+            }
+
+            vps_client
+                .location_config(
+                    TenantShardId::unsharded(tenant_id),
+                    pageserver_api::models::LocationConfig {
+                        mode: pageserver_api::models::LocationConfigMode::Secondary,
+                        generation: None,
+                        secondary_conf: Some(LocationConfigSecondary { warm: true }),
+                        shard_number: 0,
+                        shard_count: 0,
+                        shard_stripe_size: ShardParameters::DEFAULT_STRIPE_SIZE.0,
+                        tenant_conf: TenantConfig::default(),
+                    },
+                    None,
+                    true,
+                )
+                .await?;
+
+            let describe_response = storcon_client
+                .dispatch::<(), TenantDescribeResponse>(
+                    Method::GET,
+                    format!("control/v1/tenant/{tenant_id}"),
+                    None,
+                )
+                .await?;
+
+            let secondary_ps_id = describe_response
+                .shards
+                .first()
+                .unwrap()
+                .node_secondary
+                .first()
+                .unwrap();
+
+            println!("Tenant {tenant_id} warming up on pageserver {secondary_ps_id}");
+            loop {
+                let (status, progress) = vps_client
+                    .tenant_secondary_download(
+                        TenantShardId::unsharded(tenant_id),
+                        Some(Duration::from_secs(10)),
+                    )
+                    .await?;
+                println!(
+                    "Progress: {}/{} layers, {}/{} bytes",
+                    progress.layers_downloaded,
+                    progress.layers_total,
+                    progress.bytes_downloaded,
+                    progress.bytes_total
+                );
+                match status {
+                    StatusCode::OK => {
+                        println!("Download complete");
+                        break;
+                    }
+                    StatusCode::ACCEPTED => {
+                        // Loop
+                    }
+                    _ => {
+                        anyhow::bail!("Unexpected download status: {status}");
+                    }
+                }
+            }
+        }
         Command::TenantDrop { tenant_id, unclean } => {
             if !unclean {
                 anyhow::bail!(
@@ -1058,8 +1105,7 @@ async fn main() -> anyhow::Result<()> {
                                 format!("control/v1/tenant/{}/migrate", mv.tenant_shard_id),
                                 Some(TenantShardMigrateRequest {
                                     node_id: mv.to,
-                                    origin_node_id: Some(mv.from),
-                                    migration_config: MigrationConfig::default(),
+                                    migration_config: None,
                                 }),
                             )
                             .await
@@ -1238,68 +1284,3 @@ async fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
-
-static WATCH_INTERVAL: Duration = Duration::from_secs(5);
-
-async fn watch_tenant_shard(
-    storcon_client: Client,
-    tenant_shard_id: TenantShardId,
-    until_migrated_to: Option<NodeId>,
-) -> anyhow::Result<()> {
-    if let Some(until_migrated_to) = until_migrated_to {
-        println!(
-            "Waiting for tenant shard {} to be migrated to node {}",
-            tenant_shard_id, until_migrated_to
-        );
-    }
-
-    loop {
-        let desc = storcon_client
-            .dispatch::<(), TenantDescribeResponse>(
-                Method::GET,
-                format!("control/v1/tenant/{}", tenant_shard_id.tenant_id),
-                None,
-            )
-            .await?;
-
-        // Output the current state of the tenant shard
-        let shard = desc
-            .shards
-            .iter()
-            .find(|s| s.tenant_shard_id == tenant_shard_id)
-            .ok_or(anyhow::anyhow!("Tenant shard not found"))?;
-        let summary = format!(
-            "attached: {} secondary: {} {}",
-            shard
-                .node_attached
-                .map(|n| format!("{}", n))
-                .unwrap_or("none".to_string()),
-            shard
-                .node_secondary
-                .iter()
-                .map(|n| n.to_string())
-                .collect::<Vec<_>>()
-                .join(","),
-            if shard.is_reconciling {
-                "(reconciler active)"
-            } else {
-                "(reconciler idle)"
-            }
-        );
-        println!("{}", summary);
-
-        // Maybe drop out if we finished migration
-        if let Some(until_migrated_to) = until_migrated_to {
-            if shard.node_attached == Some(until_migrated_to) && !shard.is_reconciling {
-                println!(
-                    "Tenant shard {} is now on node {}",
-                    tenant_shard_id, until_migrated_to
-                );
-                break;
-            }
-        }
-
-        tokio::time::sleep(WATCH_INTERVAL).await;
-    }
-    Ok(())
-}

deny.toml

@@ -27,14 +27,6 @@ yanked = "warn"
 id = "RUSTSEC-2023-0071"
 reason = "the marvin attack only affects private key decryption, not public key signature verification"
 
-[[advisories.ignore]]
-id = "RUSTSEC-2024-0436"
-reason = "The paste crate is a build-only dependency with no runtime components. It is unlikely to have any security impact."
-
-[[advisories.ignore]]
-id = "RUSTSEC-2025-0014"
-reason = "The humantime is widely used and is not easy to replace right now. It is unmaintained, but it has no known vulnerabilities to care about. #11179"
-
 # This section is considered when running `cargo deny check licenses`
 # More documentation for the licenses section can be found here:
 # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html

docs/storage_controller.md

@@ -101,7 +101,7 @@ changes such as a pageserver node becoming unavailable, or the tenant's shard co
 postgres clients to handle such changes, the storage controller calls an API hook when a tenant's pageserver
 location changes.
 
-The hook is configured using the storage controller's `--control-plane-url` CLI option. If the hook requires
+The hook is configured using the storage controller's `--compute-hook-url` CLI option. If the hook requires
 JWT auth, the token may be provided with `--control-plane-jwt-token`. The hook will be invoked with a `PUT` request.
 
 In the Neon cloud service, this hook is implemented by Neon's internal cloud control plane. In `neon_local` systems

libs/http-utils/Cargo.toml

@@ -8,7 +8,6 @@ license.workspace = true
 anyhow.workspace = true
 bytes.workspace = true
 fail.workspace = true
-futures.workspace = true
 hyper0.workspace = true
 itertools.workspace = true
 jemalloc_pprof.workspace = true
@@ -22,7 +21,6 @@ serde_path_to_error.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
 tokio.workspace = true
-tokio-rustls.workspace = true
 tokio-util.workspace = true
 url.workspace = true
 uuid.workspace = true

libs/http-utils/src/endpoint.rs

@@ -399,10 +399,12 @@ pub async fn profile_cpu_handler(req: Request<Body>) -> Result<Response<Body>, A
     // Return the report in the requested format.
     match format {
         Format::Pprof => {
-            let body = report
+            let mut body = Vec::new();
+            report
                 .pprof()
                 .map_err(|err| ApiError::InternalServerError(err.into()))?
-                .encode_to_vec();
+                .write_to_vec(&mut body)
+                .map_err(|err| ApiError::InternalServerError(err.into()))?;
 
             Response::builder()
                 .status(200)

libs/http-utils/src/lib.rs

@@ -3,10 +3,9 @@ pub mod error;
 pub mod failpoints;
 pub mod json;
 pub mod request;
-pub mod server;
 
 extern crate hyper0 as hyper;
 
 /// Current fast way to apply simple http routing in various Neon binaries.
 /// Re-exported for sake of uniform approach, that could be later replaced with better alternatives, if needed.
-pub use routerify::{RequestServiceBuilder, RouterBuilder, RouterService, ext::RequestExt};
+pub use routerify::{RouterBuilder, RouterService, ext::RequestExt};

libs/http-utils/src/server.rs

@@ -1,155 +0,0 @@
-use std::{error::Error, sync::Arc};
-
-use futures::StreamExt;
-use futures::stream::FuturesUnordered;
-use hyper0::Body;
-use hyper0::server::conn::Http;
-use routerify::{RequestService, RequestServiceBuilder};
-use tokio::io::{AsyncRead, AsyncWrite};
-use tokio_rustls::TlsAcceptor;
-use tokio_util::sync::CancellationToken;
-use tracing::{error, info};
-
-use crate::error::ApiError;
-
-/// A simple HTTP server over hyper library.
-/// You may want to use it instead of [`hyper0::server::Server`] because:
-/// 1. hyper0's Server was removed from hyper v1.
-///    It's recommended to replace hyepr0's Server with a manual loop, which is done here.
-/// 2. hyper0's Server doesn't support TLS out of the box, and there is no way
-///    to support it efficiently with the Accept trait that hyper0's Server uses.
-///    That's one of the reasons why it was removed from v1.
-///    <https://github.com/hyperium/hyper/blob/115339d3df50f20c8717680aa35f48858e9a6205/docs/ROADMAP.md#higher-level-client-and-server-problems>
-pub struct Server {
-    request_service: Arc<RequestServiceBuilder<Body, ApiError>>,
-    listener: tokio::net::TcpListener,
-    tls_acceptor: Option<TlsAcceptor>,
-}
-
-impl Server {
-    pub fn new(
-        request_service: Arc<RequestServiceBuilder<Body, ApiError>>,
-        listener: std::net::TcpListener,
-        tls_acceptor: Option<TlsAcceptor>,
-    ) -> anyhow::Result<Self> {
-        // Note: caller of from_std is responsible for setting nonblocking mode.
-        listener.set_nonblocking(true)?;
-        let listener = tokio::net::TcpListener::from_std(listener)?;
-
-        Ok(Self {
-            request_service,
-            listener,
-            tls_acceptor,
-        })
-    }
-
-    pub async fn serve(self, cancel: CancellationToken) -> anyhow::Result<()> {
-        fn suppress_io_error(err: &std::io::Error) -> bool {
-            use std::io::ErrorKind::*;
-            matches!(err.kind(), ConnectionReset | ConnectionAborted | BrokenPipe)
-        }
-        fn suppress_hyper_error(err: &hyper0::Error) -> bool {
-            if err.is_incomplete_message() || err.is_closed() || err.is_timeout() {
-                return true;
-            }
-            if let Some(inner) = err.source() {
-                if let Some(io) = inner.downcast_ref::<std::io::Error>() {
-                    return suppress_io_error(io);
-                }
-            }
-            false
-        }
-
-        let mut connections = FuturesUnordered::new();
-        loop {
-            tokio::select! {
-                stream = self.listener.accept() => {
-                    let (tcp_stream, remote_addr) = match stream {
-                        Ok(stream) => stream,
-                        Err(err) => {
-                            if !suppress_io_error(&err) {
-                                info!("Failed to accept TCP connection: {err:#}");
-                            }
-                            continue;
-                        }
-                    };
-
-                    let service = self.request_service.build(remote_addr);
-                    let tls_acceptor = self.tls_acceptor.clone();
-                    let cancel = cancel.clone();
-
-                    connections.push(tokio::spawn(
-                        async move {
-                            match tls_acceptor {
-                                Some(tls_acceptor) => {
-                                    // Handle HTTPS connection.
-                                    let tls_stream = tokio::select! {
-                                        tls_stream = tls_acceptor.accept(tcp_stream) => tls_stream,
-                                        _ = cancel.cancelled() => return,
-                                    };
-                                    let tls_stream = match tls_stream {
-                                        Ok(tls_stream) => tls_stream,
-                                        Err(err) => {
-                                            if !suppress_io_error(&err) {
-                                                info!("Failed to accept TLS connection: {err:#}");
-                                            }
-                                            return;
-                                        }
-                                    };
-                                    if let Err(err) = Self::serve_connection(tls_stream, service, cancel).await {
-                                        if !suppress_hyper_error(&err) {
-                                            info!("Failed to serve HTTPS connection: {err:#}");
-                                        }
-                                    }
-                                }
-                                None => {
-                                    // Handle HTTP connection.
-                                    if let Err(err) = Self::serve_connection(tcp_stream, service, cancel).await {
-                                        if !suppress_hyper_error(&err) {
-                                            info!("Failed to serve HTTP connection: {err:#}");
-                                        }
-                                    }
-                                }
-                            };
-                        }));
-                 }
-                Some(conn) = connections.next() => {
-                    if let Err(err) = conn {
-                        error!("Connection panicked: {err:#}");
-                    }
-                }
-                _ = cancel.cancelled() => {
-                    // Wait for graceful shutdown of all connections.
-                    while let Some(conn) = connections.next().await {
-                        if let Err(err) = conn {
-                            error!("Connection panicked: {err:#}");
-                        }
-                    }
-                    break;
-                }
-            }
-        }
-        Ok(())
-    }
-
-    /// Serves HTTP connection with graceful shutdown.
-    async fn serve_connection<I>(
-        io: I,
-        service: RequestService<Body, ApiError>,
-        cancel: CancellationToken,
-    ) -> Result<(), hyper0::Error>
-    where
-        I: AsyncRead + AsyncWrite + Unpin + Send + 'static,
-    {
-        let mut conn = Http::new().serve_connection(io, service).with_upgrades();
-
-        tokio::select! {
-            res = &mut conn => res,
-            _ = cancel.cancelled() => {
-                Pin::new(&mut conn).graceful_shutdown();
-                // Note: connection should still be awaited for graceful shutdown to complete.
-                conn.await
-            }
-        }
-    }
-}

libs/pageserver_api/src/config.rs

@@ -35,7 +35,6 @@ pub struct NodeMetadata {
     pub postgres_port: u16,
     pub http_host: String,
     pub http_port: u16,
-    pub https_port: Option<u16>,
 
     // Deployment tools may write fields to the metadata file beyond what we
     // use in this type: this type intentionally only names fields that require.
@@ -58,9 +57,6 @@ pub struct ConfigToml {
     // types mapped 1:1 into the runtime PageServerConfig type
     pub listen_pg_addr: String,
     pub listen_http_addr: String,
-    pub listen_https_addr: Option<String>,
-    pub ssl_key_file: Utf8PathBuf,
-    pub ssl_cert_file: Utf8PathBuf,
     pub availability_zone: Option<String>,
     #[serde(with = "humantime_serde")]
     pub wait_lsn_timeout: Duration,
@@ -425,9 +421,6 @@ pub mod defaults {
 
     pub const DEFAULT_WAL_RECEIVER_PROTOCOL: utils::postgres_client::PostgresClientProtocol =
         utils::postgres_client::PostgresClientProtocol::Vanilla;
-
-    pub const DEFAULT_SSL_KEY_FILE: &str = "server.key";
-    pub const DEFAULT_SSL_CERT_FILE: &str = "server.crt";
 }
 
 impl Default for ConfigToml {
@@ -437,9 +430,6 @@ impl Default for ConfigToml {
         Self {
             listen_pg_addr: (DEFAULT_PG_LISTEN_ADDR.to_string()),
             listen_http_addr: (DEFAULT_HTTP_LISTEN_ADDR.to_string()),
-            listen_https_addr: (None),
-            ssl_key_file: Utf8PathBuf::from(DEFAULT_SSL_KEY_FILE),
-            ssl_cert_file: Utf8PathBuf::from(DEFAULT_SSL_CERT_FILE),
             availability_zone: (None),
             wait_lsn_timeout: (humantime::parse_duration(DEFAULT_WAIT_LSN_TIMEOUT)
                 .expect("cannot parse default wait lsn timeout")),

libs/pageserver_api/src/config/tests.rs

@@ -16,30 +16,6 @@ fn test_node_metadata_v1_backward_compatibilty() {
             postgres_port: 23,
             http_host: "localhost".to_string(),
             http_port: 42,
-            https_port: None,
-            other: HashMap::new(),
-        }
-    )
-}
-
-#[test]
-fn test_node_metadata_v2_backward_compatibilty() {
-    let v2 = serde_json::to_vec(&serde_json::json!({
-        "host": "localhost",
-        "port": 23,
-        "http_host": "localhost",
-        "http_port": 42,
-        "https_port": 123,
-    }));
-
-    assert_eq!(
-        serde_json::from_slice::<NodeMetadata>(&v2.unwrap()).unwrap(),
-        NodeMetadata {
-            postgres_host: "localhost".to_string(),
-            postgres_port: 23,
-            http_host: "localhost".to_string(),
-            http_port: 42,
-            https_port: Some(123),
             other: HashMap::new(),
         }
     )

libs/pageserver_api/src/controller_api.rs

@@ -182,66 +182,20 @@ pub struct TenantDescribeResponseShard {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct TenantShardMigrateRequest {
     pub node_id: NodeId,
-
-    /// Optionally, callers may specify the node they are migrating _from_, and the server will
-    /// reject the request if the shard is no longer attached there: this enables writing safer
-    /// clients that don't risk fighting with some other movement of the shard.
     #[serde(default)]
-    pub origin_node_id: Option<NodeId>,
-
-    #[serde(default)]
-    pub migration_config: MigrationConfig,
+    pub migration_config: Option<MigrationConfig>,
 }
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct MigrationConfig {
-    /// If true, the migration will be executed even if it is to a location with a sub-optimal scheduling
-    /// score: this is usually not what you want, and if you use this then you'll also need to set the
-    /// tenant's scheduling policy to Essential or Pause to avoid the optimiser reverting your migration.
-    ///
-    /// Default: false
-    #[serde(default)]
-    pub override_scheduler: bool,
-
-    /// If true, the migration will be done gracefully by creating a secondary location first and
-    /// waiting for it to warm up before cutting over.  If false, if there is no existing secondary
-    /// location at the destination, the tenant will be migrated immediately.  If the tenant's data
-    /// can't be downloaded within [`Self::secondary_warmup_timeout`], then the migration will go
-    /// ahead but run with a cold cache that can severely reduce performance until it warms up.
-    ///
-    /// When doing a graceful migration, the migration API returns as soon as it is started.
-    ///
-    /// Default: true
-    #[serde(default = "default_prewarm")]
-    pub prewarm: bool,
-
-    /// For non-prewarm migrations which will immediately enter a cutover to the new node: how long to wait
-    /// overall for secondary warmup before cutting over
     #[serde(default)]
     #[serde(with = "humantime_serde")]
     pub secondary_warmup_timeout: Option<Duration>,
-    /// For non-prewarm migrations which will immediately enter a cutover to the new node: how long to wait
-    /// within each secondary download poll call to pageserver.
     #[serde(default)]
     #[serde(with = "humantime_serde")]
     pub secondary_download_request_timeout: Option<Duration>,
 }
 
-fn default_prewarm() -> bool {
-    true
-}
-
-impl Default for MigrationConfig {
-    fn default() -> Self {
-        Self {
-            override_scheduler: false,
-            prewarm: default_prewarm(),
-            secondary_warmup_timeout: None,
-            secondary_download_request_timeout: None,
-        }
-    }
-}
-
 #[derive(Serialize, Clone, Debug)]
 #[serde(into = "NodeAvailabilityWrapper")]
 pub enum NodeAvailability {
@@ -489,7 +443,6 @@ pub struct SafekeeperDescribeResponse {
     pub host: String,
     pub port: i32,
     pub http_port: i32,
-    pub https_port: Option<i32>,
     pub availability_zone_id: String,
     pub scheduling_policy: SkSchedulingPolicy,
 }
@@ -534,43 +487,4 @@ mod test {
             err
         );
     }
-
-    /// Check that a minimal migrate request with no config results in the expected default settings
-    #[test]
-    fn test_migrate_request_decode_defaults() {
-        let json = r#"{
-            "node_id": 123
-        }"#;
-
-        let request: TenantShardMigrateRequest = serde_json::from_str(json).unwrap();
-        assert_eq!(request.node_id, NodeId(123));
-        assert_eq!(request.origin_node_id, None);
-        assert!(!request.migration_config.override_scheduler);
-        assert!(request.migration_config.prewarm);
-        assert_eq!(request.migration_config.secondary_warmup_timeout, None);
-        assert_eq!(
-            request.migration_config.secondary_download_request_timeout,
-            None
-        );
-    }
-
-    /// Check that a partially specified migration config results in the expected default settings
-    #[test]
-    fn test_migration_config_decode_defaults() {
-        // Specify just one field of the config
-        let json = r#"{
-        }"#;
-
-        let config: MigrationConfig = serde_json::from_str(json).unwrap();
-
-        // Check each field's expected default value
-        assert!(!config.override_scheduler);
-        assert!(config.prewarm);
-        assert_eq!(config.secondary_warmup_timeout, None);
-        assert_eq!(config.secondary_download_request_timeout, None);
-        assert_eq!(config.secondary_warmup_timeout, None);
-
-        // Consistency check that the Default impl agrees with our serde defaults
-        assert_eq!(MigrationConfig::default(), config);
-    }
 }

libs/pageserver_api/src/models.rs

@@ -274,31 +274,6 @@ pub struct TimelineCreateRequest {
     pub mode: TimelineCreateRequestMode,
 }
 
-/// Storage controller specific extensions to [`TimelineInfo`].
-#[derive(Serialize, Deserialize, Clone)]
-pub struct TimelineCreateResponseStorcon {
-    #[serde(flatten)]
-    pub timeline_info: TimelineInfo,
-
-    pub safekeepers: Option<SafekeepersInfo>,
-}
-
-/// Safekeepers as returned in timeline creation request to storcon or pushed to
-/// cplane in the post migration hook.
-#[derive(Serialize, Deserialize, Clone)]
-pub struct SafekeepersInfo {
-    pub tenant_id: TenantId,
-    pub timeline_id: TimelineId,
-    pub generation: u32,
-    pub safekeepers: Vec<SafekeeperInfo>,
-}
-
-#[derive(Serialize, Deserialize, Clone)]
-pub struct SafekeeperInfo {
-    pub id: NodeId,
-    pub hostname: String,
-}
-
 #[derive(Serialize, Deserialize, Clone)]
 #[serde(untagged)]
 pub enum TimelineCreateRequestMode {
@@ -1171,15 +1146,6 @@ pub struct TimelineArchivalConfigRequest {
     pub state: TimelineArchivalState,
 }
 
-#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
-pub struct TimelinePatchIndexPartRequest {
-    pub rel_size_migration: Option<RelSizeMigration>,
-    pub gc_compaction_last_completed_lsn: Option<Lsn>,
-    pub applied_gc_cutoff_lsn: Option<Lsn>,
-    #[serde(default)]
-    pub force_index_update: bool,
-}
-
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct TimelinesInfoAndOffloaded {
     pub timelines: Vec<TimelineInfo>,
@@ -1225,10 +1191,9 @@ pub struct TimelineInfo {
     pub last_record_lsn: Lsn,
     pub prev_record_lsn: Option<Lsn>,
 
-    /// Legacy field, retained for one version to enable old storage controller to
-    /// decode (it was a mandatory field).
-    #[serde(default, rename = "latest_gc_cutoff_lsn")]
-    pub _unused: Lsn,
+    /// Legacy field for compat with control plane.  Synonym of `min_readable_lsn`.
+    /// TODO: remove once control plane no longer reads it.
+    pub latest_gc_cutoff_lsn: Lsn,
 
     /// The LSN up to which GC has advanced: older data may still exist but it is not available for clients.
     /// This LSN is not suitable for deciding where to create branches etc: use [`TimelineInfo::min_readable_lsn`] instead,
@@ -1477,14 +1442,8 @@ pub struct TenantScanRemoteStorageResponse {
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "snake_case")]
 pub enum TenantSorting {
-    /// Total size of layers on local disk for all timelines in a shard.
     ResidentSize,
-    /// The logical size of the largest timeline within a _tenant_ (not shard). Only tracked on
-    /// shard 0, contains the sum across all shards.
     MaxLogicalSize,
-    /// The logical size of the largest timeline within a _tenant_ (not shard), divided by number of
-    /// shards. Only tracked on shard 0, and estimates the per-shard logical size.
-    MaxLogicalSizePerShard,
 }
 
 impl Default for TenantSorting {
@@ -1514,20 +1473,14 @@ pub struct TopTenantShardsRequest {
 pub struct TopTenantShardItem {
     pub id: TenantShardId,
 
-    /// Total size of layers on local disk for all timelines in this shard.
+    /// Total size of layers on local disk for all timelines in this tenant
     pub resident_size: u64,
 
-    /// Total size of layers in remote storage for all timelines in this shard.
+    /// Total size of layers in remote storage for all timelines in this tenant
     pub physical_size: u64,
 
-    /// The largest logical size of a timeline within this _tenant_ (not shard). This is only
-    /// tracked on shard 0, and contains the sum of the logical size across all shards.
+    /// The largest logical size of a timeline within this tenant
     pub max_logical_size: u64,
-
-    /// The largest logical size of a timeline within this _tenant_ (not shard) divided by number of
-    /// shards. This is only tracked on shard 0, and is only an estimate as we divide it evenly by
-    /// shard count, rounded up.
-    pub max_logical_size_per_shard: u64,
 }
 
 #[derive(Serialize, Deserialize, Debug, Default)]

libs/pageserver_api/src/shard.rs

@@ -112,16 +112,6 @@ impl ShardIdentity {
         }
     }
 
-    /// An unsharded identity with the given stripe size (if non-zero). This is typically used to
-    /// carry over a stripe size for an unsharded tenant from persistent storage.
-    pub fn unsharded_with_stripe_size(stripe_size: ShardStripeSize) -> Self {
-        let mut shard_identity = Self::unsharded();
-        if stripe_size.0 > 0 {
-            shard_identity.stripe_size = stripe_size;
-        }
-        shard_identity
-    }
-
     /// A broken instance of this type is only used for `TenantState::Broken` tenants,
     /// which are constructed in code paths that don't have access to proper configuration.
     ///

libs/postgres_ffi/src/lib.rs

@@ -396,14 +396,6 @@ pub mod waldecoder {
             self.lsn + self.inputbuf.remaining() as u64
         }
 
-        /// Returns the LSN up to which the WAL decoder has processed.
-        ///
-        /// If [`Self::poll_decode`] returned a record, then this will return
-        /// the end LSN of said record.
-        pub fn lsn(&self) -> Lsn {
-            self.lsn
-        }
-
         pub fn feed_bytes(&mut self, buf: &[u8]) {
             self.inputbuf.extend_from_slice(buf);
         }

libs/proxy/postgres-types2/src/lib.rs

@@ -135,8 +135,8 @@ impl Type {
 pub enum Kind {
     /// A simple type like `VARCHAR` or `INTEGER`.
     Simple,
-    /// An enumerated type.
-    Enum,
+    /// An enumerated type along with its variants.
+    Enum(Vec<String>),
     /// A pseudo-type.
     Pseudo,
     /// An array type along with the type of its elements.
@@ -146,9 +146,9 @@ pub enum Kind {
     /// A multirange type along with the type of its elements.
     Multirange(Type),
     /// A domain type along with its underlying type.
-    Domain(Oid),
-    /// A composite type.
-    Composite(Oid),
+    Domain(Type),
+    /// A composite type along with information about its fields.
+    Composite(Vec<Field>),
 }
 
 /// Information about a field of a composite type.

libs/proxy/tokio-postgres2/src/client.rs

@@ -19,10 +19,10 @@ use crate::config::{Host, SslMode};
 use crate::connection::{Request, RequestMessages};
 use crate::query::RowStream;
 use crate::simple_query::SimpleQueryStream;
-use crate::types::{Oid, Type};
+use crate::types::{Oid, ToSql, Type};
 use crate::{
-    CancelToken, Error, ReadyForQueryStatus, SimpleQueryMessage, Statement, Transaction,
-    TransactionBuilder, query, simple_query,
+    CancelToken, Error, ReadyForQueryStatus, Row, SimpleQueryMessage, Statement, Transaction,
+    TransactionBuilder, query, simple_query, slice_iter,
 };
 
 pub struct Responses {
@@ -54,18 +54,26 @@ impl Responses {
 /// A cache of type info and prepared statements for fetching type info
 /// (corresponding to the queries in the [crate::prepare] module).
 #[derive(Default)]
-pub(crate) struct CachedTypeInfo {
+struct CachedTypeInfo {
     /// A statement for basic information for a type from its
     /// OID. Corresponds to [TYPEINFO_QUERY](crate::prepare::TYPEINFO_QUERY) (or its
     /// fallback).
-    pub(crate) typeinfo: Option<Statement>,
+    typeinfo: Option<Statement>,
+    /// A statement for getting information for a composite type from its OID.
+    /// Corresponds to [TYPEINFO_QUERY](crate::prepare::TYPEINFO_COMPOSITE_QUERY).
+    typeinfo_composite: Option<Statement>,
+    /// A statement for getting information for a composite type from its OID.
+    /// Corresponds to [TYPEINFO_QUERY](crate::prepare::TYPEINFO_COMPOSITE_QUERY) (or
+    /// its fallback).
+    typeinfo_enum: Option<Statement>,
 
     /// Cache of types already looked up.
-    pub(crate) types: HashMap<Oid, Type>,
+    types: HashMap<Oid, Type>,
 }
 
 pub struct InnerClient {
     sender: mpsc::UnboundedSender<Request>,
+    cached_typeinfo: Mutex<CachedTypeInfo>,
 
     /// A buffer to use when writing out postgres commands.
     buffer: Mutex<BytesMut>,
@@ -83,6 +91,38 @@ impl InnerClient {
         })
     }
 
+    pub fn typeinfo(&self) -> Option<Statement> {
+        self.cached_typeinfo.lock().typeinfo.clone()
+    }
+
+    pub fn set_typeinfo(&self, statement: &Statement) {
+        self.cached_typeinfo.lock().typeinfo = Some(statement.clone());
+    }
+
+    pub fn typeinfo_composite(&self) -> Option<Statement> {
+        self.cached_typeinfo.lock().typeinfo_composite.clone()
+    }
+
+    pub fn set_typeinfo_composite(&self, statement: &Statement) {
+        self.cached_typeinfo.lock().typeinfo_composite = Some(statement.clone());
+    }
+
+    pub fn typeinfo_enum(&self) -> Option<Statement> {
+        self.cached_typeinfo.lock().typeinfo_enum.clone()
+    }
+
+    pub fn set_typeinfo_enum(&self, statement: &Statement) {
+        self.cached_typeinfo.lock().typeinfo_enum = Some(statement.clone());
+    }
+
+    pub fn type_(&self, oid: Oid) -> Option<Type> {
+        self.cached_typeinfo.lock().types.get(&oid).cloned()
+    }
+
+    pub fn set_type(&self, oid: Oid, type_: &Type) {
+        self.cached_typeinfo.lock().types.insert(oid, type_.clone());
+    }
+
     /// Call the given function with a buffer to be used when writing out
     /// postgres commands.
     pub fn with_buf<F, R>(&self, f: F) -> R
@@ -102,6 +142,7 @@ pub struct SocketConfig {
     pub host: Host,
     pub port: u16,
     pub connect_timeout: Option<Duration>,
+    // pub keepalive: Option<KeepaliveConfig>,
 }
 
 /// An asynchronous PostgreSQL client.
@@ -110,7 +151,6 @@ pub struct SocketConfig {
 /// through this client object.
 pub struct Client {
     inner: Arc<InnerClient>,
-    cached_typeinfo: CachedTypeInfo,
 
     socket_config: SocketConfig,
     ssl_mode: SslMode,
@@ -129,9 +169,9 @@ impl Client {
         Client {
             inner: Arc::new(InnerClient {
                 sender,
+                cached_typeinfo: Default::default(),
                 buffer: Default::default(),
             }),
-            cached_typeinfo: Default::default(),
 
             socket_config,
             ssl_mode,
@@ -149,6 +189,55 @@ impl Client {
         &self.inner
     }
 
+    /// Executes a statement, returning a vector of the resulting rows.
+    ///
+    /// A statement may contain parameters, specified by `$n`, where `n` is the index of the parameter of the list
+    /// provided, 1-indexed.
+    ///
+    /// The `statement` argument can either be a `Statement`, or a raw query string. If the same statement will be
+    /// repeatedly executed (perhaps with different query parameters), consider preparing the statement up front
+    /// with the `prepare` method.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the number of parameters provided does not match the number expected.
+    pub async fn query(
+        &self,
+        statement: Statement,
+        params: &[&(dyn ToSql + Sync)],
+    ) -> Result<Vec<Row>, Error> {
+        self.query_raw(statement, slice_iter(params))
+            .await?
+            .try_collect()
+            .await
+    }
+
+    /// The maximally flexible version of [`query`].
+    ///
+    /// A statement may contain parameters, specified by `$n`, where `n` is the index of the parameter of the list
+    /// provided, 1-indexed.
+    ///
+    /// The `statement` argument can either be a `Statement`, or a raw query string. If the same statement will be
+    /// repeatedly executed (perhaps with different query parameters), consider preparing the statement up front
+    /// with the `prepare` method.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the number of parameters provided does not match the number expected.
+    ///
+    /// [`query`]: #method.query
+    pub async fn query_raw<'a, I>(
+        &self,
+        statement: Statement,
+        params: I,
+    ) -> Result<RowStream, Error>
+    where
+        I: IntoIterator<Item = &'a (dyn ToSql + Sync)>,
+        I::IntoIter: ExactSizeIterator,
+    {
+        query::query(&self.inner, statement, params).await
+    }
+
     /// Pass text directly to the Postgres backend to allow it to sort out typing itself and
     /// to save a roundtrip
     pub async fn query_raw_txt<S, I>(&self, statement: &str, params: I) -> Result<RowStream, Error>
@@ -195,14 +284,6 @@ impl Client {
         simple_query::batch_execute(self.inner(), query).await
     }
 
-    pub async fn discard_all(&mut self) -> Result<ReadyForQueryStatus, Error> {
-        // clear the prepared statements that are about to be nuked from the postgres session
-
-        self.cached_typeinfo.typeinfo = None;
-
-        self.batch_execute("discard all").await
-    }
-
     /// Begins a new database transaction.
     ///
     /// The transaction will roll back by default - use the `commit` method to commit it.
@@ -266,8 +347,8 @@ impl Client {
     }
 
     /// Query for type information
-    pub(crate) async fn get_type_inner(&mut self, oid: Oid) -> Result<Type, Error> {
-        crate::prepare::get_type(&self.inner, &mut self.cached_typeinfo, oid).await
+    pub async fn get_type(&self, oid: Oid) -> Result<Type, Error> {
+        crate::prepare::get_type(&self.inner, oid).await
     }
 
     /// Determines if the connection to the server has already closed.

libs/proxy/tokio-postgres2/src/generic_client.rs

@@ -22,7 +22,7 @@ pub trait GenericClient: private::Sealed {
         I::IntoIter: ExactSizeIterator + Sync + Send;
 
     /// Query for type information
-    async fn get_type(&mut self, oid: Oid) -> Result<Type, Error>;
+    async fn get_type(&self, oid: Oid) -> Result<Type, Error>;
 }
 
 impl private::Sealed for Client {}
@@ -38,8 +38,8 @@ impl GenericClient for Client {
     }
 
     /// Query for type information
-    async fn get_type(&mut self, oid: Oid) -> Result<Type, Error> {
-        self.get_type_inner(oid).await
+    async fn get_type(&self, oid: Oid) -> Result<Type, Error> {
+        crate::prepare::get_type(self.inner(), oid).await
     }
 }
 
@@ -56,7 +56,7 @@ impl GenericClient for Transaction<'_> {
     }
 
     /// Query for type information
-    async fn get_type(&mut self, oid: Oid) -> Result<Type, Error> {
-        self.client_mut().get_type(oid).await
+    async fn get_type(&self, oid: Oid) -> Result<Type, Error> {
+        self.client().get_type(oid).await
     }
 }

libs/proxy/tokio-postgres2/src/prepare.rs

@@ -9,10 +9,10 @@ use log::debug;
 use postgres_protocol2::message::backend::Message;
 use postgres_protocol2::message::frontend;
 
-use crate::client::{CachedTypeInfo, InnerClient};
+use crate::client::InnerClient;
 use crate::codec::FrontendMessage;
 use crate::connection::RequestMessages;
-use crate::types::{Kind, Oid, Type};
+use crate::types::{Field, Kind, Oid, Type};
 use crate::{Column, Error, Statement, query, slice_iter};
 
 pub(crate) const TYPEINFO_QUERY: &str = "\
@@ -23,7 +23,23 @@ INNER JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid
 WHERE t.oid = $1
 ";
 
-async fn prepare_typecheck(
+const TYPEINFO_ENUM_QUERY: &str = "\
+SELECT enumlabel
+FROM pg_catalog.pg_enum
+WHERE enumtypid = $1
+ORDER BY enumsortorder
+";
+
+pub(crate) const TYPEINFO_COMPOSITE_QUERY: &str = "\
+SELECT attname, atttypid
+FROM pg_catalog.pg_attribute
+WHERE attrelid = $1
+AND NOT attisdropped
+AND attnum > 0
+ORDER BY attnum
+";
+
+pub async fn prepare(
     client: &Arc<InnerClient>,
     name: &'static str,
     query: &str,
@@ -51,7 +67,7 @@ async fn prepare_typecheck(
     let mut parameters = vec![];
     let mut it = parameter_description.parameters();
     while let Some(oid) = it.next().map_err(Error::parse)? {
-        let type_ = Type::from_oid(oid).ok_or_else(Error::unexpected_message)?;
+        let type_ = get_type(client, oid).await?;
         parameters.push(type_);
     }
 
@@ -59,7 +75,7 @@ async fn prepare_typecheck(
     if let Some(row_description) = row_description {
         let mut it = row_description.fields();
         while let Some(field) = it.next().map_err(Error::parse)? {
-            let type_ = Type::from_oid(field.type_oid()).ok_or_else(Error::unexpected_message)?;
+            let type_ = get_type(client, field.type_oid()).await?;
             let column = Column::new(field.name().to_string(), type_, field);
             columns.push(column);
         }
@@ -68,6 +84,15 @@ async fn prepare_typecheck(
     Ok(Statement::new(client, name, parameters, columns))
 }
 
+fn prepare_rec<'a>(
+    client: &'a Arc<InnerClient>,
+    name: &'static str,
+    query: &'a str,
+    types: &'a [Type],
+) -> Pin<Box<dyn Future<Output = Result<Statement, Error>> + 'a + Send>> {
+    Box::pin(prepare(client, name, query, types))
+}
+
 fn encode(client: &InnerClient, name: &str, query: &str, types: &[Type]) -> Result<Bytes, Error> {
     if types.is_empty() {
         debug!("preparing query {}: {}", name, query);
@@ -83,20 +108,16 @@ fn encode(client: &InnerClient, name: &str, query: &str, types: &[Type]) -> Resu
     })
 }
 
-pub async fn get_type(
-    client: &Arc<InnerClient>,
-    typecache: &mut CachedTypeInfo,
-    oid: Oid,
-) -> Result<Type, Error> {
+pub async fn get_type(client: &Arc<InnerClient>, oid: Oid) -> Result<Type, Error> {
     if let Some(type_) = Type::from_oid(oid) {
         return Ok(type_);
     }
 
-    if let Some(type_) = typecache.types.get(&oid) {
-        return Ok(type_.clone());
-    };
+    if let Some(type_) = client.type_(oid) {
+        return Ok(type_);
+    }
 
-    let stmt = typeinfo_statement(client, typecache).await?;
+    let stmt = typeinfo_statement(client).await?;
 
     let rows = query::query(client, stmt, slice_iter(&[&oid])).await?;
     pin_mut!(rows);
@@ -115,48 +136,100 @@ pub async fn get_type(
     let relid: Oid = row.try_get(6)?;
 
     let kind = if type_ == b'e' as i8 {
-        Kind::Enum
+        let variants = get_enum_variants(client, oid).await?;
+        Kind::Enum(variants)
     } else if type_ == b'p' as i8 {
         Kind::Pseudo
     } else if basetype != 0 {
-        Kind::Domain(basetype)
+        let type_ = get_type_rec(client, basetype).await?;
+        Kind::Domain(type_)
     } else if elem_oid != 0 {
-        let type_ = get_type_rec(client, typecache, elem_oid).await?;
+        let type_ = get_type_rec(client, elem_oid).await?;
         Kind::Array(type_)
     } else if relid != 0 {
-        Kind::Composite(relid)
+        let fields = get_composite_fields(client, relid).await?;
+        Kind::Composite(fields)
     } else if let Some(rngsubtype) = rngsubtype {
-        let type_ = get_type_rec(client, typecache, rngsubtype).await?;
+        let type_ = get_type_rec(client, rngsubtype).await?;
         Kind::Range(type_)
     } else {
         Kind::Simple
     };
 
     let type_ = Type::new(name, oid, kind, schema);
-    typecache.types.insert(oid, type_.clone());
+    client.set_type(oid, &type_);
 
     Ok(type_)
 }
 
 fn get_type_rec<'a>(
     client: &'a Arc<InnerClient>,
-    typecache: &'a mut CachedTypeInfo,
     oid: Oid,
 ) -> Pin<Box<dyn Future<Output = Result<Type, Error>> + Send + 'a>> {
-    Box::pin(get_type(client, typecache, oid))
+    Box::pin(get_type(client, oid))
 }
 
-async fn typeinfo_statement(
-    client: &Arc<InnerClient>,
-    typecache: &mut CachedTypeInfo,
-) -> Result<Statement, Error> {
-    if let Some(stmt) = &typecache.typeinfo {
-        return Ok(stmt.clone());
+async fn typeinfo_statement(client: &Arc<InnerClient>) -> Result<Statement, Error> {
+    if let Some(stmt) = client.typeinfo() {
+        return Ok(stmt);
     }
 
     let typeinfo = "neon_proxy_typeinfo";
-    let stmt = prepare_typecheck(client, typeinfo, TYPEINFO_QUERY, &[]).await?;
+    let stmt = prepare_rec(client, typeinfo, TYPEINFO_QUERY, &[]).await?;
 
-    typecache.typeinfo = Some(stmt.clone());
+    client.set_typeinfo(&stmt);
+    Ok(stmt)
+}
+
+async fn get_enum_variants(client: &Arc<InnerClient>, oid: Oid) -> Result<Vec<String>, Error> {
+    let stmt = typeinfo_enum_statement(client).await?;
+
+    query::query(client, stmt, slice_iter(&[&oid]))
+        .await?
+        .and_then(|row| async move { row.try_get(0) })
+        .try_collect()
+        .await
+}
+
+async fn typeinfo_enum_statement(client: &Arc<InnerClient>) -> Result<Statement, Error> {
+    if let Some(stmt) = client.typeinfo_enum() {
+        return Ok(stmt);
+    }
+
+    let typeinfo = "neon_proxy_typeinfo_enum";
+    let stmt = prepare_rec(client, typeinfo, TYPEINFO_ENUM_QUERY, &[]).await?;
+
+    client.set_typeinfo_enum(&stmt);
+    Ok(stmt)
+}
+
+async fn get_composite_fields(client: &Arc<InnerClient>, oid: Oid) -> Result<Vec<Field>, Error> {
+    let stmt = typeinfo_composite_statement(client).await?;
+
+    let rows = query::query(client, stmt, slice_iter(&[&oid]))
+        .await?
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    let mut fields = vec![];
+    for row in rows {
+        let name = row.try_get(0)?;
+        let oid = row.try_get(1)?;
+        let type_ = get_type_rec(client, oid).await?;
+        fields.push(Field::new(name, type_));
+    }
+
+    Ok(fields)
+}
+
+async fn typeinfo_composite_statement(client: &Arc<InnerClient>) -> Result<Statement, Error> {
+    if let Some(stmt) = client.typeinfo_composite() {
+        return Ok(stmt);
+    }
+
+    let typeinfo = "neon_proxy_typeinfo_composite";
+    let stmt = prepare_rec(client, typeinfo, TYPEINFO_COMPOSITE_QUERY, &[]).await?;
+
+    client.set_typeinfo_composite(&stmt);
     Ok(stmt)
 }

libs/proxy/tokio-postgres2/src/transaction.rs

@@ -72,9 +72,4 @@ impl<'a> Transaction<'a> {
     pub fn client(&self) -> &Client {
         self.client
     }
-
-    /// Returns a reference to the underlying `Client`.
-    pub fn client_mut(&mut self) -> &mut Client {
-        self.client
-    }
 }

libs/safekeeper_api/src/membership.rs

@@ -131,14 +131,6 @@ impl Configuration {
         }
     }
 
-    pub fn new(members: MemberSet) -> Self {
-        Configuration {
-            generation: INITIAL_GENERATION,
-            members,
-            new_members: None,
-        }
-    }
-
     /// Is `sk_id` member of the configuration?
     pub fn contains(&self, sk_id: NodeId) -> bool {
         self.members.contains(sk_id) || self.new_members.as_ref().is_some_and(|m| m.contains(sk_id))

libs/safekeeper_api/src/models.rs

@@ -18,7 +18,7 @@ pub struct SafekeeperStatus {
     pub id: NodeId,
 }
 
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize)]
 pub struct TimelineCreateRequest {
     pub tenant_id: TenantId,
     pub timeline_id: TimelineId,
@@ -283,7 +283,7 @@ pub struct SafekeeperUtilization {
 }
 
 /// pull_timeline request body.
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct PullTimelineRequest {
     pub tenant_id: TenantId,
     pub timeline_id: TimelineId,

libs/tracing-utils/src/lib.rs

@@ -21,7 +21,7 @@
 //!         .with_writer(std::io::stderr);
 //!
 //!     // Initialize OpenTelemetry. Exports tracing spans as OpenTelemetry traces
-//!     let otlp_layer = tracing_utils::init_tracing("my_application", tracing_utils::ExportConfig::default()).await;
+//!     let otlp_layer = tracing_utils::init_tracing("my_application").await;
 //!
 //!     // Put it all together
 //!     tracing_subscriber::registry()
@@ -38,12 +38,8 @@ pub mod http;
 
 use opentelemetry::KeyValue;
 use opentelemetry::trace::TracerProvider;
-use opentelemetry_otlp::WithExportConfig;
-pub use opentelemetry_otlp::{ExportConfig, Protocol};
-use tracing::level_filters::LevelFilter;
-use tracing::{Dispatch, Subscriber};
+use tracing::Subscriber;
 use tracing_subscriber::Layer;
-use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::registry::LookupSpan;
 
 /// Set up OpenTelemetry exporter, using configuration from environment variables.
@@ -73,28 +69,19 @@ use tracing_subscriber::registry::LookupSpan;
 ///
 /// This doesn't block, but is marked as 'async' to hint that this must be called in
 /// asynchronous execution context.
-pub async fn init_tracing<S>(
-    service_name: &str,
-    export_config: ExportConfig,
-) -> Option<impl Layer<S>>
+pub async fn init_tracing<S>(service_name: &str) -> Option<impl Layer<S>>
 where
     S: Subscriber + for<'span> LookupSpan<'span>,
 {
     if std::env::var("OTEL_SDK_DISABLED") == Ok("true".to_string()) {
         return None;
     };
-    Some(init_tracing_internal(
-        service_name.to_string(),
-        export_config,
-    ))
+    Some(init_tracing_internal(service_name.to_string()))
 }
 
 /// Like `init_tracing`, but creates a separate tokio Runtime for the tracing
 /// tasks.
-pub fn init_tracing_without_runtime<S>(
-    service_name: &str,
-    export_config: ExportConfig,
-) -> Option<impl Layer<S>>
+pub fn init_tracing_without_runtime<S>(service_name: &str) -> Option<impl Layer<S>>
 where
     S: Subscriber + for<'span> LookupSpan<'span>,
 {
@@ -125,22 +112,16 @@ where
     ));
     let _guard = runtime.enter();
 
-    Some(init_tracing_internal(
-        service_name.to_string(),
-        export_config,
-    ))
+    Some(init_tracing_internal(service_name.to_string()))
 }
 
-fn init_tracing_internal<S>(service_name: String, export_config: ExportConfig) -> impl Layer<S>
+fn init_tracing_internal<S>(service_name: String) -> impl Layer<S>
 where
     S: Subscriber + for<'span> LookupSpan<'span>,
 {
-    // Sets up exporter from the provided [`ExportConfig`] parameter.
-    // If the endpoint is not specified, it is loaded from the
-    // OTEL_EXPORTER_OTLP_ENDPOINT environment variable.
+    // Sets up exporter from the OTEL_EXPORTER_* environment variables.
     let exporter = opentelemetry_otlp::SpanExporter::builder()
         .with_http()
-        .with_export_config(export_config)
         .build()
         .expect("could not initialize opentelemetry exporter");
 
@@ -170,51 +151,3 @@ where
 pub fn shutdown_tracing() {
     opentelemetry::global::shutdown_tracer_provider();
 }
-
-pub enum OtelEnablement {
-    Disabled,
-    Enabled {
-        service_name: String,
-        export_config: ExportConfig,
-        runtime: &'static tokio::runtime::Runtime,
-    },
-}
-
-pub struct OtelGuard {
-    pub dispatch: Dispatch,
-}
-
-impl Drop for OtelGuard {
-    fn drop(&mut self) {
-        shutdown_tracing();
-    }
-}
-
-/// Initializes OTEL infrastructure for performance tracing according to the provided configuration
-///
-/// Performance tracing is handled by a different [`tracing::Subscriber`]. This functions returns
-/// an [`OtelGuard`] containing a [`tracing::Dispatch`] associated with a newly created subscriber.
-/// Applications should use this dispatch for their performance traces.
-///
-/// The lifetime of the guard should match taht of the application. On drop, it tears down the
-/// OTEL infra.
-pub fn init_performance_tracing(otel_enablement: OtelEnablement) -> Option<OtelGuard> {
-    let otel_subscriber = match otel_enablement {
-        OtelEnablement::Disabled => None,
-        OtelEnablement::Enabled {
-            service_name,
-            export_config,
-            runtime,
-        } => {
-            let otel_layer = runtime
-                .block_on(init_tracing(&service_name, export_config))
-                .with_filter(LevelFilter::INFO);
-            let otel_subscriber = tracing_subscriber::registry().with(otel_layer);
-            let otel_dispatch = Dispatch::new(otel_subscriber);
-
-            Some(otel_dispatch)
-        }
-    };
-
-    otel_subscriber.map(|dispatch| OtelGuard { dispatch })
-}

libs/utils/Cargo.toml

@@ -42,7 +42,6 @@ toml_edit = { workspace = true, features = ["serde"] }
 tracing.workspace = true
 tracing-error.workspace = true
 tracing-subscriber = { workspace = true, features = ["json", "registry"] }
-tracing-utils.workspace = true
 rand.workspace = true
 scopeguard.workspace = true
 strum.workspace = true

libs/utils/src/logging.rs

@@ -165,7 +165,6 @@ pub fn init(
         };
         log_layer.with_filter(rust_log_env_filter())
     });
-
     let r = r.with(
         TracingEventCountLayer(&TRACING_EVENT_COUNT_METRIC).with_filter(rust_log_env_filter()),
     );

pageserver/Cargo.toml

@@ -48,9 +48,6 @@ pprof.workspace = true
 rand.workspace = true
 range-set-blaze = { version = "0.1.16", features = ["alloc"] }
 regex.workspace = true
-rustls-pemfile.workspace = true
-rustls-pki-types.workspace = true
-rustls.workspace = true
 scopeguard.workspace = true
 send-future.workspace = true
 serde.workspace = true
@@ -65,12 +62,10 @@ tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util"
 tokio-epoll-uring.workspace = true
 tokio-io-timeout.workspace = true
 tokio-postgres.workspace = true
-tokio-rustls.workspace = true
 tokio-stream.workspace = true
 tokio-util.workspace = true
 toml_edit = { workspace = true, features = [ "serde" ] }
 tracing.workspace = true
-tracing-utils.workspace = true
 url.workspace = true
 walkdir.workspace = true
 metrics.workspace = true
@@ -121,10 +116,6 @@ harness = false
 name = "upload_queue"
 harness = false
 
-[[bench]]
-name = "bench_metrics"
-harness = false
-
 [[bin]]
 name = "test_helper_slow_client_reads"
 required-features = [ "testing" ]

pageserver/benches/bench_metrics.rs

@@ -1,366 +0,0 @@
-use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
-use utils::id::{TenantId, TimelineId};
-
-//
-// Demonstrates that repeat label values lookup is a multicore scalability bottleneck
-// that is worth avoiding.
-//
-criterion_group!(
-    label_values,
-    label_values::bench_naive_usage,
-    label_values::bench_cache_label_values_lookup
-);
-mod label_values {
-    use super::*;
-
-    pub fn bench_naive_usage(c: &mut Criterion) {
-        let mut g = c.benchmark_group("label_values__naive_usage");
-
-        for ntimelines in [1, 4, 8] {
-            g.bench_with_input(
-                BenchmarkId::new("ntimelines", ntimelines),
-                &ntimelines,
-                |b, ntimelines| {
-                    b.iter_custom(|iters| {
-                        let barrier = std::sync::Barrier::new(*ntimelines + 1);
-
-                        let timelines = (0..*ntimelines)
-                            .map(|_| {
-                                (
-                                    TenantId::generate().to_string(),
-                                    "0000".to_string(),
-                                    TimelineId::generate().to_string(),
-                                )
-                            })
-                            .collect::<Vec<_>>();
-
-                        let metric_vec = metrics::UIntGaugeVec::new(
-                            metrics::opts!("testmetric", "testhelp"),
-                            &["tenant_id", "shard_id", "timeline_id"],
-                        )
-                        .unwrap();
-
-                        std::thread::scope(|s| {
-                            for (tenant_id, shard_id, timeline_id) in &timelines {
-                                s.spawn(|| {
-                                    barrier.wait();
-                                    for _ in 0..iters {
-                                        metric_vec
-                                            .with_label_values(&[tenant_id, shard_id, timeline_id])
-                                            .inc();
-                                    }
-                                    barrier.wait();
-                                });
-                            }
-                            barrier.wait();
-                            let start = std::time::Instant::now();
-                            barrier.wait();
-                            start.elapsed()
-                        })
-                    })
-                },
-            );
-        }
-        g.finish();
-    }
-
-    pub fn bench_cache_label_values_lookup(c: &mut Criterion) {
-        let mut g = c.benchmark_group("label_values__cache_label_values_lookup");
-
-        for ntimelines in [1, 4, 8] {
-            g.bench_with_input(
-                BenchmarkId::new("ntimelines", ntimelines),
-                &ntimelines,
-                |b, ntimelines| {
-                    b.iter_custom(|iters| {
-                        let barrier = std::sync::Barrier::new(*ntimelines + 1);
-
-                        let timelines = (0..*ntimelines)
-                            .map(|_| {
-                                (
-                                    TenantId::generate().to_string(),
-                                    "0000".to_string(),
-                                    TimelineId::generate().to_string(),
-                                )
-                            })
-                            .collect::<Vec<_>>();
-
-                        let metric_vec = metrics::UIntGaugeVec::new(
-                            metrics::opts!("testmetric", "testhelp"),
-                            &["tenant_id", "shard_id", "timeline_id"],
-                        )
-                        .unwrap();
-
-                        std::thread::scope(|s| {
-                            for (tenant_id, shard_id, timeline_id) in &timelines {
-                                s.spawn(|| {
-                                    let metric = metric_vec.with_label_values(&[
-                                        tenant_id,
-                                        shard_id,
-                                        timeline_id,
-                                    ]);
-                                    barrier.wait();
-                                    for _ in 0..iters {
-                                        metric.inc();
-                                    }
-                                    barrier.wait();
-                                });
-                            }
-                            barrier.wait();
-                            let start = std::time::Instant::now();
-                            barrier.wait();
-                            start.elapsed()
-                        })
-                    })
-                },
-            );
-        }
-        g.finish();
-    }
-}
-
-//
-// Demonstrates that even a single metric can be a scalability bottleneck
-// if multiple threads in it concurrently but there's nothing we can do
-// about it without changing the metrics framework to use e.g. sharded counte atomics.
-//
-criterion_group!(
-    single_metric_multicore_scalability,
-    single_metric_multicore_scalability::bench,
-);
-mod single_metric_multicore_scalability {
-    use super::*;
-
-    pub fn bench(c: &mut Criterion) {
-        let mut g = c.benchmark_group("single_metric_multicore_scalability");
-
-        for nthreads in [1, 4, 8] {
-            g.bench_with_input(
-                BenchmarkId::new("nthreads", nthreads),
-                &nthreads,
-                |b, nthreads| {
-                    b.iter_custom(|iters| {
-                        let barrier = std::sync::Barrier::new(*nthreads + 1);
-
-                        let metric = metrics::UIntGauge::new("testmetric", "testhelp").unwrap();
-
-                        std::thread::scope(|s| {
-                            for _ in 0..*nthreads {
-                                s.spawn(|| {
-                                    barrier.wait();
-                                    for _ in 0..iters {
-                                        metric.inc();
-                                    }
-                                    barrier.wait();
-                                });
-                            }
-                            barrier.wait();
-                            let start = std::time::Instant::now();
-                            barrier.wait();
-                            start.elapsed()
-                        })
-                    })
-                },
-            );
-        }
-        g.finish();
-    }
-}
-
-//
-// Demonstrates that even if we cache label value, the propagation of such a cached metric value
-// by Clone'ing it is a scalability bottleneck.
-// The reason is that it's an Arc internally and thus there's contention on the reference count atomics.
-//
-// We can avoid that by having long-lived references per thread (= indirection).
-//
-criterion_group!(
-    propagation_of_cached_label_value,
-    propagation_of_cached_label_value::bench_naive,
-    propagation_of_cached_label_value::bench_long_lived_reference_per_thread,
-);
-mod propagation_of_cached_label_value {
-    use std::sync::Arc;
-
-    use super::*;
-
-    pub fn bench_naive(c: &mut Criterion) {
-        let mut g = c.benchmark_group("propagation_of_cached_label_value__naive");
-
-        for nthreads in [1, 4, 8] {
-            g.bench_with_input(
-                BenchmarkId::new("nthreads", nthreads),
-                &nthreads,
-                |b, nthreads| {
-                    b.iter_custom(|iters| {
-                        let barrier = std::sync::Barrier::new(*nthreads + 1);
-
-                        let metric = metrics::UIntGauge::new("testmetric", "testhelp").unwrap();
-
-                        std::thread::scope(|s| {
-                            for _ in 0..*nthreads {
-                                s.spawn(|| {
-                                    barrier.wait();
-                                    for _ in 0..iters {
-                                        // propagating the metric means we'd clone it into the child RequestContext
-                                        let propagated = metric.clone();
-                                        // simulate some work
-                                        criterion::black_box(propagated);
-                                    }
-                                    barrier.wait();
-                                });
-                            }
-                            barrier.wait();
-                            let start = std::time::Instant::now();
-                            barrier.wait();
-                            start.elapsed()
-                        })
-                    })
-                },
-            );
-        }
-        g.finish();
-    }
-
-    pub fn bench_long_lived_reference_per_thread(c: &mut Criterion) {
-        let mut g =
-            c.benchmark_group("propagation_of_cached_label_value__long_lived_reference_per_thread");
-
-        for nthreads in [1, 4, 8] {
-            g.bench_with_input(
-                BenchmarkId::new("nthreads", nthreads),
-                &nthreads,
-                |b, nthreads| {
-                    b.iter_custom(|iters| {
-                        let barrier = std::sync::Barrier::new(*nthreads + 1);
-
-                        let metric = metrics::UIntGauge::new("testmetric", "testhelp").unwrap();
-
-                        std::thread::scope(|s| {
-                            for _ in 0..*nthreads {
-                                s.spawn(|| {
-                                    // This is the technique.
-                                    let this_threads_metric_reference = Arc::new(metric.clone());
-
-                                    barrier.wait();
-                                    for _ in 0..iters {
-                                        // propagating the metric means we'd clone it into the child RequestContext
-                                        let propagated = Arc::clone(&this_threads_metric_reference);
-                                        // simulate some work (include the pointer chase!)
-                                        criterion::black_box(&*propagated);
-                                    }
-                                    barrier.wait();
-                                });
-                            }
-                            barrier.wait();
-                            let start = std::time::Instant::now();
-                            barrier.wait();
-                            start.elapsed()
-                        })
-                    })
-                },
-            );
-        }
-    }
-}
-
-criterion_main!(
-    label_values,
-    single_metric_multicore_scalability,
-    propagation_of_cached_label_value
-);
-
-/*
-RUST_BACKTRACE=full cargo bench --bench bench_metrics --  --discard-baseline --noplot
-
-Results on an im4gn.2xlarge instance
-
-label_values__naive_usage/ntimelines/1 time:   [178.71 ns 178.74 ns 178.76 ns]
-label_values__naive_usage/ntimelines/4 time:   [532.94 ns 539.59 ns 546.31 ns]
-label_values__naive_usage/ntimelines/8 time:   [1.1082 µs 1.1109 µs 1.1135 µs]
-label_values__cache_label_values_lookup/ntimelines/1 time:   [6.4116 ns 6.4119 ns 6.4123 ns]
-label_values__cache_label_values_lookup/ntimelines/4 time:   [6.3482 ns 6.3819 ns 6.4079 ns]
-label_values__cache_label_values_lookup/ntimelines/8 time:   [6.4213 ns 6.5279 ns 6.6293 ns]
-single_metric_multicore_scalability/nthreads/1 time:   [6.0102 ns 6.0104 ns 6.0106 ns]
-single_metric_multicore_scalability/nthreads/4 time:   [38.127 ns 38.275 ns 38.416 ns]
-single_metric_multicore_scalability/nthreads/8 time:   [73.698 ns 74.882 ns 75.864 ns]
-propagation_of_cached_label_value__naive/nthreads/1 time:   [14.424 ns 14.425 ns 14.426 ns]
-propagation_of_cached_label_value__naive/nthreads/4 time:   [100.71 ns 102.53 ns 104.35 ns]
-propagation_of_cached_label_value__naive/nthreads/8 time:   [211.50 ns 214.44 ns 216.87 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/1 time:   [14.135 ns 14.147 ns 14.160 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/4 time:   [14.243 ns 14.255 ns 14.268 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/8 time:   [14.470 ns 14.682 ns 14.895 ns]
-
-Results on an i3en.3xlarge instance
-
-label_values__naive_usage/ntimelines/1      time:   [117.32 ns 117.53 ns 117.74 ns]
-label_values__naive_usage/ntimelines/4      time:   [736.58 ns 741.12 ns 745.61 ns]
-label_values__naive_usage/ntimelines/8      time:   [1.4513 µs 1.4596 µs 1.4665 µs]
-label_values__cache_label_values_lookup/ntimelines/1      time:   [8.0964 ns 8.0979 ns 8.0995 ns]
-label_values__cache_label_values_lookup/ntimelines/4      time:   [8.1620 ns 8.2912 ns 8.4491 ns]
-label_values__cache_label_values_lookup/ntimelines/8      time:   [14.148 ns 14.237 ns 14.324 ns]
-single_metric_multicore_scalability/nthreads/1      time:   [8.0993 ns 8.1013 ns 8.1046 ns]
-single_metric_multicore_scalability/nthreads/4      time:   [80.039 ns 80.672 ns 81.297 ns]
-single_metric_multicore_scalability/nthreads/8      time:   [153.58 ns 154.23 ns 154.90 ns]
-propagation_of_cached_label_value__naive/nthreads/1     time:   [13.924 ns 13.926 ns 13.928 ns]
-propagation_of_cached_label_value__naive/nthreads/4     time:   [143.66 ns 145.27 ns 146.59 ns]
-propagation_of_cached_label_value__naive/nthreads/8     time:   [296.51 ns 297.90 ns 299.30 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/1     time:   [14.013 ns 14.149 ns 14.308 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/4     time:   [14.311 ns 14.625 ns 14.984 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/8     time:   [25.981 ns 26.227 ns 26.476 ns]
-
-Results on an Standard L16s v3 (16 vcpus, 128 GiB memory)  Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz
-
-label_values__naive_usage/ntimelines/1      time:   [101.63 ns 101.84 ns 102.06 ns]
-label_values__naive_usage/ntimelines/4      time:   [417.55 ns 424.73 ns 432.63 ns]
-label_values__naive_usage/ntimelines/8      time:   [874.91 ns 889.51 ns 904.25 ns]
-label_values__cache_label_values_lookup/ntimelines/1      time:   [5.7724 ns 5.7760 ns 5.7804 ns]
-label_values__cache_label_values_lookup/ntimelines/4      time:   [7.8878 ns 7.9401 ns 8.0034 ns]
-label_values__cache_label_values_lookup/ntimelines/8      time:   [7.2621 ns 7.6354 ns 8.0337 ns]
-single_metric_multicore_scalability/nthreads/1      time:   [5.7710 ns 5.7744 ns 5.7785 ns]
-single_metric_multicore_scalability/nthreads/4      time:   [66.629 ns 66.994 ns 67.336 ns]
-single_metric_multicore_scalability/nthreads/8      time:   [130.85 ns 131.98 ns 132.91 ns]
-propagation_of_cached_label_value__naive/nthreads/1     time:   [11.540 ns 11.546 ns 11.553 ns]
-propagation_of_cached_label_value__naive/nthreads/4     time:   [131.22 ns 131.90 ns 132.56 ns]
-propagation_of_cached_label_value__naive/nthreads/8     time:   [260.99 ns 262.75 ns 264.26 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/1     time:   [11.544 ns 11.550 ns 11.557 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/4     time:   [11.568 ns 11.642 ns 11.763 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/8     time:   [13.416 ns 14.121 ns 14.886 ns
-
-Results on an M4 MAX MacBook Pro   Total Number of Cores:	14 (10 performance and 4 efficiency)
-
-label_values__naive_usage/ntimelines/1      time:   [52.711 ns 53.026 ns 53.381 ns]
-label_values__naive_usage/ntimelines/4      time:   [323.99 ns 330.40 ns 337.53 ns]
-label_values__naive_usage/ntimelines/8      time:   [1.1615 µs 1.1998 µs 1.2399 µs]
-label_values__cache_label_values_lookup/ntimelines/1      time:   [1.6635 ns 1.6715 ns 1.6809 ns]
-label_values__cache_label_values_lookup/ntimelines/4      time:   [1.7786 ns 1.7876 ns 1.8028 ns]
-label_values__cache_label_values_lookup/ntimelines/8      time:   [1.8195 ns 1.8371 ns 1.8665 ns]
-single_metric_multicore_scalability/nthreads/1      time:   [1.7764 ns 1.7909 ns 1.8079 ns]
-single_metric_multicore_scalability/nthreads/4      time:   [33.875 ns 34.868 ns 35.923 ns]
-single_metric_multicore_scalability/nthreads/8      time:   [226.85 ns 235.30 ns 244.18 ns]
-propagation_of_cached_label_value__naive/nthreads/1     time:   [3.4337 ns 3.4491 ns 3.4660 ns]
-propagation_of_cached_label_value__naive/nthreads/4     time:   [69.486 ns 71.937 ns 74.472 ns]
-propagation_of_cached_label_value__naive/nthreads/8     time:   [434.87 ns 456.47 ns 477.84 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/1     time:   [3.3767 ns 3.3974 ns 3.4220 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/4     time:   [3.6105 ns 4.2355 ns 5.1463 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/8     time:   [4.0889 ns 4.9714 ns 6.0779 ns]
-
-Results on a Hetzner AX102 AMD Ryzen 9 7950X3D 16-Core Processor
-
-label_values__naive_usage/ntimelines/1      time:   [64.510 ns 64.559 ns 64.610 ns]
-label_values__naive_usage/ntimelines/4      time:   [309.71 ns 326.09 ns 342.32 ns]
-label_values__naive_usage/ntimelines/8      time:   [776.92 ns 819.35 ns 856.93 ns]
-label_values__cache_label_values_lookup/ntimelines/1      time:   [1.2855 ns 1.2943 ns 1.3021 ns]
-label_values__cache_label_values_lookup/ntimelines/4      time:   [1.3865 ns 1.4139 ns 1.4441 ns]
-label_values__cache_label_values_lookup/ntimelines/8      time:   [1.5311 ns 1.5669 ns 1.6046 ns]
-single_metric_multicore_scalability/nthreads/1      time:   [1.1927 ns 1.1981 ns 1.2049 ns]
-single_metric_multicore_scalability/nthreads/4      time:   [24.346 ns 25.439 ns 26.634 ns]
-single_metric_multicore_scalability/nthreads/8      time:   [58.666 ns 60.137 ns 61.486 ns]
-propagation_of_cached_label_value__naive/nthreads/1     time:   [2.7067 ns 2.7238 ns 2.7402 ns]
-propagation_of_cached_label_value__naive/nthreads/4     time:   [62.723 ns 66.214 ns 69.787 ns]
-propagation_of_cached_label_value__naive/nthreads/8     time:   [164.24 ns 170.10 ns 175.68 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/1     time:   [2.2915 ns 2.2960 ns 2.3012 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/4     time:   [2.5726 ns 2.6158 ns 2.6624 ns]
-propagation_of_cached_label_value__long_lived_reference_per_thread/nthreads/8     time:   [2.7068 ns 2.8243 ns 2.9824 ns]
-
-*/

pageserver/client/src/mgmt_api.rs

@@ -7,7 +7,7 @@ use http_utils::error::HttpErrorBody;
 use pageserver_api::models::*;
 use pageserver_api::shard::TenantShardId;
 pub use reqwest::Body as ReqwestBody;
-use reqwest::{Certificate, IntoUrl, Method, StatusCode};
+use reqwest::{IntoUrl, Method, StatusCode};
 use utils::id::{TenantId, TimelineId};
 use utils::lsn::Lsn;
 
@@ -38,9 +38,6 @@ pub enum Error {
 
     #[error("Cancelled")]
     Cancelled,
-
-    #[error("create client: {0}{}", .0.source().map(|e| format!(": {e}")).unwrap_or_default())]
-    CreateClient(reqwest::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -72,17 +69,8 @@ pub enum ForceAwaitLogicalSize {
 }
 
 impl Client {
-    pub fn new(
-        mgmt_api_endpoint: String,
-        jwt: Option<&str>,
-        ssl_ca_cert: Option<Certificate>,
-    ) -> Result<Self> {
-        let mut http_client = reqwest::Client::builder();
-        if let Some(ssl_ca_cert) = ssl_ca_cert {
-            http_client = http_client.add_root_certificate(ssl_ca_cert);
-        }
-        let http_client = http_client.build().map_err(Error::CreateClient)?;
-        Ok(Self::from_client(http_client, mgmt_api_endpoint, jwt))
+    pub fn new(mgmt_api_endpoint: String, jwt: Option<&str>) -> Self {
+        Self::from_client(reqwest::Client::new(), mgmt_api_endpoint, jwt)
     }
 
     pub fn from_client(
@@ -113,10 +101,12 @@ impl Client {
         debug_assert!(path.starts_with('/'));
         let uri = format!("{}{}", self.mgmt_api_endpoint, path);
 
-        let mut req = self.client.request(Method::GET, uri);
-        if let Some(value) = &self.authorization_header {
-            req = req.header(reqwest::header::AUTHORIZATION, value);
-        }
+        let req = self.client.request(Method::GET, uri);
+        let req = if let Some(value) = &self.authorization_header {
+            req.header(reqwest::header::AUTHORIZATION, value)
+        } else {
+            req
+        };
         req.send().await.map_err(Error::ReceiveBody)
     }
 

pageserver/compaction/tests/tests.rs

@@ -12,7 +12,7 @@ pub(crate) fn setup_logging() {
             logging::TracingErrorLayerEnablement::EnableWithRustLogFilter,
             logging::Output::Stdout,
         )
-        .expect("Failed to init test logging");
+        .expect("Failed to init test logging")
     });
 }
 

pageserver/pagebench/Cargo.toml

@@ -15,7 +15,6 @@ hdrhistogram.workspace = true
 humantime.workspace = true
 humantime-serde.workspace = true
 rand.workspace = true
-reqwest.workspace=true
 serde.workspace = true
 serde_json.workspace = true
 tracing.workspace = true

pageserver/pagebench/src/cmd/aux_files.rs

@@ -36,8 +36,7 @@ async fn main_impl(args: Args) -> anyhow::Result<()> {
     let mgmt_api_client = Arc::new(pageserver_client::mgmt_api::Client::new(
         args.mgmt_api_endpoint.clone(),
         args.pageserver_jwt.as_deref(),
-        None, // TODO: support ssl_ca_file for https APIs in pagebench.
-    )?);
+    ));
 
     // discover targets
     let timelines: Vec<TenantTimelineId> = crate::util::cli::targets::discover(

pageserver/pagebench/src/cmd/basebackup.rs

@@ -77,8 +77,7 @@ async fn main_impl(
     let mgmt_api_client = Arc::new(pageserver_client::mgmt_api::Client::new(
         args.mgmt_api_endpoint.clone(),
         args.pageserver_jwt.as_deref(),
-        None, // TODO: support ssl_ca_file for https APIs in pagebench.
-    )?);
+    ));
 
     // discover targets
     let timelines: Vec<TenantTimelineId> = crate::util::cli::targets::discover(

pageserver/pagebench/src/cmd/getpage_latest_lsn.rs

@@ -125,8 +125,7 @@ async fn main_impl(
     let mgmt_api_client = Arc::new(pageserver_client::mgmt_api::Client::new(
         args.mgmt_api_endpoint.clone(),
         args.pageserver_jwt.as_deref(),
-        None, // TODO: support ssl_ca_file for https APIs in pagebench.
-    )?);
+    ));
 
     if let Some(engine_str) = &args.set_io_engine {
         mgmt_api_client.put_io_engine(engine_str).await?;

pageserver/pagebench/src/cmd/ondemand_download_churn.rs

@@ -83,8 +83,7 @@ async fn main_impl(args: Args) -> anyhow::Result<()> {
     let mgmt_api_client = Arc::new(pageserver_client::mgmt_api::Client::new(
         args.mgmt_api_endpoint.clone(),
         args.pageserver_jwt.as_deref(),
-        None, // TODO: support ssl_ca_file for https APIs in pagebench.
-    )?);
+    ));
 
     if let Some(engine_str) = &args.set_io_engine {
         mgmt_api_client.put_io_engine(engine_str).await?;

pageserver/pagebench/src/cmd/trigger_initial_size_calculation.rs

@@ -40,8 +40,7 @@ async fn main_impl(args: Args) -> anyhow::Result<()> {
     let mgmt_api_client = Arc::new(pageserver_client::mgmt_api::Client::new(
         args.mgmt_api_endpoint.clone(),
         args.pageserver_jwt.as_deref(),
-        None, // TODO: support ssl_ca_file for https APIs in pagebench.
-    )?);
+    ));
 
     // discover targets
     let timelines: Vec<TenantTimelineId> = crate::util::cli::targets::discover(

pageserver/src/bin/pageserver.rs

@@ -25,12 +25,11 @@ use pageserver::task_mgr::{
 };
 use pageserver::tenant::{TenantSharedResources, mgr, secondary};
 use pageserver::{
-    CancellableTask, ConsumptionMetricsTasks, HttpEndpointListener, HttpsEndpointListener, http,
-    page_cache, page_service, task_mgr, virtual_file,
+    CancellableTask, ConsumptionMetricsTasks, HttpEndpointListener, http, page_cache, page_service,
+    task_mgr, virtual_file,
 };
 use postgres_backend::AuthType;
 use remote_storage::GenericRemoteStorage;
-use rustls_pki_types::{CertificateDer, PrivateKeyDer};
 use tokio::signal::unix::SignalKind;
 use tokio::time::Instant;
 use tokio_util::sync::CancellationToken;
@@ -111,7 +110,6 @@ fn main() -> anyhow::Result<()> {
     } else {
         TracingErrorLayerEnablement::Disabled
     };
-
     logging::init(
         conf.log_format,
         tracing_error_layer_enablement,
@@ -345,15 +343,8 @@ fn start_pageserver(
     info!("Starting pageserver http handler on {http_addr}");
     let http_listener = tcp_listener::bind(http_addr)?;
 
-    let https_listener = match conf.listen_https_addr.as_ref() {
-        Some(https_addr) => {
-            info!("Starting pageserver https handler on {https_addr}");
-            Some(tcp_listener::bind(https_addr)?)
-        }
-        None => None,
-    };
-
     let pg_addr = &conf.listen_pg_addr;
+
     info!("Starting pageserver pg protocol handler on {pg_addr}");
     let pageserver_listener = tcp_listener::bind(pg_addr)?;
 
@@ -584,8 +575,9 @@ fn start_pageserver(
 
     // Start up the service to handle HTTP mgmt API request. We created the
     // listener earlier already.
-    let (http_endpoint_listener, https_endpoint_listener) = {
+    let http_endpoint_listener = {
         let _rt_guard = MGMT_REQUEST_RUNTIME.enter(); // for hyper
+        let cancel = CancellationToken::new();
 
         let router_state = Arc::new(
             http::routes::State::new(
@@ -600,51 +592,22 @@ fn start_pageserver(
             )
             .context("Failed to initialize router state")?,
         );
-
         let router = http::make_router(router_state, launch_ts, http_auth.clone())?
             .build()
             .map_err(|err| anyhow!(err))?;
+        let service = http_utils::RouterService::new(router).unwrap();
+        let server = hyper0::Server::from_tcp(http_listener)?
+            .serve(service)
+            .with_graceful_shutdown({
+                let cancel = cancel.clone();
+                async move { cancel.clone().cancelled().await }
+            });
 
-        let service =
-            Arc::new(http_utils::RequestServiceBuilder::new(router).map_err(|err| anyhow!(err))?);
-
-        let http_task = {
-            let server =
-                http_utils::server::Server::new(Arc::clone(&service), http_listener, None)?;
-            let cancel = CancellationToken::new();
-
-            let task = MGMT_REQUEST_RUNTIME.spawn(task_mgr::exit_on_panic_or_error(
-                "http endpoint listener",
-                server.serve(cancel.clone()),
-            ));
-            HttpEndpointListener(CancellableTask { task, cancel })
-        };
-
-        let https_task = match https_listener {
-            Some(https_listener) => {
-                let certs = load_certs(&conf.ssl_cert_file)?;
-                let key = load_private_key(&conf.ssl_key_file)?;
-
-                let server_config = rustls::ServerConfig::builder()
-                    .with_no_client_auth()
-                    .with_single_cert(certs, key)?;
-
-                let tls_acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(server_config));
-
-                let server =
-                    http_utils::server::Server::new(service, https_listener, Some(tls_acceptor))?;
-                let cancel = CancellationToken::new();
-
-                let task = MGMT_REQUEST_RUNTIME.spawn(task_mgr::exit_on_panic_or_error(
-                    "https endpoint listener",
-                    server.serve(cancel.clone()),
-                ));
-                Some(HttpsEndpointListener(CancellableTask { task, cancel }))
-            }
-            None => None,
-        };
-
-        (http_task, https_task)
+        let task = MGMT_REQUEST_RUNTIME.spawn(task_mgr::exit_on_panic_or_error(
+            "http endpoint listener",
+            server,
+        ));
+        HttpEndpointListener(CancellableTask { task, cancel })
     };
 
     let consumption_metrics_tasks = {
@@ -720,7 +683,6 @@ fn start_pageserver(
         shutdown_pageserver.cancel();
         pageserver::shutdown_pageserver(
             http_endpoint_listener,
-            https_endpoint_listener,
             page_service,
             consumption_metrics_tasks,
             disk_usage_eviction_task,
@@ -735,25 +697,6 @@ fn start_pageserver(
     })
 }
 
-fn load_certs(filename: &Utf8Path) -> std::io::Result<Vec<CertificateDer<'static>>> {
-    let file = std::fs::File::open(filename)?;
-    let mut reader = std::io::BufReader::new(file);
-
-    rustls_pemfile::certs(&mut reader).collect()
-}
-
-fn load_private_key(filename: &Utf8Path) -> anyhow::Result<PrivateKeyDer<'static>> {
-    let file = std::fs::File::open(filename)?;
-    let mut reader = std::io::BufReader::new(file);
-
-    let key = rustls_pemfile::private_key(&mut reader)?;
-
-    key.ok_or(anyhow::anyhow!(
-        "no private key found in {}",
-        filename.as_str(),
-    ))
-}
-
 async fn create_remote_storage_client(
     conf: &'static PageServerConf,
 ) -> anyhow::Result<GenericRemoteStorage> {

pageserver/src/config.rs

@@ -53,11 +53,6 @@ pub struct PageServerConf {
     pub listen_pg_addr: String,
     /// Example (default): 127.0.0.1:9898
     pub listen_http_addr: String,
-    /// Example: 127.0.0.1:9899
-    pub listen_https_addr: Option<String>,
-
-    pub ssl_key_file: Utf8PathBuf,
-    pub ssl_cert_file: Utf8PathBuf,
 
     /// Current availability zone. Used for traffic metrics.
     pub availability_zone: Option<String>,
@@ -322,9 +317,6 @@ impl PageServerConf {
         let pageserver_api::config::ConfigToml {
             listen_pg_addr,
             listen_http_addr,
-            listen_https_addr,
-            ssl_key_file,
-            ssl_cert_file,
             availability_zone,
             wait_lsn_timeout,
             wal_redo_timeout,
@@ -383,9 +375,6 @@ impl PageServerConf {
             // ------------------------------------------------------------
             listen_pg_addr,
             listen_http_addr,
-            listen_https_addr,
-            ssl_key_file,
-            ssl_cert_file,
             availability_zone,
             wait_lsn_timeout,
             wal_redo_timeout,
@@ -467,8 +456,8 @@ impl PageServerConf {
             no_sync: no_sync.unwrap_or(false),
             enable_read_path_debugging: enable_read_path_debugging.unwrap_or(false),
             validate_wal_contiguity: validate_wal_contiguity.unwrap_or(false),
-            load_previous_heatmap: load_previous_heatmap.unwrap_or(true),
-            generate_unarchival_heatmap: generate_unarchival_heatmap.unwrap_or(true),
+            load_previous_heatmap: load_previous_heatmap.unwrap_or(false),
+            generate_unarchival_heatmap: generate_unarchival_heatmap.unwrap_or(false),
         };
 
         // ------------------------------------------------------------

pageserver/src/context.rs

@@ -89,112 +89,16 @@
 //! [`RequestContext`] argument. Functions in the middle of the call chain
 //! only need to pass it on.
 
-use std::sync::Arc;
-
-use once_cell::sync::Lazy;
-use tracing::warn;
-use utils::{id::TimelineId, shard::TenantShardId};
-
-use crate::{
-    metrics::{StorageIoSizeMetrics, TimelineMetrics},
-    task_mgr::TaskKind,
-    tenant::Timeline,
-};
+use crate::task_mgr::TaskKind;
 
 // The main structure of this module, see module-level comment.
+#[derive(Debug)]
 pub struct RequestContext {
     task_kind: TaskKind,
     download_behavior: DownloadBehavior,
     access_stats_behavior: AccessStatsBehavior,
     page_content_kind: PageContentKind,
     read_path_debug: bool,
-    scope: Scope,
-}
-
-#[derive(Clone)]
-pub(crate) enum Scope {
-    Global {
-        io_size_metrics: &'static crate::metrics::StorageIoSizeMetrics,
-    },
-    SecondaryTenant {
-        io_size_metrics: &'static crate::metrics::StorageIoSizeMetrics,
-    },
-    SecondaryTimeline {
-        io_size_metrics: crate::metrics::StorageIoSizeMetrics,
-    },
-    Timeline {
-        // We wrap the `Arc<TimelineMetrics>`s inside another Arc to avoid child
-        // context creation contending for the ref counters of the Arc<TimelineMetrics>,
-        // which are shared among all tasks that operate on the timeline, especially
-        // concurrent page_service connections.
-        #[allow(clippy::redundant_allocation)]
-        arc_arc: Arc<Arc<TimelineMetrics>>,
-    },
-    #[cfg(test)]
-    UnitTest {
-        io_size_metrics: &'static crate::metrics::StorageIoSizeMetrics,
-    },
-}
-
-static GLOBAL_IO_SIZE_METRICS: Lazy<crate::metrics::StorageIoSizeMetrics> =
-    Lazy::new(|| crate::metrics::StorageIoSizeMetrics::new("*", "*", "*"));
-
-impl Scope {
-    pub(crate) fn new_global() -> Self {
-        Scope::Global {
-            io_size_metrics: &GLOBAL_IO_SIZE_METRICS,
-        }
-    }
-    /// NB: this allocates, so, use only at relatively long-lived roots, e.g., at start
-    /// of a compaction iteration.
-    pub(crate) fn new_timeline(timeline: &Timeline) -> Self {
-        Scope::Timeline {
-            arc_arc: Arc::new(Arc::clone(&timeline.metrics)),
-        }
-    }
-    pub(crate) fn new_page_service_pagestream(
-        timeline_handle: &crate::tenant::timeline::handle::Handle<
-            crate::page_service::TenantManagerTypes,
-        >,
-    ) -> Self {
-        Scope::Timeline {
-            arc_arc: Arc::clone(&timeline_handle.metrics),
-        }
-    }
-    pub(crate) fn new_secondary_timeline(
-        tenant_shard_id: &TenantShardId,
-        timeline_id: &TimelineId,
-    ) -> Self {
-        // TODO(https://github.com/neondatabase/neon/issues/11156): secondary timelines have no infrastructure for metrics lifecycle.
-
-        let tenant_id = tenant_shard_id.tenant_id.to_string();
-        let shard_id = tenant_shard_id.shard_slug().to_string();
-        let timeline_id = timeline_id.to_string();
-
-        let io_size_metrics =
-            crate::metrics::StorageIoSizeMetrics::new(&tenant_id, &shard_id, &timeline_id);
-        Scope::SecondaryTimeline { io_size_metrics }
-    }
-    pub(crate) fn new_secondary_tenant(_tenant_shard_id: &TenantShardId) -> Self {
-        // Before propagating metrics via RequestContext, the labels were inferred from file path.
-        // The only user of VirtualFile at tenant scope is the heatmap download & read.
-        // The inferred labels for the path of the heatmap file on local disk were that of the global metric (*,*,*).
-        // Thus, we do the same here, and extend that for anything secondary-tenant scoped.
-        //
-        // If we want to have (tenant_id, shard_id, '*') labels for secondary tenants in the future,
-        // we will need to think about the metric lifecycle, i.e., remove them during secondary tenant shutdown,
-        // like we do for attached timelines. (We don't have attached-tenant-scoped usage of VirtualFile
-        // at this point, so, we were able to completely side-step tenant-scoped stuff there).
-        Scope::SecondaryTenant {
-            io_size_metrics: &GLOBAL_IO_SIZE_METRICS,
-        }
-    }
-    #[cfg(test)]
-    pub(crate) fn new_unit_test() -> Self {
-        Scope::UnitTest {
-            io_size_metrics: &GLOBAL_IO_SIZE_METRICS,
-        }
-    }
 }
 
 /// The kind of access to the page cache.
@@ -253,7 +157,6 @@ impl RequestContextBuilder {
                 access_stats_behavior: AccessStatsBehavior::Update,
                 page_content_kind: PageContentKind::Unknown,
                 read_path_debug: false,
-                scope: Scope::new_global(),
             },
         }
     }
@@ -268,16 +171,10 @@ impl RequestContextBuilder {
                 access_stats_behavior: original.access_stats_behavior,
                 page_content_kind: original.page_content_kind,
                 read_path_debug: original.read_path_debug,
-                scope: original.scope.clone(),
             },
         }
     }
 
-    pub fn task_kind(mut self, k: TaskKind) -> Self {
-        self.inner.task_kind = k;
-        self
-    }
-
     /// Configure the DownloadBehavior of the context: whether to
     /// download missing layers, and/or warn on the download.
     pub fn download_behavior(mut self, b: DownloadBehavior) -> Self {
@@ -302,11 +199,6 @@ impl RequestContextBuilder {
         self
     }
 
-    pub(crate) fn scope(mut self, s: Scope) -> Self {
-        self.inner.scope = s;
-        self
-    }
-
     pub fn build(self) -> RequestContext {
         self.inner
     }
@@ -389,50 +281,7 @@ impl RequestContext {
     }
 
     fn child_impl(&self, task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self {
-        RequestContextBuilder::extend(self)
-            .task_kind(task_kind)
-            .download_behavior(download_behavior)
-            .build()
-    }
-
-    pub fn with_scope_timeline(&self, timeline: &Arc<Timeline>) -> Self {
-        RequestContextBuilder::extend(self)
-            .scope(Scope::new_timeline(timeline))
-            .build()
-    }
-
-    pub(crate) fn with_scope_page_service_pagestream(
-        &self,
-        timeline_handle: &crate::tenant::timeline::handle::Handle<
-            crate::page_service::TenantManagerTypes,
-        >,
-    ) -> Self {
-        RequestContextBuilder::extend(self)
-            .scope(Scope::new_page_service_pagestream(timeline_handle))
-            .build()
-    }
-
-    pub fn with_scope_secondary_timeline(
-        &self,
-        tenant_shard_id: &TenantShardId,
-        timeline_id: &TimelineId,
-    ) -> Self {
-        RequestContextBuilder::extend(self)
-            .scope(Scope::new_secondary_timeline(tenant_shard_id, timeline_id))
-            .build()
-    }
-
-    pub fn with_scope_secondary_tenant(&self, tenant_shard_id: &TenantShardId) -> Self {
-        RequestContextBuilder::extend(self)
-            .scope(Scope::new_secondary_tenant(tenant_shard_id))
-            .build()
-    }
-
-    #[cfg(test)]
-    pub fn with_scope_unit_test(&self) -> Self {
-        RequestContextBuilder::new(TaskKind::UnitTest)
-            .scope(Scope::new_unit_test())
-            .build()
+        Self::new(task_kind, download_behavior)
     }
 
     pub fn task_kind(&self) -> TaskKind {
@@ -454,38 +303,4 @@ impl RequestContext {
     pub(crate) fn read_path_debug(&self) -> bool {
         self.read_path_debug
     }
-
-    pub(crate) fn io_size_metrics(&self) -> &StorageIoSizeMetrics {
-        match &self.scope {
-            Scope::Global { io_size_metrics } => {
-                let is_unit_test = cfg!(test);
-                let is_regress_test_build = cfg!(feature = "testing");
-                if is_unit_test || is_regress_test_build {
-                    panic!("all VirtualFile instances are timeline-scoped");
-                } else {
-                    use once_cell::sync::Lazy;
-                    use std::sync::Mutex;
-                    use std::time::Duration;
-                    use utils::rate_limit::RateLimit;
-                    static LIMIT: Lazy<Mutex<RateLimit>> =
-                        Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(1))));
-                    let mut guard = LIMIT.lock().unwrap();
-                    guard.call2(|rate_limit_stats| {
-                        warn!(
-                            %rate_limit_stats,
-                            backtrace=%std::backtrace::Backtrace::force_capture(),
-                            "all VirtualFile instances are timeline-scoped",
-                        );
-                    });
-
-                    io_size_metrics
-                }
-            }
-            Scope::Timeline { arc_arc } => &arc_arc.storage_io_size,
-            Scope::SecondaryTimeline { io_size_metrics } => io_size_metrics,
-            Scope::SecondaryTenant { io_size_metrics } => io_size_metrics,
-            #[cfg(test)]
-            Scope::UnitTest { io_size_metrics } => io_size_metrics,
-        }
-    }
 }

pageserver/src/controller_upcall_client.rs

@@ -181,7 +181,7 @@ impl ControlPlaneGenerationsApi for ControllerUpcallClient {
                         listen_pg_port: m.postgres_port,
                         listen_http_addr: m.http_host,
                         listen_http_port: m.http_port,
-                        listen_https_port: m.https_port,
+                        listen_https_port: None, // TODO: Support https.
                         availability_zone_id: az_id.expect("Checked above"),
                     })
                 }

pageserver/src/http/openapi_spec.yml

@@ -1079,6 +1079,7 @@ components:
         - last_record_lsn
         - disk_consistent_lsn
         - state
+        - latest_gc_cutoff_lsn
       properties:
         timeline_id:
           type: string
@@ -1122,6 +1123,9 @@ components:
         min_readable_lsn:
           type: string
           format: hex
+        latest_gc_cutoff_lsn:
+          type: string
+          format: hex
         applied_gc_cutoff_lsn:
           type: string
           format: hex

pageserver/src/http/routes.rs

@@ -37,8 +37,7 @@ use pageserver_api::models::{
     TenantShardSplitResponse, TenantSorting, TenantState, TenantWaitLsnRequest,
     TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateRequestMode,
     TimelineCreateRequestModeImportPgdata, TimelineGcRequest, TimelineInfo,
-    TimelinePatchIndexPartRequest, TimelinesInfoAndOffloaded, TopTenantShardItem,
-    TopTenantShardsRequest, TopTenantShardsResponse,
+    TimelinesInfoAndOffloaded, TopTenantShardItem, TopTenantShardsRequest, TopTenantShardsResponse,
 };
 use pageserver_api::shard::{ShardCount, TenantShardId};
 use remote_storage::{DownloadError, GenericRemoteStorage, TimeTravelError};
@@ -55,7 +54,6 @@ use utils::id::{TenantId, TimelineId};
 use utils::lsn::Lsn;
 
 use crate::config::PageServerConf;
-use crate::context;
 use crate::context::{DownloadBehavior, RequestContext, RequestContextBuilder};
 use crate::deletion_queue::DeletionQueueClient;
 use crate::pgdatadir_mapping::LsnForTimestamp;
@@ -65,14 +63,12 @@ use crate::tenant::mgr::{
     GetActiveTenantError, GetTenantError, TenantManager, TenantMapError, TenantMapInsertError,
     TenantSlot, TenantSlotError, TenantSlotUpsertError, TenantStateError, UpsertLocationError,
 };
-use crate::tenant::remote_timeline_client::index::GcCompactionState;
 use crate::tenant::remote_timeline_client::{
     download_index_part, list_remote_tenant_shards, list_remote_timelines,
 };
 use crate::tenant::secondary::SecondaryController;
 use crate::tenant::size::ModelInputs;
 use crate::tenant::storage_layer::{IoConcurrency, LayerAccessStatsReset, LayerName};
-use crate::tenant::timeline::detach_ancestor::DetachBehavior;
 use crate::tenant::timeline::offload::{OffloadError, offload_timeline};
 use crate::tenant::timeline::{
     CompactFlags, CompactOptions, CompactRequest, CompactionError, Timeline, WaitLsnTimeout,
@@ -461,7 +457,10 @@ async fn build_timeline_info_common(
         initdb_lsn,
         last_record_lsn,
         prev_record_lsn: Some(timeline.get_prev_record_lsn()),
-        _unused: Default::default(), // Unused, for legacy decode only
+        // Externally, expose the lowest LSN that can be used to create a branch as the "GC cutoff", although internally
+        // we distinguish between the "planned" GC cutoff (PITR point) and the "latest" GC cutoff (where we
+        // actually trimmed data to), which can pass each other when PITR is changed.
+        latest_gc_cutoff_lsn: min_readable_lsn,
         min_readable_lsn,
         applied_gc_cutoff_lsn: *timeline.get_applied_gc_cutoff_lsn(),
         current_logical_size: current_logical_size.size_dont_care_about_accuracy(),
@@ -859,75 +858,6 @@ async fn timeline_archival_config_handler(
     json_response(StatusCode::OK, ())
 }
 
-/// This API is used to patch the index part of a timeline. You must ensure such patches are safe to apply. Use this API as an emergency
-/// measure only.
-///
-/// Some examples of safe patches:
-/// - Increase the gc_cutoff and gc_compaction_cutoff to a larger value in case of a bug that didn't bump the cutoff and cause read errors.
-/// - Force set the index part to use reldir v2 (migrating/migrated).
-///
-/// Some examples of unsafe patches:
-/// - Force set the index part from v2 to v1 (legacy). This will cause the code path to ignore anything written to the new keyspace and cause
-///   errors.
-/// - Decrease the gc_cutoff without validating the data really exists. It will cause read errors in the background.
-async fn timeline_patch_index_part_handler(
-    mut request: Request<Body>,
-    _cancel: CancellationToken,
-) -> Result<Response<Body>, ApiError> {
-    let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
-    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-
-    let request_data: TimelinePatchIndexPartRequest = json_request(&mut request).await?;
-    check_permission(&request, None)?; // require global permission for this request
-    let state = get_state(&request);
-
-    async {
-        let timeline =
-            active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
-                .await?;
-
-        if let Some(rel_size_migration) = request_data.rel_size_migration {
-            timeline
-                .update_rel_size_v2_status(rel_size_migration)
-                .map_err(ApiError::InternalServerError)?;
-        }
-
-        if let Some(gc_compaction_last_completed_lsn) =
-            request_data.gc_compaction_last_completed_lsn
-        {
-            timeline
-                .update_gc_compaction_state(GcCompactionState {
-                    last_completed_lsn: gc_compaction_last_completed_lsn,
-                })
-                .map_err(ApiError::InternalServerError)?;
-        }
-
-        if let Some(applied_gc_cutoff_lsn) = request_data.applied_gc_cutoff_lsn {
-            {
-                let guard = timeline.applied_gc_cutoff_lsn.lock_for_write();
-                guard.store_and_unlock(applied_gc_cutoff_lsn);
-            }
-        }
-
-        if request_data.force_index_update {
-            timeline
-                .remote_client
-                .force_schedule_index_upload()
-                .context("force schedule index upload")
-                .map_err(ApiError::InternalServerError)?;
-        }
-
-        Ok::<_, ApiError>(())
-    }
-    .instrument(info_span!("timeline_patch_index_part",
-                tenant_id = %tenant_shard_id.tenant_id,
-                shard_id = %tenant_shard_id.shard_slug(),
-                %timeline_id))
-    .await?;
-
-    json_response(StatusCode::OK, ())
-}
-
 async fn timeline_detail_handler(
     request: Request<Body>,
     _cancel: CancellationToken,
@@ -952,13 +882,12 @@ async fn timeline_detail_handler(
         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
 
         let timeline = tenant.get_timeline(timeline_id, false)?;
-        let ctx = &ctx.with_scope_timeline(&timeline);
 
         let timeline_info = build_timeline_info(
             &timeline,
             include_non_incremental_logical_size.unwrap_or(false),
             force_await_initial_logical_size.unwrap_or(false),
-            ctx,
+            &ctx,
         )
         .await
         .context("get local timeline info")
@@ -1002,8 +931,7 @@ async fn get_lsn_by_timestamp_handler(
     let timeline =
         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
             .await?;
-    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
-        .with_scope_timeline(&timeline);
+    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     let result = timeline
         .find_lsn_for_timestamp(timestamp_pg, &cancel, &ctx)
         .await?;
@@ -1075,8 +1003,7 @@ async fn get_timestamp_of_lsn_handler(
     let timeline =
         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
             .await?;
-    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
-        .with_scope_timeline(&timeline);
+    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     let result = timeline.get_timestamp_for_lsn(lsn, &ctx).await?;
 
     match result {
@@ -1431,8 +1358,7 @@ async fn timeline_layer_scan_disposable_keys(
         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
             .await?;
 
-    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
-        .with_scope_timeline(&timeline);
+    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
 
     let guard = timeline.layers.read().await;
     let Some(layer) = guard.try_get_from_key(&layer_name.clone().into()) else {
@@ -1518,8 +1444,7 @@ async fn timeline_download_heatmap_layers_handler(
     let timeline =
         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
             .await?;
-    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
-        .with_scope_timeline(&timeline);
+    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
 
     let max_concurrency = get_config(&request)
         .remote_storage_config
@@ -1567,8 +1492,7 @@ async fn layer_download_handler(
     let timeline =
         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
             .await?;
-    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
-        .with_scope_timeline(&timeline);
+    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     let downloaded = timeline
         .download_layer(&layer_name, &ctx)
         .await
@@ -2304,8 +2228,8 @@ async fn timeline_compact_handler(
         .unwrap_or(false);
 
     async {
+        let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
-        let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
         if scheduled {
             let tenant = state
                 .tenant_manager
@@ -2412,8 +2336,8 @@ async fn timeline_checkpoint_handler(
         parse_query_param::<_, bool>(&request, "wait_until_uploaded")?.unwrap_or(false);
 
     async {
+        let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
-        let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
         if wait_until_flushed {
             timeline.freeze_and_flush().await
         } else {
@@ -2468,8 +2392,7 @@ async fn timeline_download_remote_layers_handler_post(
     let timeline =
         active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
             .await?;
-    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download)
-        .with_scope_timeline(&timeline);
+    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
     match timeline.spawn_download_all_remote_layers(body, &ctx).await {
         Ok(st) => json_response(StatusCode::ACCEPTED, st),
         Err(st) => json_response(StatusCode::CONFLICT, st),
@@ -2506,8 +2429,6 @@ async fn timeline_detach_ancestor_handler(
     let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
     let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    let behavior: Option<DetachBehavior> = parse_query_param(&request, "detach_behavior")?;
-    let behavior = behavior.unwrap_or_default();
 
     let span = tracing::info_span!("detach_ancestor", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id);
 
@@ -2554,10 +2475,9 @@ async fn timeline_detach_ancestor_handler(
         tracing::info!("all timeline upload queues are drained");
 
         let timeline = tenant.get_timeline(timeline_id, true)?;
-        let ctx = &ctx.with_scope_timeline(&timeline);
 
         let progress = timeline
-            .prepare_to_detach_from_ancestor(&tenant, options, behavior, ctx)
+            .prepare_to_detach_from_ancestor(&tenant, options, ctx)
             .await?;
 
         // uncomment to allow early as possible Tenant::drop
@@ -2572,7 +2492,6 @@ async fn timeline_detach_ancestor_handler(
                         tenant_shard_id,
                         timeline_id,
                         prepared,
-                        behavior,
                         attempt,
                         ctx,
                     )
@@ -2662,9 +2581,8 @@ async fn getpage_at_lsn_handler_inner(
     async {
         let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
         // Enable read path debugging
+        let ctx = RequestContextBuilder::extend(&ctx).read_path_debug(true).build();
         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
-        let ctx = RequestContextBuilder::extend(&ctx).read_path_debug(true)
-        .scope(context::Scope::new_timeline(&timeline)).build();
 
         // Use last_record_lsn if no lsn is provided
         let lsn = lsn.unwrap_or_else(|| timeline.get_last_record_lsn());
@@ -2698,8 +2616,8 @@ async fn timeline_collect_keyspace(
     let at_lsn: Option<Lsn> = parse_query_param(&request, "at_lsn")?;
 
     async {
+        let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download);
         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id).await?;
-        let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download).with_scope_timeline(&timeline);
         let at_lsn = at_lsn.unwrap_or_else(|| timeline.get_last_record_lsn());
         let (dense_ks, sparse_ks) = timeline
             .collect_keyspace(at_lsn, &ctx)
@@ -3224,7 +3142,6 @@ async fn post_top_tenants(
         match order_by {
             TenantSorting::ResidentSize => sizes.resident_size,
             TenantSorting::MaxLogicalSize => sizes.max_logical_size,
-            TenantSorting::MaxLogicalSizePerShard => sizes.max_logical_size_per_shard,
         }
     }
 
@@ -3337,7 +3254,7 @@ async fn put_tenant_timeline_import_basebackup(
 
         tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
 
-        let (timeline, timeline_ctx) = tenant
+        let timeline = tenant
             .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx)
             .map_err(ApiError::InternalServerError)
             .await?;
@@ -3356,13 +3273,7 @@ async fn put_tenant_timeline_import_basebackup(
         info!("importing basebackup");
 
         timeline
-            .import_basebackup_from_tar(
-                tenant.clone(),
-                &mut body,
-                base_lsn,
-                broker_client,
-                &timeline_ctx,
-            )
+            .import_basebackup_from_tar(tenant.clone(), &mut body, base_lsn, broker_client, &ctx)
             .await
             .map_err(ApiError::InternalServerError)?;
 
@@ -3402,7 +3313,6 @@ async fn put_tenant_timeline_import_wal(
         let state = get_state(&request);
 
         let timeline = active_timeline_of_active_tenant(&state.tenant_manager, TenantShardId::unsharded(tenant_id), timeline_id).await?;
-        let ctx = RequestContextBuilder::extend(&ctx).scope(context::Scope::new_timeline(&timeline)).build();
 
         let mut body = StreamReader::new(request.into_body().map(|res| {
             res.map_err(|error| {
@@ -3719,10 +3629,6 @@ pub fn make_router(
             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/get_timestamp_of_lsn",
             |r| api_handler(r, get_timestamp_of_lsn_handler),
         )
-        .post(
-            "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/patch_index_part",
-            |r| api_handler(r, timeline_patch_index_part_handler),
-        )
         .post(
             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/lsn_lease",
             |r| api_handler(r, lsn_lease_handler),

pageserver/src/lib.rs

@@ -64,7 +64,6 @@ pub struct CancellableTask {
     pub cancel: CancellationToken,
 }
 pub struct HttpEndpointListener(pub CancellableTask);
-pub struct HttpsEndpointListener(pub CancellableTask);
 pub struct ConsumptionMetricsTasks(pub CancellableTask);
 pub struct DiskUsageEvictionTask(pub CancellableTask);
 impl CancellableTask {
@@ -78,7 +77,6 @@ impl CancellableTask {
 #[allow(clippy::too_many_arguments)]
 pub async fn shutdown_pageserver(
     http_listener: HttpEndpointListener,
-    https_listener: Option<HttpsEndpointListener>,
     page_service: page_service::Listener,
     consumption_metrics_worker: ConsumptionMetricsTasks,
     disk_usage_eviction_task: Option<DiskUsageEvictionTask>,
@@ -215,15 +213,6 @@ pub async fn shutdown_pageserver(
     )
     .await;
 
-    if let Some(https_listener) = https_listener {
-        timed(
-            https_listener.0.shutdown(),
-            "shutdown https",
-            Duration::from_secs(1),
-        )
-        .await;
-    }
-
     // Shut down the HTTP endpoint last, so that you can still check the server's
     // status while it's shutting down.
     // FIXME: We should probably stop accepting commands like attach/detach earlier.

pageserver/src/metrics.rs

@@ -1227,24 +1227,11 @@ impl StorageIoTime {
 
 pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
 
-#[derive(Clone, Copy)]
-#[repr(usize)]
-enum StorageIoSizeOperation {
-    Read,
-    Write,
-}
-
-impl StorageIoSizeOperation {
-    const VARIANTS: &'static [&'static str] = &["read", "write"];
-
-    fn as_str(&self) -> &'static str {
-        Self::VARIANTS[*self as usize]
-    }
-}
+const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
 
 // Needed for the https://neonprod.grafana.net/d/5uK9tHL4k/picking-tenant-for-relocation?orgId=1
-static STORAGE_IO_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
-    register_uint_gauge_vec!(
+pub(crate) static STORAGE_IO_SIZE: Lazy<IntGaugeVec> = Lazy::new(|| {
+    register_int_gauge_vec!(
         "pageserver_io_operations_bytes_total",
         "Total amount of bytes read/written in IO operations",
         &["operation", "tenant_id", "shard_id", "timeline_id"]
@@ -1252,34 +1239,6 @@ static STORAGE_IO_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
     .expect("failed to define a metric")
 });
 
-#[derive(Clone, Debug)]
-pub(crate) struct StorageIoSizeMetrics {
-    pub read: UIntGauge,
-    pub write: UIntGauge,
-}
-
-impl StorageIoSizeMetrics {
-    pub(crate) fn new(tenant_id: &str, shard_id: &str, timeline_id: &str) -> Self {
-        let read = STORAGE_IO_SIZE
-            .get_metric_with_label_values(&[
-                StorageIoSizeOperation::Read.as_str(),
-                tenant_id,
-                shard_id,
-                timeline_id,
-            ])
-            .unwrap();
-        let write = STORAGE_IO_SIZE
-            .get_metric_with_label_values(&[
-                StorageIoSizeOperation::Write.as_str(),
-                tenant_id,
-                shard_id,
-                timeline_id,
-            ])
-            .unwrap();
-        Self { read, write }
-    }
-}
-
 #[cfg(not(test))]
 pub(crate) mod virtual_file_descriptor_cache {
     use super::*;
@@ -2862,7 +2821,6 @@ pub(crate) struct TimelineMetrics {
     /// Number of valid LSN leases.
     pub valid_lsn_lease_count_gauge: UIntGauge,
     pub wal_records_received: IntCounter,
-    pub storage_io_size: StorageIoSizeMetrics,
     shutdown: std::sync::atomic::AtomicBool,
 }
 
@@ -2998,8 +2956,6 @@ impl TimelineMetrics {
             .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
             .unwrap();
 
-        let storage_io_size = StorageIoSizeMetrics::new(&tenant_id, &shard_id, &timeline_id);
-
         TimelineMetrics {
             tenant_id,
             shard_id,
@@ -3029,7 +2985,6 @@ impl TimelineMetrics {
             evictions_with_low_residence_duration: std::sync::RwLock::new(
                 evictions_with_low_residence_duration,
             ),
-            storage_io_size,
             valid_lsn_lease_count_gauge,
             wal_records_received,
             shutdown: std::sync::atomic::AtomicBool::default(),
@@ -3220,7 +3175,7 @@ impl TimelineMetrics {
             ]);
         }
 
-        for op in StorageIoSizeOperation::VARIANTS {
+        for op in STORAGE_IO_SIZE_OPERATIONS {
             let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]);
         }
 

pageserver/src/page_service.rs

@@ -56,7 +56,6 @@ use crate::config::PageServerConf;
 use crate::context::{DownloadBehavior, RequestContext};
 use crate::metrics::{
     self, COMPUTE_COMMANDS_COUNTERS, ComputeCommandKind, LIVE_CONNECTIONS, SmgrOpTimer,
-    TimelineMetrics,
 };
 use crate::pgdatadir_mapping::Version;
 use crate::span::{
@@ -424,9 +423,6 @@ impl timeline::handle::Types for TenantManagerTypes {
 
 pub(crate) struct TenantManagerCacheItem {
     pub(crate) timeline: Arc<Timeline>,
-    // allow() for cheap propagation through RequestContext inside a task
-    #[allow(clippy::redundant_allocation)]
-    pub(crate) metrics: Arc<Arc<TimelineMetrics>>,
     #[allow(dead_code)] // we store it to keep the gate open
     pub(crate) gate_guard: GateGuard,
 }
@@ -510,11 +506,8 @@ impl timeline::handle::TenantManager<TenantManagerTypes> for TenantManagerWrappe
             }
         };
 
-        let metrics = Arc::new(Arc::clone(&timeline.metrics));
-
         Ok(TenantManagerCacheItem {
             timeline,
-            metrics,
             gate_guard,
         })
     }
@@ -1245,14 +1238,6 @@ impl PageServerHandler {
         ),
         QueryError,
     > {
-        macro_rules! upgrade_handle_and_set_context {
-            ($shard:ident) => {{
-                let weak_handle = &$shard;
-                let handle = weak_handle.upgrade()?;
-                let ctx = ctx.with_scope_page_service_pagestream(&handle);
-                (handle, ctx)
-            }};
-        }
         Ok(match batch {
             BatchedFeMessage::Exists {
                 span,
@@ -1261,10 +1246,9 @@ impl PageServerHandler {
                 req,
             } => {
                 fail::fail_point!("ps::handle-pagerequest-message::exists");
-                let (shard, ctx) = upgrade_handle_and_set_context!(shard);
                 (
                     vec![
-                        self.handle_get_rel_exists_request(&shard, &req, &ctx)
+                        self.handle_get_rel_exists_request(&*shard.upgrade()?, &req, ctx)
                             .instrument(span.clone())
                             .await
                             .map(|msg| (msg, timer))
@@ -1280,10 +1264,9 @@ impl PageServerHandler {
                 req,
             } => {
                 fail::fail_point!("ps::handle-pagerequest-message::nblocks");
-                let (shard, ctx) = upgrade_handle_and_set_context!(shard);
                 (
                     vec![
-                        self.handle_get_nblocks_request(&shard, &req, &ctx)
+                        self.handle_get_nblocks_request(&*shard.upgrade()?, &req, ctx)
                             .instrument(span.clone())
                             .await
                             .map(|msg| (msg, timer))
@@ -1299,18 +1282,17 @@ impl PageServerHandler {
                 pages,
             } => {
                 fail::fail_point!("ps::handle-pagerequest-message::getpage");
-                let (shard, ctx) = upgrade_handle_and_set_context!(shard);
                 (
                     {
                         let npages = pages.len();
                         trace!(npages, "handling getpage request");
                         let res = self
                             .handle_get_page_at_lsn_request_batched(
-                                &shard,
+                                &*shard.upgrade()?,
                                 effective_request_lsn,
                                 pages,
                                 io_concurrency,
-                                &ctx,
+                                ctx,
                             )
                             .instrument(span.clone())
                             .await;
@@ -1327,10 +1309,9 @@ impl PageServerHandler {
                 req,
             } => {
                 fail::fail_point!("ps::handle-pagerequest-message::dbsize");
-                let (shard, ctx) = upgrade_handle_and_set_context!(shard);
                 (
                     vec![
-                        self.handle_db_size_request(&shard, &req, &ctx)
+                        self.handle_db_size_request(&*shard.upgrade()?, &req, ctx)
                             .instrument(span.clone())
                             .await
                             .map(|msg| (msg, timer))
@@ -1346,10 +1327,9 @@ impl PageServerHandler {
                 req,
             } => {
                 fail::fail_point!("ps::handle-pagerequest-message::slrusegment");
-                let (shard, ctx) = upgrade_handle_and_set_context!(shard);
                 (
                     vec![
-                        self.handle_get_slru_segment_request(&shard, &req, &ctx)
+                        self.handle_get_slru_segment_request(&*shard.upgrade()?, &req, ctx)
                             .instrument(span.clone())
                             .await
                             .map(|msg| (msg, timer))
@@ -1365,13 +1345,12 @@ impl PageServerHandler {
                 requests,
             } => {
                 fail::fail_point!("ps::handle-pagerequest-message::test");
-                let (shard, ctx) = upgrade_handle_and_set_context!(shard);
                 (
                     {
                         let npages = requests.len();
                         trace!(npages, "handling getpage request");
                         let res = self
-                            .handle_test_request_batch(&shard, requests, &ctx)
+                            .handle_test_request_batch(&*shard.upgrade()?, requests, ctx)
                             .instrument(span.clone())
                             .await;
                         assert_eq!(res.len(), npages);
@@ -2147,7 +2126,6 @@ impl PageServerHandler {
             .get(tenant_id, timeline_id, ShardSelector::Zero)
             .await?;
         set_tracing_field_shard_id(&timeline);
-        let ctx = ctx.with_scope_timeline(&timeline);
 
         if timeline.is_archived() == Some(true) {
             tracing::info!(
@@ -2165,7 +2143,7 @@ impl PageServerHandler {
                     lsn,
                     crate::tenant::timeline::WaitLsnWaiter::PageService,
                     crate::tenant::timeline::WaitLsnTimeout::Default,
-                    &ctx,
+                    ctx,
                 )
                 .await?;
             timeline
@@ -2191,7 +2169,7 @@ impl PageServerHandler {
                 prev_lsn,
                 full_backup,
                 replica,
-                &ctx,
+                ctx,
             )
             .await
             .map_err(map_basebackup_error)?;
@@ -2214,7 +2192,7 @@ impl PageServerHandler {
                     prev_lsn,
                     full_backup,
                     replica,
-                    &ctx,
+                    ctx,
                 )
                 .await
                 .map_err(map_basebackup_error)?;
@@ -2231,7 +2209,7 @@ impl PageServerHandler {
                     prev_lsn,
                     full_backup,
                     replica,
-                    &ctx,
+                    ctx,
                 )
                 .await
                 .map_err(map_basebackup_error)?;

pageserver/src/pgdatadir_mapping.rs

@@ -2758,7 +2758,7 @@ mod tests {
             TimelineId::from_array(hex!("11223344556677881122334455667788"));
 
         let (tenant, ctx) = harness.load().await;
-        let (tline, ctx) = tenant
+        let tline = tenant
             .create_empty_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
             .await?;
         let tline = tline.raw_timeline().unwrap();

pageserver/src/tenant.rs

@@ -77,8 +77,6 @@ use self::timeline::{
     EvictionTaskTenantState, GcCutoffs, TimelineDeleteProgress, TimelineResources, WaitLsnError,
 };
 use crate::config::PageServerConf;
-use crate::context;
-use crate::context::RequestContextBuilder;
 use crate::context::{DownloadBehavior, RequestContext};
 use crate::deletion_queue::{DeletionQueueClient, DeletionQueueError};
 use crate::l0_flush::L0FlushGlobalState;
@@ -1116,7 +1114,7 @@ impl Tenant {
             }
         };
 
-        let (timeline, timeline_ctx) = self.create_timeline_struct(
+        let timeline = self.create_timeline_struct(
             timeline_id,
             &metadata,
             previous_heatmap,
@@ -1126,7 +1124,6 @@ impl Tenant {
             idempotency.clone(),
             index_part.gc_compaction.clone(),
             index_part.rel_size_migration.clone(),
-            ctx,
         )?;
         let disk_consistent_lsn = timeline.get_disk_consistent_lsn();
         anyhow::ensure!(
@@ -1260,7 +1257,7 @@ impl Tenant {
                         match activate {
                             ActivateTimelineArgs::Yes { broker_client } => {
                                 info!("activating timeline after reload from pgdata import task");
-                                timeline.activate(self.clone(), broker_client, None, &timeline_ctx);
+                                timeline.activate(self.clone(), broker_client, None, ctx);
                             }
                             ActivateTimelineArgs::No => (),
                         }
@@ -1768,7 +1765,6 @@ impl Tenant {
                         import_pgdata,
                         ActivateTimelineArgs::No,
                         guard,
-                        ctx.detached_child(TaskKind::ImportPgdata, DownloadBehavior::Warn),
                     ));
                 }
             }
@@ -1786,7 +1782,6 @@ impl Tenant {
                 timeline_id,
                 &index_part.metadata,
                 remote_timeline_client,
-                ctx,
             )
             .instrument(tracing::info_span!("timeline_delete", %timeline_id))
             .await
@@ -2224,7 +2219,7 @@ impl Tenant {
                 self.clone(),
                 broker_client.clone(),
                 background_jobs_can_start,
-                &ctx.with_scope_timeline(&timeline),
+                &ctx,
             );
         }
 
@@ -2421,8 +2416,8 @@ impl Tenant {
         new_timeline_id: TimelineId,
         initdb_lsn: Lsn,
         pg_version: u32,
-        ctx: &RequestContext,
-    ) -> anyhow::Result<(UninitializedTimeline, RequestContext)> {
+        _ctx: &RequestContext,
+    ) -> anyhow::Result<UninitializedTimeline> {
         anyhow::ensure!(
             self.is_active(),
             "Cannot create empty timelines on inactive tenant"
@@ -2457,7 +2452,6 @@ impl Tenant {
             initdb_lsn,
             None,
             None,
-            ctx,
         )
         .await
     }
@@ -2475,7 +2469,7 @@ impl Tenant {
         pg_version: u32,
         ctx: &RequestContext,
     ) -> anyhow::Result<Arc<Timeline>> {
-        let (uninit_tl, ctx) = self
+        let uninit_tl = self
             .create_empty_timeline(new_timeline_id, initdb_lsn, pg_version, ctx)
             .await?;
         let tline = uninit_tl.raw_timeline().expect("we just created it");
@@ -2487,7 +2481,7 @@ impl Tenant {
             .init_empty_test_timeline()
             .context("init_empty_test_timeline")?;
         modification
-            .commit(&ctx)
+            .commit(ctx)
             .await
             .context("commit init_empty_test_timeline modification")?;
 
@@ -2705,12 +2699,7 @@ impl Tenant {
         // doing stuff before the IndexPart is durable in S3, which is done by the previous section.
         let activated_timeline = match result {
             CreateTimelineResult::Created(timeline) => {
-                timeline.activate(
-                    self.clone(),
-                    broker_client,
-                    None,
-                    &ctx.with_scope_timeline(&timeline),
-                );
+                timeline.activate(self.clone(), broker_client, None, ctx);
                 timeline
             }
             CreateTimelineResult::Idempotent(timeline) => {
@@ -2772,9 +2761,10 @@ impl Tenant {
             }
         };
 
-        let (mut uninit_timeline, timeline_ctx) = {
+        let mut uninit_timeline = {
             let this = &self;
             let initdb_lsn = Lsn(0);
+            let _ctx = ctx;
             async move {
                 let new_metadata = TimelineMetadata::new(
                     // Initialize disk_consistent LSN to 0, The caller must import some data to
@@ -2794,7 +2784,6 @@ impl Tenant {
                     initdb_lsn,
                     None,
                     None,
-                    ctx,
                 )
                 .await
             }
@@ -2824,7 +2813,6 @@ impl Tenant {
             index_part,
             activate,
             timeline_create_guard,
-            timeline_ctx.detached_child(TaskKind::ImportPgdata, DownloadBehavior::Warn),
         ));
 
         // NB: the timeline doesn't exist in self.timelines at this point
@@ -2838,7 +2826,6 @@ impl Tenant {
         index_part: import_pgdata::index_part_format::Root,
         activate: ActivateTimelineArgs,
         timeline_create_guard: TimelineCreateGuard,
-        ctx: RequestContext,
     ) {
         debug_assert_current_span_has_tenant_and_timeline_id();
         info!("starting");
@@ -2850,7 +2837,6 @@ impl Tenant {
                 index_part,
                 activate,
                 timeline_create_guard,
-                ctx,
             )
             .await;
         if let Err(err) = &res {
@@ -2866,8 +2852,9 @@ impl Tenant {
         index_part: import_pgdata::index_part_format::Root,
         activate: ActivateTimelineArgs,
         timeline_create_guard: TimelineCreateGuard,
-        ctx: RequestContext,
     ) -> Result<(), anyhow::Error> {
+        let ctx = RequestContext::new(TaskKind::ImportPgdata, DownloadBehavior::Warn);
+
         info!("importing pgdata");
         import_pgdata::doit(&timeline, index_part, &ctx, self.cancel.clone())
             .await
@@ -3076,7 +3063,6 @@ impl Tenant {
 
             let mut has_pending_l0 = false;
             for timeline in compact_l0 {
-                let ctx = &ctx.with_scope_timeline(&timeline);
                 let outcome = timeline
                     .compact(cancel, CompactFlags::OnlyL0Compaction.into(), ctx)
                     .instrument(info_span!("compact_timeline", timeline_id = %timeline.timeline_id))
@@ -3110,7 +3096,6 @@ impl Tenant {
             if !timeline.is_active() {
                 continue;
             }
-            let ctx = &ctx.with_scope_timeline(&timeline);
 
             let mut outcome = timeline
                 .compact(cancel, EnumSet::default(), ctx)
@@ -3336,7 +3321,7 @@ impl Tenant {
                     self.clone(),
                     broker_client.clone(),
                     background_jobs_can_start,
-                    &ctx.with_scope_timeline(timeline),
+                    ctx,
                 );
                 activated_timelines += 1;
             }
@@ -3842,7 +3827,6 @@ impl Tenant {
             resident_size: 0,
             physical_size: 0,
             max_logical_size: 0,
-            max_logical_size_per_shard: 0,
         };
 
         for timeline in self.timelines.lock().unwrap().values() {
@@ -3859,10 +3843,6 @@ impl Tenant {
             );
         }
 
-        result.max_logical_size_per_shard = result
-            .max_logical_size
-            .div_ceil(self.tenant_shard_id.shard_count.count() as u64);
-
         result
     }
 }
@@ -4156,8 +4136,7 @@ impl Tenant {
         create_idempotency: CreateTimelineIdempotency,
         gc_compaction_state: Option<GcCompactionState>,
         rel_size_v2_status: Option<RelSizeMigration>,
-        ctx: &RequestContext,
-    ) -> anyhow::Result<(Arc<Timeline>, RequestContext)> {
+    ) -> anyhow::Result<Arc<Timeline>> {
         let state = match cause {
             CreateTimelineCause::Load => {
                 let ancestor_id = new_metadata.ancestor_timeline();
@@ -4193,11 +4172,7 @@ impl Tenant {
             self.cancel.child_token(),
         );
 
-        let timeline_ctx = RequestContextBuilder::extend(ctx)
-            .scope(context::Scope::new_timeline(&timeline))
-            .build();
-
-        Ok((timeline, timeline_ctx))
+        Ok(timeline)
     }
 
     /// [`Tenant::shutdown`] must be called before dropping the returned [`Tenant`] object
@@ -4613,7 +4588,6 @@ impl Tenant {
         // Ensures all timelines use the same start time when computing the time cutoff.
         let now_ts_for_pitr_calc = SystemTime::now();
         for timeline in timelines.iter() {
-            let ctx = &ctx.with_scope_timeline(timeline);
             let cutoff = timeline
                 .get_last_record_lsn()
                 .checked_sub(horizon)
@@ -4787,7 +4761,7 @@ impl Tenant {
         src_timeline: &Arc<Timeline>,
         dst_id: TimelineId,
         start_lsn: Option<Lsn>,
-        ctx: &RequestContext,
+        _ctx: &RequestContext,
     ) -> Result<CreateTimelineResult, CreateTimelineError> {
         let src_id = src_timeline.timeline_id;
 
@@ -4890,7 +4864,7 @@ impl Tenant {
             src_timeline.pg_version,
         );
 
-        let (uninitialized_timeline, _timeline_ctx) = self
+        let uninitialized_timeline = self
             .prepare_new_timeline(
                 dst_id,
                 &metadata,
@@ -4898,7 +4872,6 @@ impl Tenant {
                 start_lsn + 1,
                 Some(Arc::clone(src_timeline)),
                 Some(src_timeline.get_rel_size_v2_status()),
-                ctx,
             )
             .await?;
 
@@ -5165,7 +5138,7 @@ impl Tenant {
             pgdata_lsn,
             pg_version,
         );
-        let (mut raw_timeline, timeline_ctx) = self
+        let mut raw_timeline = self
             .prepare_new_timeline(
                 timeline_id,
                 &new_metadata,
@@ -5173,7 +5146,6 @@ impl Tenant {
                 pgdata_lsn,
                 None,
                 None,
-                ctx,
             )
             .await?;
 
@@ -5184,7 +5156,7 @@ impl Tenant {
                     &unfinished_timeline,
                     &pgdata_path,
                     pgdata_lsn,
-                    &timeline_ctx,
+                    ctx,
                 )
                 .await
                 .with_context(|| {
@@ -5245,7 +5217,6 @@ impl Tenant {
     /// An empty layer map is initialized, and new data and WAL can be imported starting
     /// at 'disk_consistent_lsn'. After any initial data has been imported, call
     /// `finish_creation` to insert the Timeline into the timelines map.
-    #[allow(clippy::too_many_arguments)]
     async fn prepare_new_timeline<'a>(
         &'a self,
         new_timeline_id: TimelineId,
@@ -5254,8 +5225,7 @@ impl Tenant {
         start_lsn: Lsn,
         ancestor: Option<Arc<Timeline>>,
         rel_size_v2_status: Option<RelSizeMigration>,
-        ctx: &RequestContext,
-    ) -> anyhow::Result<(UninitializedTimeline<'a>, RequestContext)> {
+    ) -> anyhow::Result<UninitializedTimeline<'a>> {
         let tenant_shard_id = self.tenant_shard_id;
 
         let resources = self.build_timeline_resources(new_timeline_id);
@@ -5263,7 +5233,7 @@ impl Tenant {
             .remote_client
             .init_upload_queue_for_empty_remote(new_metadata, rel_size_v2_status.clone())?;
 
-        let (timeline_struct, timeline_ctx) = self
+        let timeline_struct = self
             .create_timeline_struct(
                 new_timeline_id,
                 new_metadata,
@@ -5274,7 +5244,6 @@ impl Tenant {
                 create_guard.idempotency.clone(),
                 None,
                 rel_size_v2_status,
-                ctx,
             )
             .context("Failed to create timeline data structure")?;
 
@@ -5295,13 +5264,10 @@ impl Tenant {
             "Successfully created initial files for timeline {tenant_shard_id}/{new_timeline_id}"
         );
 
-        Ok((
-            UninitializedTimeline::new(
-                self,
-                new_timeline_id,
-                Some((timeline_struct, create_guard)),
-            ),
-            timeline_ctx,
+        Ok(UninitializedTimeline::new(
+            self,
+            new_timeline_id,
+            Some((timeline_struct, create_guard)),
         ))
     }
 
@@ -5754,7 +5720,7 @@ pub(crate) mod harness {
                 logging::TracingErrorLayerEnablement::EnableWithRustLogFilter,
                 logging::Output::Stdout,
             )
-            .expect("Failed to init test logging");
+            .expect("Failed to init test logging")
         });
     }
 
@@ -5836,8 +5802,7 @@ pub(crate) mod harness {
         }
 
         pub(crate) async fn load(&self) -> (Arc<Tenant>, RequestContext) {
-            let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error)
-                .with_scope_unit_test();
+            let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
             (
                 self.do_try_load(&ctx)
                     .await
@@ -6860,7 +6825,7 @@ mod tests {
 
         let (tenant, ctx) = harness.load().await;
         let io_concurrency = IoConcurrency::spawn_for_test();
-        let (tline, ctx) = tenant
+        let tline = tenant
             .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)
             .await?;
         let tline = tline.raw_timeline().unwrap();
@@ -7482,7 +7447,7 @@ mod tests {
             .await;
 
         let initdb_lsn = Lsn(0x20);
-        let (utline, ctx) = tenant
+        let utline = tenant
             .create_empty_timeline(TIMELINE_ID, initdb_lsn, DEFAULT_PG_VERSION, &ctx)
             .await?;
         let tline = utline.raw_timeline().unwrap();
@@ -7549,7 +7514,7 @@ mod tests {
         let harness = TenantHarness::create(name).await?;
         {
             let (tenant, ctx) = harness.load().await;
-            let (tline, _ctx) = tenant
+            let tline = tenant
                 .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)
                 .await?;
             // Leave the timeline ID in [`Tenant::timelines_creating`] to exclude attempting to create it again

pageserver/src/tenant/blob_io.rs

@@ -471,8 +471,7 @@ pub(crate) mod tests {
         blobs: &[Vec<u8>],
         compression: bool,
     ) -> Result<(), Error> {
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
         let (_temp_dir, pathbuf, offsets) =
             write_maybe_compressed::<BUFFERED>(blobs, compression, &ctx).await?;
 

pageserver/src/tenant/config.rs

@@ -219,11 +219,7 @@ impl LocationConf {
         };
 
         let shard = if conf.shard_count == 0 {
-            // NB: carry over the persisted stripe size instead of using the default. This doesn't
-            // matter for most practical purposes, since unsharded tenants don't use the stripe
-            // size, but can cause inconsistencies between storcon and Pageserver and cause manual
-            // splits without `new_stripe_size` to use an unintended stripe size.
-            ShardIdentity::unsharded_with_stripe_size(ShardStripeSize(conf.shard_stripe_size))
+            ShardIdentity::unsharded()
         } else {
             ShardIdentity::new(
                 ShardNumber(conf.shard_number),

pageserver/src/tenant/disk_btree.rs

@@ -32,7 +32,8 @@ use hex;
 use thiserror::Error;
 use tracing::error;
 
-use crate::context::RequestContext;
+use crate::context::{DownloadBehavior, RequestContext};
+use crate::task_mgr::TaskKind;
 use crate::tenant::block_io::{BlockReader, BlockWriter};
 
 // The maximum size of a value stored in the B-tree. 5 bytes is enough currently.
@@ -476,15 +477,16 @@ where
     }
 
     #[allow(dead_code)]
-    pub async fn dump(&self, ctx: &RequestContext) -> Result<()> {
+    pub async fn dump(&self) -> Result<()> {
         let mut stack = Vec::new();
+        let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
 
         stack.push((self.root_blk, String::new(), 0, 0, 0));
 
         let block_cursor = self.reader.block_cursor();
 
         while let Some((blknum, path, depth, child_idx, key_off)) = stack.pop() {
-            let blk = block_cursor.read_blk(self.start_blk + blknum, ctx).await?;
+            let blk = block_cursor.read_blk(self.start_blk + blknum, &ctx).await?;
             let buf: &[u8] = blk.as_ref();
             let node = OnDiskNode::<L>::deparse(buf)?;
 
@@ -833,8 +835,6 @@ pub(crate) mod tests {
     use rand::Rng;
 
     use super::*;
-    use crate::context::DownloadBehavior;
-    use crate::task_mgr::TaskKind;
     use crate::tenant::block_io::{BlockCursor, BlockLease, BlockReaderRef};
 
     #[derive(Clone, Default)]
@@ -869,8 +869,7 @@ pub(crate) mod tests {
         let mut disk = TestDisk::new();
         let mut writer = DiskBtreeBuilder::<_, 6>::new(&mut disk);
 
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
 
         let all_keys: Vec<&[u8; 6]> = vec![
             b"xaaaaa", b"xaaaba", b"xaaaca", b"xabaaa", b"xababa", b"xabaca", b"xabada", b"xabadb",
@@ -888,7 +887,7 @@ pub(crate) mod tests {
 
         let reader = DiskBtreeReader::new(0, root_offset, disk);
 
-        reader.dump(&ctx).await?;
+        reader.dump().await?;
 
         // Test the `get` function on all the keys.
         for (key, val) in all_data.iter() {
@@ -980,8 +979,7 @@ pub(crate) mod tests {
     async fn lots_of_keys() -> Result<()> {
         let mut disk = TestDisk::new();
         let mut writer = DiskBtreeBuilder::<_, 8>::new(&mut disk);
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
 
         const NUM_KEYS: u64 = 1000;
 
@@ -999,7 +997,7 @@ pub(crate) mod tests {
 
         let reader = DiskBtreeReader::new(0, root_offset, disk);
 
-        reader.dump(&ctx).await?;
+        reader.dump().await?;
 
         use std::sync::Mutex;
 
@@ -1169,8 +1167,7 @@ pub(crate) mod tests {
         // Build a tree from it
         let mut disk = TestDisk::new();
         let mut writer = DiskBtreeBuilder::<_, 26>::new(&mut disk);
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
 
         for (key, val) in disk_btree_test_data::TEST_DATA {
             writer.append(&key, val)?;
@@ -1201,7 +1198,7 @@ pub(crate) mod tests {
             .await?;
         assert_eq!(count, disk_btree_test_data::TEST_DATA.len());
 
-        reader.dump(&ctx).await?;
+        reader.dump().await?;
 
         Ok(())
     }

pageserver/src/tenant/ephemeral_file.rs

@@ -9,7 +9,7 @@ use camino::Utf8PathBuf;
 use num_traits::Num;
 use pageserver_api::shard::TenantShardId;
 use tokio_epoll_uring::{BoundedBuf, Slice};
-use tracing::{error, info_span};
+use tracing::error;
 use utils::id::TimelineId;
 
 use crate::assert_u64_eq_usize::{U64IsUsize, UsizeIsU64};
@@ -76,7 +76,6 @@ impl EphemeralFile {
                 || IoBufferMut::with_capacity(TAIL_SZ),
                 gate.enter()?,
                 ctx,
-                info_span!(parent: None, "ephemeral_file_buffered_writer", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), timeline_id=%timeline_id, path = %filename),
             ),
             _gate_guard: gate.enter()?,
         })
@@ -352,8 +351,7 @@ mod tests {
         let timeline_id = TimelineId::from_str("22000000000000000000000000000000").unwrap();
         fs::create_dir_all(conf.timeline_path(&tenant_shard_id, &timeline_id))?;
 
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
 
         Ok((conf, tenant_shard_id, timeline_id, ctx))
     }

pageserver/src/tenant/metadata.rs

@@ -300,8 +300,9 @@ impl TimelineMetadata {
 
     /// Returns true if anything was changed
     pub fn detach_from_ancestor(&mut self, branchpoint: &(TimelineId, Lsn)) {
-        // Detaching from ancestor now doesn't always detach directly to the direct ancestor, but we
-        // ensure the LSN is the same. So we don't check the timeline ID.
+        if let Some(ancestor) = self.body.ancestor_timeline {
+            assert_eq!(ancestor, branchpoint.0);
+        }
         if self.body.ancestor_lsn != Lsn(0) {
             assert_eq!(self.body.ancestor_lsn, branchpoint.1);
         }

pageserver/src/tenant/mgr.rs

@@ -1914,7 +1914,6 @@ impl TenantManager {
         tenant_shard_id: TenantShardId,
         timeline_id: TimelineId,
         prepared: PreparedTimelineDetach,
-        behavior: detach_ancestor::DetachBehavior,
         mut attempt: detach_ancestor::Attempt,
         ctx: &RequestContext,
     ) -> Result<HashSet<TimelineId>, detach_ancestor::Error> {
@@ -1958,14 +1957,7 @@ impl TenantManager {
             .map_err(Error::NotFound)?;
 
         let resp = timeline
-            .detach_from_ancestor_and_reparent(
-                &tenant,
-                prepared,
-                attempt.ancestor_timeline_id,
-                attempt.ancestor_lsn,
-                behavior,
-                ctx,
-            )
+            .detach_from_ancestor_and_reparent(&tenant, prepared, ctx)
             .await?;
 
         let mut slot_guard = slot_guard;

pageserver/src/tenant/remote_timeline_client.rs

@@ -954,14 +954,6 @@ impl RemoteTimelineClient {
         Ok(())
     }
 
-    /// Only used in the `patch_index_part` HTTP API to force trigger an index upload.
-    pub fn force_schedule_index_upload(self: &Arc<Self>) -> Result<(), NotInitialized> {
-        let mut guard = self.upload_queue.lock().unwrap();
-        let upload_queue = guard.initialized_mut()?;
-        self.schedule_index_upload(upload_queue);
-        Ok(())
-    }
-
     /// Launch an index-file upload operation in the background (internal function)
     fn schedule_index_upload(self: &Arc<Self>, upload_queue: &mut UploadQueueInitialized) {
         let disk_consistent_lsn = upload_queue.dirty.metadata.disk_consistent_lsn();

pageserver/src/tenant/remote_timeline_client/download.rs

@@ -229,7 +229,6 @@ async fn download_object(
                     || IoBufferMut::with_capacity(super::BUFFER_SIZE),
                     gate.enter().map_err(|_| DownloadError::Cancelled)?,
                     ctx,
-                    tracing::info_span!(parent: None, "download_object_buffered_writer", %dst_path),
                 );
 
                 // TODO: use vectored write (writev) once supported by tokio-epoll-uring.

pageserver/src/tenant/secondary/downloader.rs

@@ -491,10 +491,7 @@ impl JobGenerator<PendingDownload, RunningDownload, CompleteDownload, DownloadCo
         let remote_storage = self.remote_storage.clone();
         let conf = self.tenant_manager.get_conf();
         let tenant_shard_id = *secondary_state.get_tenant_shard_id();
-        let download_ctx = self
-            .root_ctx
-            .attached_child()
-            .with_scope_secondary_tenant(&tenant_shard_id);
+        let download_ctx = self.root_ctx.attached_child();
         (RunningDownload { barrier }, Box::pin(async move {
             let _completion = completion;
 
@@ -774,7 +771,6 @@ impl<'a> TenantDownloader<'a> {
 
         // Download the layers in the heatmap
         for timeline in heatmap.timelines {
-            let ctx = &ctx.with_scope_secondary_timeline(tenant_shard_id, &timeline.timeline_id);
             let timeline_state = timeline_states
                 .remove(&timeline.timeline_id)
                 .expect("Just populated above");

pageserver/src/tenant/size.rs

@@ -474,7 +474,7 @@ async fn fill_logical_sizes(
             if cached_size.is_none() {
                 let timeline = Arc::clone(timeline_hash.get(&timeline_id).unwrap());
                 let parallel_size_calcs = Arc::clone(limit);
-                let ctx = ctx.attached_child().with_scope_timeline(&timeline);
+                let ctx = ctx.attached_child();
                 joinset.spawn(
                     calculate_logical_size(parallel_size_calcs, timeline, lsn, cause, ctx)
                         .in_current_span(),

pageserver/src/tenant/storage_layer/delta_layer.rs

@@ -1334,7 +1334,7 @@ impl DeltaLayerInner {
             block_reader,
         );
 
-        tree_reader.dump(ctx).await?;
+        tree_reader.dump().await?;
 
         let keys = self.index_entries(ctx).await?;
 
@@ -1972,7 +1972,6 @@ pub(crate) mod test {
             .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, ctx)
             .await
             .unwrap();
-        let ctx = &ctx.with_scope_timeline(&timeline);
 
         let initdb_layer = timeline
             .layers

pageserver/src/tenant/storage_layer/image_layer.rs

@@ -199,7 +199,7 @@ impl ImageLayerInner {
             block_reader,
         );
 
-        tree_reader.dump(ctx).await?;
+        tree_reader.dump().await?;
 
         tree_reader
             .visit(

pageserver/src/tenant/storage_layer/layer/tests.rs

@@ -8,6 +8,7 @@ use utils::id::TimelineId;
 use super::failpoints::{Failpoint, FailpointKind};
 use super::*;
 use crate::context::DownloadBehavior;
+use crate::task_mgr::TaskKind;
 use crate::tenant::harness::{TenantHarness, test_img};
 use crate::tenant::storage_layer::{IoConcurrency, LayerVisibilityHint};
 
@@ -26,9 +27,11 @@ async fn smoke_test() {
     let h = TenantHarness::create("smoke_test").await.unwrap();
     let span = h.span();
     let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1));
-    let (tenant, ctx) = h.load().await;
+    let (tenant, _) = h.load().await;
     let io_concurrency = IoConcurrency::spawn_for_test();
 
+    let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Download);
+
     let image_layers = vec![(
         Lsn(0x40),
         vec![(
@@ -53,7 +56,6 @@ async fn smoke_test() {
         )
         .await
         .unwrap();
-    let ctx = &ctx.with_scope_timeline(&timeline);
 
     // Grab one of the timeline's layers to exercise in the test, and the other layer that is just
     // there to avoid the timeline being illegally empty
@@ -92,7 +94,7 @@ async fn smoke_test() {
                 controlfile_keyspace.clone(),
                 Lsn(0x10)..Lsn(0x11),
                 &mut data,
-                ctx,
+                &ctx,
             )
             .await
             .unwrap();
@@ -127,7 +129,7 @@ async fn smoke_test() {
                 controlfile_keyspace.clone(),
                 Lsn(0x10)..Lsn(0x11),
                 &mut data,
-                ctx,
+                &ctx,
             )
             .instrument(download_span.clone())
             .await
@@ -177,7 +179,7 @@ async fn smoke_test() {
 
     // plain downloading is rarely needed
     layer
-        .download_and_keep_resident(ctx)
+        .download_and_keep_resident(&ctx)
         .instrument(download_span)
         .await
         .unwrap();
@@ -339,7 +341,6 @@ fn read_wins_pending_eviction() {
             .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, &ctx)
             .await
             .unwrap();
-        let ctx = ctx.with_scope_timeline(&timeline);
 
         let layer = {
             let mut layers = {
@@ -472,7 +473,6 @@ fn multiple_pending_evictions_scenario(name: &'static str, in_order: bool) {
             .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, &ctx)
             .await
             .unwrap();
-        let ctx = ctx.with_scope_timeline(&timeline);
 
         let layer = {
             let mut layers = {
@@ -642,12 +642,12 @@ async fn cancelled_get_or_maybe_download_does_not_cancel_eviction() {
         .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, &ctx)
         .await
         .unwrap();
-    let ctx = ctx.with_scope_timeline(&timeline);
 
     // This test does downloads
     let ctx = RequestContextBuilder::extend(&ctx)
         .download_behavior(DownloadBehavior::Download)
         .build();
+
     let layer = {
         let mut layers = {
             let layers = timeline.layers.read().await;
@@ -727,7 +727,6 @@ async fn evict_and_wait_does_not_wait_for_download() {
         .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, &ctx)
         .await
         .unwrap();
-    let ctx = ctx.with_scope_timeline(&timeline);
 
     // This test does downloads
     let ctx = RequestContextBuilder::extend(&ctx)

pageserver/src/tenant/timeline.rs

@@ -287,7 +287,7 @@ pub struct Timeline {
     // The LSN of gc-compaction that was last applied to this timeline.
     gc_compaction_state: ArcSwap<Option<GcCompactionState>>,
 
-    pub(crate) metrics: Arc<TimelineMetrics>,
+    pub(super) metrics: TimelineMetrics,
 
     // `Timeline` doesn't write these metrics itself, but it manages the lifetime.  Code
     // in `crate::page_service` writes these metrics.
@@ -2685,14 +2685,14 @@ impl Timeline {
         }
 
         Arc::new_cyclic(|myself| {
-            let metrics = Arc::new(TimelineMetrics::new(
+            let metrics = TimelineMetrics::new(
                 &tenant_shard_id,
                 &timeline_id,
                 crate::metrics::EvictionsWithLowResidenceDurationBuilder::new(
                     "mtime",
                     evictions_low_residence_duration_metric_threshold,
                 ),
-            ));
+            );
             let aux_file_metrics = metrics.aux_file_size_gauge.clone();
 
             let mut result = Timeline {
@@ -2876,7 +2876,7 @@ impl Timeline {
             "layer flush task",
             async move {
                 let _guard = guard;
-                let background_ctx = RequestContext::todo_child(TaskKind::LayerFlushTask, DownloadBehavior::Error).with_scope_timeline(&self_clone);
+                let background_ctx = RequestContext::todo_child(TaskKind::LayerFlushTask, DownloadBehavior::Error);
                 self_clone.flush_loop(layer_flush_start_rx, &background_ctx).await;
                 let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap();
                 assert!(matches!(*flush_loop_state, FlushLoopState::Running{..}));
@@ -5388,10 +5388,9 @@ impl Timeline {
         self: &Arc<Timeline>,
         tenant: &crate::tenant::Tenant,
         options: detach_ancestor::Options,
-        behavior: detach_ancestor::DetachBehavior,
         ctx: &RequestContext,
     ) -> Result<detach_ancestor::Progress, detach_ancestor::Error> {
-        detach_ancestor::prepare(self, tenant, behavior, options, ctx).await
+        detach_ancestor::prepare(self, tenant, options, ctx).await
     }
 
     /// Second step of detach from ancestor; detaches the `self` from it's current ancestor and
@@ -5407,21 +5406,9 @@ impl Timeline {
         self: &Arc<Timeline>,
         tenant: &crate::tenant::Tenant,
         prepared: detach_ancestor::PreparedTimelineDetach,
-        ancestor_timeline_id: TimelineId,
-        ancestor_lsn: Lsn,
-        behavior: detach_ancestor::DetachBehavior,
         ctx: &RequestContext,
     ) -> Result<detach_ancestor::DetachingAndReparenting, detach_ancestor::Error> {
-        detach_ancestor::detach_and_reparent(
-            self,
-            tenant,
-            prepared,
-            ancestor_timeline_id,
-            ancestor_lsn,
-            behavior,
-            ctx,
-        )
-        .await
+        detach_ancestor::detach_and_reparent(self, tenant, prepared, ctx).await
     }
 
     /// Final step which unblocks the GC.
@@ -7140,7 +7127,6 @@ mod tests {
             )
             .await
             .unwrap();
-        let ctx = &ctx.with_scope_timeline(&timeline);
 
         // Layer visibility is an input to heatmap generation, so refresh it first
         timeline.update_layer_visibility().await.unwrap();
@@ -7206,7 +7192,7 @@ mod tests {
 
             eprintln!("Downloading {layer} and re-generating heatmap");
 
-            let ctx = &RequestContextBuilder::extend(ctx)
+            let ctx = &RequestContextBuilder::extend(&ctx)
                 .download_behavior(crate::context::DownloadBehavior::Download)
                 .build();
 

pageserver/src/tenant/timeline/compaction.rs

@@ -393,9 +393,6 @@ impl GcCompactionQueue {
                 if job.dry_run {
                     flags |= CompactFlags::DryRun;
                 }
-                if options.flags.contains(CompactFlags::NoYield) {
-                    flags |= CompactFlags::NoYield;
-                }
                 let options = CompactOptions {
                     flags,
                     sub_compaction: false,
@@ -1094,7 +1091,7 @@ impl Timeline {
         let latest_gc_cutoff = self.get_applied_gc_cutoff_lsn();
 
         tracing::info!(
-            "starting shard ancestor compaction, latest_gc_cutoff: {}, pitr cutoff {}",
+            "latest_gc_cutoff: {}, pitr cutoff {}",
             *latest_gc_cutoff,
             self.gc_info.read().unwrap().cutoffs.time
         );
@@ -1123,7 +1120,6 @@ impl Timeline {
                     // Expensive, exhaustive check of keys in this layer: this guards against ShardedRange's calculations being
                     // wrong.  If ShardedRange claims the local page count is zero, then no keys in this layer
                     // should be !is_key_disposable()
-                    // TODO: exclude sparse keyspace from this check, otherwise it will infinitely loop.
                     let range = layer_desc.get_key_range();
                     let mut key = range.start;
                     while key < range.end {
@@ -2620,7 +2616,6 @@ impl Timeline {
     ) -> Result<CompactionOutcome, CompactionError> {
         let sub_compaction = options.sub_compaction;
         let job = GcCompactJob::from_compact_options(options.clone());
-        let no_yield = options.flags.contains(CompactFlags::NoYield);
         if sub_compaction {
             info!(
                 "running enhanced gc bottom-most compaction with sub-compaction, splitting compaction jobs"
@@ -2635,15 +2630,14 @@ impl Timeline {
                     idx + 1,
                     jobs_len
                 );
-                self.compact_with_gc_inner(cancel, job, ctx, no_yield)
-                    .await?;
+                self.compact_with_gc_inner(cancel, job, ctx).await?;
             }
             if jobs_len == 0 {
                 info!("no jobs to run, skipping gc bottom-most compaction");
             }
             return Ok(CompactionOutcome::Done);
         }
-        self.compact_with_gc_inner(cancel, job, ctx, no_yield).await
+        self.compact_with_gc_inner(cancel, job, ctx).await
     }
 
     async fn compact_with_gc_inner(
@@ -2651,7 +2645,6 @@ impl Timeline {
         cancel: &CancellationToken,
         job: GcCompactJob,
         ctx: &RequestContext,
-        no_yield: bool,
     ) -> Result<CompactionOutcome, CompactionError> {
         // Block other compaction/GC tasks from running for now. GC-compaction could run along
         // with legacy compaction tasks in the future. Always ensure the lock order is compaction -> gc.
@@ -2921,18 +2914,14 @@ impl Timeline {
             if cancel.is_cancelled() {
                 return Err(CompactionError::ShuttingDown);
             }
-            if !no_yield {
-                let should_yield = self
-                    .l0_compaction_trigger
-                    .notified()
-                    .now_or_never()
-                    .is_some();
-                if should_yield {
-                    tracing::info!(
-                        "preempt gc-compaction when downloading layers: too many L0 layers"
-                    );
-                    return Ok(CompactionOutcome::YieldForL0);
-                }
+            let should_yield = self
+                .l0_compaction_trigger
+                .notified()
+                .now_or_never()
+                .is_some();
+            if should_yield {
+                tracing::info!("preempt gc-compaction when downloading layers: too many L0 layers");
+                return Ok(CompactionOutcome::YieldForL0);
             }
             let resident_layer = layer
                 .download_and_keep_resident(ctx)
@@ -3065,21 +3054,16 @@ impl Timeline {
             if cancel.is_cancelled() {
                 return Err(CompactionError::ShuttingDown);
             }
-
-            if !no_yield {
-                keys_processed += 1;
-                if keys_processed % 1000 == 0 {
-                    let should_yield = self
-                        .l0_compaction_trigger
-                        .notified()
-                        .now_or_never()
-                        .is_some();
-                    if should_yield {
-                        tracing::info!(
-                            "preempt gc-compaction in the main loop: too many L0 layers"
-                        );
-                        return Ok(CompactionOutcome::YieldForL0);
-                    }
+            keys_processed += 1;
+            if keys_processed % 1000 == 0 {
+                let should_yield = self
+                    .l0_compaction_trigger
+                    .notified()
+                    .now_or_never()
+                    .is_some();
+                if should_yield {
+                    tracing::info!("preempt gc-compaction in the main loop: too many L0 layers");
+                    return Ok(CompactionOutcome::YieldForL0);
                 }
             }
             if self.shard_identity.is_key_disposable(&key) {

pageserver/src/tenant/timeline/delete.rs

@@ -11,7 +11,6 @@ use utils::id::TimelineId;
 use utils::{crashsafe, fs_ext, pausable_failpoint};
 
 use crate::config::PageServerConf;
-use crate::context::RequestContext;
 use crate::task_mgr::{self, TaskKind};
 use crate::tenant::metadata::TimelineMetadata;
 use crate::tenant::remote_timeline_client::{
@@ -292,11 +291,10 @@ impl DeleteTimelineFlow {
         timeline_id: TimelineId,
         local_metadata: &TimelineMetadata,
         remote_client: RemoteTimelineClient,
-        ctx: &RequestContext,
     ) -> anyhow::Result<()> {
         // Note: here we even skip populating layer map. Timeline is essentially uninitialized.
         // RemoteTimelineClient is the only functioning part.
-        let (timeline, _timeline_ctx) = tenant
+        let timeline = tenant
             .create_timeline_struct(
                 timeline_id,
                 local_metadata,
@@ -309,7 +307,6 @@ impl DeleteTimelineFlow {
                 crate::tenant::CreateTimelineIdempotency::FailWithConflict, // doesn't matter what we put here
                 None, // doesn't matter what we put here
                 None, // doesn't matter what we put here
-                ctx,
             )
             .context("create_timeline_struct")?;
 

pageserver/src/tenant/timeline/detach_ancestor.rs

@@ -32,9 +32,6 @@ pub(crate) enum Error {
     #[error("too many ancestors")]
     TooManyAncestors,
 
-    #[error("ancestor is not empty")]
-    AncestorNotEmpty,
-
     #[error("shutting down, please retry later")]
     ShuttingDown,
 
@@ -92,9 +89,7 @@ impl From<Error> for ApiError {
     fn from(value: Error) -> Self {
         match value {
             Error::NoAncestor => ApiError::Conflict(value.to_string()),
-            Error::TooManyAncestors | Error::AncestorNotEmpty => {
-                ApiError::BadRequest(anyhow::anyhow!("{value}"))
-            }
+            Error::TooManyAncestors => ApiError::BadRequest(anyhow::anyhow!("{value}")),
             Error::ShuttingDown => ApiError::ShuttingDown,
             Error::Archived(_) => ApiError::BadRequest(anyhow::anyhow!("{value}")),
             Error::OtherTimelineDetachOngoing(_) | Error::FailedToReparentAll => {
@@ -132,37 +127,13 @@ pub(crate) struct PreparedTimelineDetach {
     layers: Vec<Layer>,
 }
 
-// TODO: this should be part of PageserverConf because we cannot easily modify cplane arguments.
+/// TODO: this should be part of PageserverConf because we cannot easily modify cplane arguments.
 #[derive(Debug)]
 pub(crate) struct Options {
     pub(crate) rewrite_concurrency: std::num::NonZeroUsize,
     pub(crate) copy_concurrency: std::num::NonZeroUsize,
 }
 
-/// Controls the detach ancestor behavior.
-/// - When set to `NoAncestorAndReparent`, we will only detach a branch if its ancestor is a root branch. It will automatically reparent any children of the ancestor before and at the branch point.
-/// - When set to `MultiLevelAndNoReparent`, we will detach a branch from multiple levels of ancestors, and no reparenting will happen at all.
-#[derive(Debug, Clone, Copy, Default)]
-pub enum DetachBehavior {
-    #[default]
-    NoAncestorAndReparent,
-    MultiLevelAndNoReparent,
-}
-
-impl std::str::FromStr for DetachBehavior {
-    type Err = &'static str;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "no_ancestor_and_reparent" => Ok(DetachBehavior::NoAncestorAndReparent),
-            "multi_level_and_no_reparent" => Ok(DetachBehavior::MultiLevelAndNoReparent),
-            "v1" => Ok(DetachBehavior::NoAncestorAndReparent),
-            "v2" => Ok(DetachBehavior::MultiLevelAndNoReparent),
-            _ => Err("cannot parse detach behavior"),
-        }
-    }
-}
-
 impl Default for Options {
     fn default() -> Self {
         Self {
@@ -176,8 +147,7 @@ impl Default for Options {
 #[derive(Debug)]
 pub(crate) struct Attempt {
     pub(crate) timeline_id: TimelineId,
-    pub(crate) ancestor_timeline_id: TimelineId,
-    pub(crate) ancestor_lsn: Lsn,
+
     _guard: completion::Completion,
     gate_entered: Option<utils::sync::gate::GateGuard>,
 }
@@ -197,30 +167,25 @@ impl Attempt {
 pub(super) async fn prepare(
     detached: &Arc<Timeline>,
     tenant: &Tenant,
-    behavior: DetachBehavior,
     options: Options,
     ctx: &RequestContext,
 ) -> Result<Progress, Error> {
     use Error::*;
 
-    let Some((mut ancestor, mut ancestor_lsn)) = detached
+    let Some((ancestor, ancestor_lsn)) = detached
         .ancestor_timeline
         .as_ref()
         .map(|tl| (tl.clone(), detached.ancestor_lsn))
     else {
-        let ancestor_id;
-        let ancestor_lsn;
         let still_in_progress = {
             let accessor = detached.remote_client.initialized_upload_queue()?;
 
             // we are safe to inspect the latest uploaded, because we can only witness this after
             // restart is complete and ancestor is no more.
             let latest = accessor.latest_uploaded_index_part();
-            let Some((id, lsn)) = latest.lineage.detached_previous_ancestor() else {
+            if latest.lineage.detached_previous_ancestor().is_none() {
                 return Err(NoAncestor);
             };
-            ancestor_id = id;
-            ancestor_lsn = lsn;
 
             latest
                 .gc_blocking
@@ -231,8 +196,7 @@ pub(super) async fn prepare(
         if still_in_progress {
             // gc is still blocked, we can still reparent and complete.
             // we are safe to reparent remaining, because they were locked in in the beginning.
-            let attempt =
-                continue_with_blocked_gc(detached, tenant, ancestor_id, ancestor_lsn).await?;
+            let attempt = continue_with_blocked_gc(detached, tenant).await?;
 
             // because the ancestor of detached is already set to none, we have published all
             // of the layers, so we are still "prepared."
@@ -260,34 +224,13 @@ pub(super) async fn prepare(
 
     check_no_archived_children_of_ancestor(tenant, detached, &ancestor, ancestor_lsn)?;
 
-    if let DetachBehavior::MultiLevelAndNoReparent = behavior {
-        // If the ancestor has an ancestor, we might be able to fast-path detach it if the current ancestor does not have any data written/used by the detaching timeline.
-        while let Some(ancestor_of_ancestor) = ancestor.ancestor_timeline.clone() {
-            if ancestor_lsn != ancestor.ancestor_lsn {
-                // non-technical requirement; we could flatten still if ancestor LSN does not match but that needs
-                // us to copy and cut more layers.
-                return Err(AncestorNotEmpty);
-            }
-            // Use the ancestor of the ancestor as the new ancestor (only when the ancestor LSNs are the same)
-            ancestor_lsn = ancestor.ancestor_lsn; // Get the LSN first before resetting the `ancestor` variable
-            ancestor = ancestor_of_ancestor;
-            // TODO: do we still need to check if we don't want to reparent?
-            check_no_archived_children_of_ancestor(tenant, detached, &ancestor, ancestor_lsn)?;
-        }
-    } else if ancestor.ancestor_timeline.is_some() {
+    if ancestor.ancestor_timeline.is_some() {
         // non-technical requirement; we could flatten N ancestors just as easily but we chose
         // not to, at least initially
         return Err(TooManyAncestors);
     }
 
-    tracing::info!(
-        "attempt to detach the timeline from the ancestor: {}@{}, behavior={:?}",
-        ancestor.timeline_id,
-        ancestor_lsn,
-        behavior
-    );
-
-    let attempt = start_new_attempt(detached, tenant, ancestor.timeline_id, ancestor_lsn).await?;
+    let attempt = start_new_attempt(detached, tenant).await?;
 
     utils::pausable_failpoint!("timeline-detach-ancestor::before_starting_after_locking-pausable");
 
@@ -507,13 +450,8 @@ pub(super) async fn prepare(
     Ok(Progress::Prepared(attempt, prepared))
 }
 
-async fn start_new_attempt(
-    detached: &Timeline,
-    tenant: &Tenant,
-    ancestor_timeline_id: TimelineId,
-    ancestor_lsn: Lsn,
-) -> Result<Attempt, Error> {
-    let attempt = obtain_exclusive_attempt(detached, tenant, ancestor_timeline_id, ancestor_lsn)?;
+async fn start_new_attempt(detached: &Timeline, tenant: &Tenant) -> Result<Attempt, Error> {
+    let attempt = obtain_exclusive_attempt(detached, tenant)?;
 
     // insert the block in the index_part.json, if not already there.
     let _dont_care = tenant
@@ -528,23 +466,13 @@ async fn start_new_attempt(
     Ok(attempt)
 }
 
-async fn continue_with_blocked_gc(
-    detached: &Timeline,
-    tenant: &Tenant,
-    ancestor_timeline_id: TimelineId,
-    ancestor_lsn: Lsn,
-) -> Result<Attempt, Error> {
+async fn continue_with_blocked_gc(detached: &Timeline, tenant: &Tenant) -> Result<Attempt, Error> {
     // FIXME: it would be nice to confirm that there is an in-memory version, since we've just
     // verified there is a persistent one?
-    obtain_exclusive_attempt(detached, tenant, ancestor_timeline_id, ancestor_lsn)
+    obtain_exclusive_attempt(detached, tenant)
 }
 
-fn obtain_exclusive_attempt(
-    detached: &Timeline,
-    tenant: &Tenant,
-    ancestor_timeline_id: TimelineId,
-    ancestor_lsn: Lsn,
-) -> Result<Attempt, Error> {
+fn obtain_exclusive_attempt(detached: &Timeline, tenant: &Tenant) -> Result<Attempt, Error> {
     use Error::{OtherTimelineDetachOngoing, ShuttingDown};
 
     // ensure we are the only active attempt for this tenant
@@ -565,8 +493,6 @@ fn obtain_exclusive_attempt(
 
     Ok(Attempt {
         timeline_id: detached.timeline_id,
-        ancestor_timeline_id,
-        ancestor_lsn,
         _guard: guard,
         gate_entered: Some(_gate_entered),
     })
@@ -869,9 +795,6 @@ pub(super) async fn detach_and_reparent(
     detached: &Arc<Timeline>,
     tenant: &Tenant,
     prepared: PreparedTimelineDetach,
-    ancestor_timeline_id: TimelineId,
-    ancestor_lsn: Lsn,
-    behavior: DetachBehavior,
     _ctx: &RequestContext,
 ) -> Result<DetachingAndReparenting, Error> {
     let PreparedTimelineDetach { layers } = prepared;
@@ -899,30 +822,7 @@ pub(super) async fn detach_and_reparent(
         "cannot (detach? reparent)? complete if the operation is not still ongoing"
     );
 
-    let ancestor_to_detach = match detached.ancestor_timeline.as_ref() {
-        Some(mut ancestor) => {
-            while ancestor.timeline_id != ancestor_timeline_id {
-                match ancestor.ancestor_timeline.as_ref() {
-                    Some(found) => {
-                        if ancestor_lsn != ancestor.ancestor_lsn {
-                            return Err(Error::DetachReparent(anyhow::anyhow!(
-                                "cannot find the ancestor timeline to detach from: wrong ancestor lsn"
-                            )));
-                        }
-                        ancestor = found;
-                    }
-                    None => {
-                        return Err(Error::DetachReparent(anyhow::anyhow!(
-                            "cannot find the ancestor timeline to detach from"
-                        )));
-                    }
-                }
-            }
-            Some(ancestor)
-        }
-        None => None,
-    };
-    let ancestor = match (ancestor_to_detach, recorded_branchpoint) {
+    let ancestor = match (detached.ancestor_timeline.as_ref(), recorded_branchpoint) {
         (Some(ancestor), None) => {
             assert!(
                 !layers.is_empty(),
@@ -995,11 +895,6 @@ pub(super) async fn detach_and_reparent(
         Ancestor::Detached(ancestor, ancestor_lsn) => (ancestor, ancestor_lsn, false),
     };
 
-    if let DetachBehavior::MultiLevelAndNoReparent = behavior {
-        // Do not reparent if the user requests to behave so.
-        return Ok(DetachingAndReparenting::Reparented(HashSet::new()));
-    }
-
     let mut tasks = tokio::task::JoinSet::new();
 
     // Returns a single permit semaphore which will be used to make one reparenting succeed,
@@ -1137,11 +1032,6 @@ pub(super) async fn complete(
 }
 
 /// Query against a locked `Tenant::timelines`.
-///
-/// A timeline is reparentable if:
-///
-/// - It is not the timeline being detached.
-/// - It has the same ancestor as the timeline being detached. Note that the ancestor might not be the direct ancestor.
 fn reparentable_timelines<'a, I>(
     timelines: I,
     detached: &'a Arc<Timeline>,

pageserver/src/tenant/timeline/eviction_task.rs

@@ -93,8 +93,7 @@ impl Timeline {
             }
         }
 
-        let ctx = RequestContext::new(TaskKind::Eviction, DownloadBehavior::Warn)
-            .with_scope_timeline(&self);
+        let ctx = RequestContext::new(TaskKind::Eviction, DownloadBehavior::Warn);
         loop {
             let policy = self.get_eviction_policy();
             let cf = self

pageserver/src/tenant/vectored_blob_io.rs

@@ -961,8 +961,7 @@ mod tests {
     }
 
     async fn round_trip_test_compressed(blobs: &[Vec<u8>], compression: bool) -> Result<(), Error> {
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
         let (_temp_dir, pathbuf, offsets) =
             write_maybe_compressed::<true>(blobs, compression, &ctx).await?;
 

pageserver/src/virtual_file.rs

@@ -26,14 +26,15 @@ use owned_buffers_io::io_buf_aligned::{IoBufAligned, IoBufAlignedMut};
 use owned_buffers_io::io_buf_ext::FullSlice;
 use pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT;
 pub use pageserver_api::models::virtual_file as api;
+use pageserver_api::shard::TenantShardId;
 use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 use tokio::time::Instant;
 use tokio_epoll_uring::{BoundedBuf, IoBuf, IoBufMut, Slice};
 
-use crate::assert_u64_eq_usize::UsizeIsU64;
 use crate::context::RequestContext;
-use crate::metrics::{STORAGE_IO_TIME_METRIC, StorageIoOperation};
+use crate::metrics::{STORAGE_IO_SIZE, STORAGE_IO_TIME_METRIC, StorageIoOperation};
 use crate::page_cache::{PAGE_SZ, PageWriteGuard};
+use crate::tenant::TENANTS_SEGMENT_NAME;
 pub(crate) mod io_engine;
 pub use io_engine::{
     FeatureTestResult as IoEngineFeatureTestResult, feature_test as io_engine_feature_test,
@@ -120,7 +121,7 @@ impl VirtualFile {
     pub async fn open_with_options<P: AsRef<Utf8Path>>(
         path: P,
         open_options: &OpenOptions,
-        ctx: &RequestContext,
+        ctx: &RequestContext, /* TODO: carry a pointer to the metrics in the RequestContext instead of the parsing https://github.com/neondatabase/neon/issues/6107 */
     ) -> Result<Self, std::io::Error> {
         let inner = VirtualFileInner::open_with_options(path, open_options, ctx).await?;
         Ok(VirtualFile {
@@ -132,7 +133,7 @@ impl VirtualFile {
     pub async fn open_with_options_v2<P: AsRef<Utf8Path>>(
         path: P,
         open_options: &OpenOptions,
-        ctx: &RequestContext,
+        ctx: &RequestContext, /* TODO: carry a pointer to the metrics in the RequestContext instead of the parsing https://github.com/neondatabase/neon/issues/6107 */
     ) -> Result<Self, std::io::Error> {
         let file = match get_io_mode() {
             IoMode::Buffered => {
@@ -303,6 +304,13 @@ pub struct VirtualFileInner {
     /// storing it here.
     pub path: Utf8PathBuf,
     open_options: OpenOptions,
+
+    // These are strings becase we only use them for metrics, and those expect strings.
+    // It makes no sense for us to constantly turn the `TimelineId` and `TenantId` into
+    // strings.
+    tenant_id: String,
+    shard_id: String,
+    timeline_id: String,
 }
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -584,16 +592,36 @@ impl VirtualFileInner {
     pub async fn open_with_options<P: AsRef<Utf8Path>>(
         path: P,
         open_options: &OpenOptions,
-        _ctx: &RequestContext,
+        _ctx: &RequestContext, /* TODO: carry a pointer to the metrics in the RequestContext instead of the parsing https://github.com/neondatabase/neon/issues/6107 */
     ) -> Result<VirtualFileInner, std::io::Error> {
-        let path = path.as_ref();
+        let path_ref = path.as_ref();
+        let path_str = path_ref.to_string();
+        let parts = path_str.split('/').collect::<Vec<&str>>();
+        let (tenant_id, shard_id, timeline_id) =
+            if parts.len() > 5 && parts[parts.len() - 5] == TENANTS_SEGMENT_NAME {
+                let tenant_shard_part = parts[parts.len() - 4];
+                let (tenant_id, shard_id) = match tenant_shard_part.parse::<TenantShardId>() {
+                    Ok(tenant_shard_id) => (
+                        tenant_shard_id.tenant_id.to_string(),
+                        format!("{}", tenant_shard_id.shard_slug()),
+                    ),
+                    Err(_) => {
+                        // Malformed path: this ID is just for observability, so tolerate it
+                        // and pass through
+                        (tenant_shard_part.to_string(), "*".to_string())
+                    }
+                };
+                (tenant_id, shard_id, parts[parts.len() - 2].to_string())
+            } else {
+                ("*".to_string(), "*".to_string(), "*".to_string())
+            };
         let (handle, mut slot_guard) = get_open_files().find_victim_slot().await;
 
         // NB: there is also StorageIoOperation::OpenAfterReplace which is for the case
         // where our caller doesn't get to use the returned VirtualFile before its
         // slot gets re-used by someone else.
         let file = observe_duration!(StorageIoOperation::Open, {
-            open_options.open(path.as_std_path()).await?
+            open_options.open(path_ref.as_std_path()).await?
         });
 
         // Strip all options other than read and write.
@@ -609,8 +637,11 @@ impl VirtualFileInner {
         let vfile = VirtualFileInner {
             handle: RwLock::new(handle),
             pos: 0,
-            path: path.to_owned(),
+            path: path_ref.to_path_buf(),
             open_options: reopen_options,
+            tenant_id,
+            shard_id,
+            timeline_id,
         };
 
         // TODO: Under pressure, it's likely the slot will get re-used and
@@ -913,7 +944,7 @@ impl VirtualFileInner {
         &self,
         buf: tokio_epoll_uring::Slice<Buf>,
         offset: u64,
-        ctx: &RequestContext,
+        _ctx: &RequestContext, /* TODO: use for metrics: https://github.com/neondatabase/neon/issues/6107 */
     ) -> (tokio_epoll_uring::Slice<Buf>, Result<usize, Error>)
     where
         Buf: tokio_epoll_uring::IoBufMut + Send,
@@ -931,7 +962,14 @@ impl VirtualFileInner {
             let ((_file_guard, buf), res) = io_engine::get().read_at(file_guard, offset, buf).await;
             let res = res.maybe_fatal_err("io_engine read_at inside VirtualFileInner::read_at");
             if let Ok(size) = res {
-                ctx.io_size_metrics().read.add(size.into_u64());
+                STORAGE_IO_SIZE
+                    .with_label_values(&[
+                        "read",
+                        &self.tenant_id,
+                        &self.shard_id,
+                        &self.timeline_id,
+                    ])
+                    .add(size as i64);
             }
             (buf, res)
         })
@@ -942,9 +980,9 @@ impl VirtualFileInner {
         &self,
         buf: FullSlice<B>,
         offset: u64,
-        ctx: &RequestContext,
+        _ctx: &RequestContext, /* TODO: use for metrics: https://github.com/neondatabase/neon/issues/6107 */
     ) -> (FullSlice<B>, Result<usize, Error>) {
-        let (slice, result) = self.write_at_inner(buf, offset, ctx).await;
+        let (slice, result) = self.write_at_inner(buf, offset, _ctx).await;
         let result = result.maybe_fatal_err("write_at");
         (slice, result)
     }
@@ -953,7 +991,7 @@ impl VirtualFileInner {
         &self,
         buf: FullSlice<B>,
         offset: u64,
-        ctx: &RequestContext,
+        _ctx: &RequestContext, /* TODO: use for metrics: https://github.com/neondatabase/neon/issues/6107 */
     ) -> (FullSlice<B>, Result<usize, Error>) {
         let file_guard = match self.lock_file().await {
             Ok(file_guard) => file_guard,
@@ -963,7 +1001,14 @@ impl VirtualFileInner {
             let ((_file_guard, buf), result) =
                 io_engine::get().write_at(file_guard, offset, buf).await;
             if let Ok(size) = result {
-                ctx.io_size_metrics().write.add(size.into_u64());
+                STORAGE_IO_SIZE
+                    .with_label_values(&[
+                        "write",
+                        &self.tenant_id,
+                        &self.shard_id,
+                        &self.timeline_id,
+                    ])
+                    .add(size as i64);
             }
             (buf, result)
         })
@@ -1299,8 +1344,9 @@ impl OwnedAsyncWriter for VirtualFile {
         buf: FullSlice<Buf>,
         offset: u64,
         ctx: &RequestContext,
-    ) -> (FullSlice<Buf>, std::io::Result<()>) {
-        VirtualFile::write_all_at(self, buf, offset, ctx).await
+    ) -> std::io::Result<FullSlice<Buf>> {
+        let (buf, res) = VirtualFile::write_all_at(self, buf, offset, ctx).await;
+        res.map(|_| buf)
     }
 }
 
@@ -1547,8 +1593,7 @@ mod tests {
     where
         A: Adapter,
     {
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
         let testdir = crate::config::PageServerConf::test_repo_dir(testname);
         std::fs::create_dir_all(&testdir)?;
 
@@ -1675,8 +1720,7 @@ mod tests {
         const THREADS: usize = 100;
         const SAMPLE: [u8; SIZE] = [0xADu8; SIZE];
 
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
         let testdir = crate::config::PageServerConf::test_repo_dir("vfile_concurrency");
         std::fs::create_dir_all(&testdir)?;
 
@@ -1735,8 +1779,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_atomic_overwrite_basic() {
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
         let testdir = crate::config::PageServerConf::test_repo_dir("test_atomic_overwrite_basic");
         std::fs::create_dir_all(&testdir).unwrap();
 
@@ -1764,8 +1807,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_atomic_overwrite_preexisting_tmp() {
-        let ctx =
-            RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error).with_scope_unit_test();
+        let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error);
         let testdir =
             crate::config::PageServerConf::test_repo_dir("test_atomic_overwrite_preexisting_tmp");
         std::fs::create_dir_all(&testdir).unwrap();

pageserver/src/virtual_file/owned_buffers_io/write.rs

@@ -31,7 +31,7 @@ pub trait OwnedAsyncWriter {
         buf: FullSlice<Buf>,
         offset: u64,
         ctx: &RequestContext,
-    ) -> impl std::future::Future<Output = (FullSlice<Buf>, std::io::Result<()>)> + Send;
+    ) -> impl std::future::Future<Output = std::io::Result<FullSlice<Buf>>> + Send;
 }
 
 /// A wrapper aorund an [`OwnedAsyncWriter`] that uses a [`Buffer`] to batch
@@ -66,7 +66,6 @@ where
         buf_new: impl Fn() -> B,
         gate_guard: utils::sync::gate::GateGuard,
         ctx: &RequestContext,
-        flush_task_span: tracing::Span,
     ) -> Self {
         Self {
             writer: writer.clone(),
@@ -76,7 +75,6 @@ where
                 buf_new(),
                 gate_guard,
                 ctx.attached_child(),
-                flush_task_span,
             ),
             bytes_submitted: 0,
         }
@@ -271,12 +269,12 @@ mod tests {
             buf: FullSlice<Buf>,
             offset: u64,
             _: &RequestContext,
-        ) -> (FullSlice<Buf>, std::io::Result<()>) {
+        ) -> std::io::Result<FullSlice<Buf>> {
             self.writes
                 .lock()
                 .unwrap()
                 .push((Vec::from(&buf[..]), offset));
-            (buf, Ok(()))
+            Ok(buf)
         }
     }
 
@@ -295,7 +293,6 @@ mod tests {
             || IoBufferMut::with_capacity(2),
             gate.enter()?,
             ctx,
-            tracing::Span::none(),
         );
 
         writer.write_buffered_borrowed(b"abc", ctx).await?;

pageserver/src/virtual_file/owned_buffers_io/write/flush.rs

@@ -1,14 +1,9 @@
-use std::ops::ControlFlow;
 use std::sync::Arc;
 
-use once_cell::sync::Lazy;
-use tokio_util::sync::CancellationToken;
-use tracing::{Instrument, info, info_span, warn};
 use utils::sync::duplex;
 
 use super::{Buffer, CheapCloneForRead, OwnedAsyncWriter};
 use crate::context::RequestContext;
-use crate::virtual_file::MaybeFatalIo;
 use crate::virtual_file::owned_buffers_io::io_buf_aligned::IoBufAligned;
 use crate::virtual_file::owned_buffers_io::io_buf_ext::FullSlice;
 
@@ -123,7 +118,6 @@ where
         buf: B,
         gate_guard: utils::sync::gate::GateGuard,
         ctx: RequestContext,
-        span: tracing::Span,
     ) -> Self
     where
         B: Buffer<IoBuf = Buf> + Send + 'static,
@@ -131,14 +125,11 @@ where
         // It is fine to buffer up to only 1 message. We only 1 message in-flight at a time.
         let (front, back) = duplex::mpsc::channel(1);
 
-        let join_handle = tokio::spawn(
-            async move {
-                FlushBackgroundTask::new(back, file, gate_guard, ctx)
-                    .run(buf.flush())
-                    .await
-            }
-            .instrument(span),
-        );
+        let join_handle = tokio::spawn(async move {
+            FlushBackgroundTask::new(back, file, gate_guard, ctx)
+                .run(buf.flush())
+                .await
+        });
 
         FlushHandle {
             inner: Some(FlushHandleInner {
@@ -245,7 +236,6 @@ where
     /// The passed in slice is immediately sent back to the flush handle through the duplex channel.
     async fn run(mut self, slice: FullSlice<Buf>) -> std::io::Result<Arc<W>> {
         // Sends the extra buffer back to the handle.
-        // TODO: can this ever await and or fail? I think not.
         self.channel.send(slice).await.map_err(|_| {
             std::io::Error::new(std::io::ErrorKind::BrokenPipe, "flush handle closed early")
         })?;
@@ -261,47 +251,10 @@ where
             }
 
             // Write slice to disk at `offset`.
-            //
-            // Error handling happens according to the current policy of crashing
-            // on fatal IO errors and retrying in place otherwise (deeming all other errors retryable).
-            // (The upper layers of the Pageserver write path are not equipped to retry write errors
-            //  becasuse they often deallocate the buffers that were already written).
-            //
-            // TODO: cancellation sensitiity.
-            // Without it, if we hit a bug where retrying is never successful,
-            // then we can't shut down the timeline/tenant/pageserver cleanly because
-            // layers of the Pageserver write path are holding the gate open for EphemeralFile.
-            //
-            // TODO: use utils::backoff::retry once async closures are actually usable
-            //
-            let mut slice_storage = Some(request.slice);
-            for attempt in 1.. {
-                let result = async {
-                    if attempt > 1 {
-                        info!("retrying flush");
-                    }
-                    let slice = slice_storage.take().expect(
-                        "likely previous invocation of this future didn't get polled to completion",
-                    );
-                    let (slice, res) = self.writer.write_all_at(slice, request.offset, &self.ctx).await;
-                    slice_storage = Some(slice);
-                    let res = res.maybe_fatal_err("owned_buffers_io flush");
-                    let Err(err) = res else {
-                        return ControlFlow::Break(());
-                    };
-                    warn!(%err, "error flushing buffered writer buffer to disk, retrying after backoff");
-                    static NO_CANCELLATION: Lazy<CancellationToken> = Lazy::new(CancellationToken::new);
-                    utils::backoff::exponential_backoff(attempt, 1.0, 10.0, &NO_CANCELLATION).await;
-                    ControlFlow::Continue(())
-                }
-                .instrument(info_span!("flush_attempt", %attempt))
-                .await;
-                match result {
-                    ControlFlow::Break(()) => break,
-                    ControlFlow::Continue(()) => continue,
-                }
-            }
-            let slice = slice_storage.expect("loop must have run at least once");
+            let slice = self
+                .writer
+                .write_all_at(request.slice, request.offset, &self.ctx)
+                .await?;
 
             #[cfg(test)]
             {

pgxn/neon/file_cache.c

@@ -1195,11 +1195,9 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 				state = GET_STATE(entry, chunk_offs + i);
 				if (state == PENDING) {
 					SET_STATE(entry, chunk_offs + i, REQUESTED);
-				} else if (state == UNAVAILABLE) {
+				} else if (state != REQUESTED) {
 					SET_STATE(entry, chunk_offs + i, PENDING);
 					break;
-				} else if (state == AVAILABLE) {
-					break;
 				}
 				if (!sleeping)
 				{

pgxn/neon/libpagestore.c

@@ -16,8 +16,6 @@
 
 #include <math.h>
 
-#include "libpq-int.h"
-
 #include "access/xlog.h"
 #include "common/hashfn.h"
 #include "fmgr.h"
@@ -817,10 +815,9 @@ retry:
 			get_socket_stats(PQsocket(pageserver_conn), &sndbuf, &recvbuf);
 
 			neon_shard_log(shard_no, LOG,
-						   "no response received from pageserver for %0.3f s, still waiting (sent " UINT64_FORMAT " requests, received " UINT64_FORMAT " responses) (socket sndbuf=%d recvbuf=%d) (conn start=%d end=%d)",
+						   "no response received from pageserver for %0.3f s, still waiting (sent " UINT64_FORMAT " requests, received " UINT64_FORMAT " responses) (socket sndbuf=%d recvbuf=%d)",
 						   INSTR_TIME_GET_DOUBLE(since_start),
-						   shard->nrequests_sent, shard->nresponses_received, sndbuf, recvbuf,
-				           pageserver_conn->inStart, pageserver_conn->inEnd);
+						   shard->nrequests_sent, shard->nresponses_received, sndbuf, recvbuf);
 			shard->receive_last_log_time = now;
 			shard->receive_logged = true;
 		}

pgxn/neon/neon.c

@@ -457,15 +457,6 @@ _PG_init(void)
 							PGC_SIGHUP,
 							0,
 							NULL, NULL, NULL);
-	DefineCustomBoolVariable(
-							"neon.disable_wal_prevlink_checks",
-							"Disable validation of prev link in WAL records",
-							NULL,
-							&disable_wal_prev_lsn_checks,
-							false,
-							PGC_SIGHUP,
-							0,
-							NULL, NULL, NULL);
 
 	DefineCustomBoolVariable(
 							"neon.allow_replica_misconfig",

pgxn/neon/neon.h

@@ -23,7 +23,6 @@ extern char *wal_acceptors_list;
 extern int	wal_acceptor_reconnect_timeout;
 extern int	wal_acceptor_connection_timeout;
 extern int	readahead_getpage_pull_timeout_ms;
-extern bool	disable_wal_prev_lsn_checks;
 
 #if PG_MAJORVERSION_NUM >= 17
 extern uint32		WAIT_EVENT_NEON_LFC_MAINTENANCE;

pgxn/neon/pagestore_smgr.c

@@ -76,10 +76,6 @@
 #include "access/xlogrecovery.h"
 #endif
 
-#if PG_VERSION_NUM < 160000
-typedef PGAlignedBlock PGIOAlignedBlock;
-#endif
-
 /*
  * If DEBUG_COMPARE_LOCAL is defined, we pass through all the SMGR API
  * calls to md.c, and *also* do the calls to the Page Server. On every
@@ -1807,7 +1803,7 @@ static XLogRecPtr
 log_newpage_copy(NRelFileInfo * rinfo, ForkNumber forkNum, BlockNumber blkno,
 				 Page page, bool page_std)
 {
-	PGIOAlignedBlock copied_buffer;
+	PGAlignedBlock copied_buffer;
 
 	memcpy(copied_buffer.data, page, BLCKSZ);
 	return log_newpage(rinfo, forkNum, blkno, copied_buffer.data, page_std);
@@ -1824,7 +1820,7 @@ static XLogRecPtr
 log_newpages_copy(NRelFileInfo * rinfo, ForkNumber forkNum, BlockNumber blkno,
 				  BlockNumber nblocks, Page *pages, bool page_std)
 {
-	PGIOAlignedBlock copied_buffer[XLR_MAX_BLOCK_ID];
+	PGAlignedBlock copied_buffer[XLR_MAX_BLOCK_ID];
 	BlockNumber	blknos[XLR_MAX_BLOCK_ID];
 	Page		pageptrs[XLR_MAX_BLOCK_ID];
 	int			nregistered = 0;
@@ -1862,7 +1858,7 @@ log_newpages_copy(NRelFileInfo * rinfo, ForkNumber forkNum, BlockNumber blkno,
 static bool
 PageIsEmptyHeapPage(char *buffer)
 {
-	PGIOAlignedBlock empty_page;
+	PGAlignedBlock empty_page;
 
 	PageInit((Page) empty_page.data, BLCKSZ, 0);
 
@@ -2851,7 +2847,7 @@ static void
 neon_zeroextend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blocknum,
 				int nblocks, bool skipFsync)
 {
-	const PGIOAlignedBlock buffer = {0};
+	const PGAlignedBlock buffer = {0};
 	int			remblocks = nblocks;
 	XLogRecPtr	lsn = 0;
 
@@ -3393,16 +3389,15 @@ neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void *buffer
 	if (forkNum == MAIN_FORKNUM && IS_LOCAL_REL(reln))
 	{
 		char		pageserver_masked[BLCKSZ];
-		PGIOAlignedBlock mdbuf;
-		PGIOAlignedBlock mdbuf_masked;
-		XLogRecPtr  request_lsn = request_lsns.request_lsn;
+		char		mdbuf[BLCKSZ];
+		char		mdbuf_masked[BLCKSZ];
 
-		mdread(reln, forkNum, blkno, mdbuf.data);
+		mdread(reln, forkNum, blkno, mdbuf);
 
 		memcpy(pageserver_masked, buffer, BLCKSZ);
-		memcpy(mdbuf_masked.data, mdbuf.data, BLCKSZ);
+		memcpy(mdbuf_masked, mdbuf, BLCKSZ);
 
-		if (PageIsNew((Page) mdbuf.data))
+		if (PageIsNew((Page) mdbuf))
 		{
 			if (!PageIsNew((Page) pageserver_masked))
 			{
@@ -3421,41 +3416,41 @@ neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void *buffer
 				 RelFileInfoFmt(InfoFromSMgrRel(reln)),
 				 forkNum,
 				 (uint32) (request_lsn >> 32), (uint32) request_lsn,
-				 hexdump_page(mdbuf.data));
+				 hexdump_page(mdbuf));
 		}
-		else if (PageGetSpecialSize(mdbuf.data) == 0)
+		else if (PageGetSpecialSize(mdbuf) == 0)
 		{
 			/* assume heap */
-			RmgrTable[RM_HEAP_ID].rm_mask(mdbuf_masked.data, blkno);
+			RmgrTable[RM_HEAP_ID].rm_mask(mdbuf_masked, blkno);
 			RmgrTable[RM_HEAP_ID].rm_mask(pageserver_masked, blkno);
 
-			if (memcmp(mdbuf_masked.data, pageserver_masked, BLCKSZ) != 0)
+			if (memcmp(mdbuf_masked, pageserver_masked, BLCKSZ) != 0)
 			{
 				neon_log(PANIC, "heap buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n",
 					 blkno,
 					 RelFileInfoFmt(InfoFromSMgrRel(reln)),
 					 forkNum,
 					 (uint32) (request_lsn >> 32), (uint32) request_lsn,
-					 hexdump_page(mdbuf_masked.data),
+					 hexdump_page(mdbuf_masked),
 					 hexdump_page(pageserver_masked));
 			}
 		}
-		else if (PageGetSpecialSize(mdbuf.data) == MAXALIGN(sizeof(BTPageOpaqueData)))
+		else if (PageGetSpecialSize(mdbuf) == MAXALIGN(sizeof(BTPageOpaqueData)))
 		{
-			if (((BTPageOpaqueData *) PageGetSpecialPointer(mdbuf.data))->btpo_cycleid < MAX_BT_CYCLE_ID)
+			if (((BTPageOpaqueData *) PageGetSpecialPointer(mdbuf))->btpo_cycleid < MAX_BT_CYCLE_ID)
 			{
 				/* assume btree */
-				RmgrTable[RM_BTREE_ID].rm_mask(mdbuf_masked.data, blkno);
+				RmgrTable[RM_BTREE_ID].rm_mask(mdbuf_masked, blkno);
 				RmgrTable[RM_BTREE_ID].rm_mask(pageserver_masked, blkno);
 
-				if (memcmp(mdbuf_masked.data, pageserver_masked, BLCKSZ) != 0)
+				if (memcmp(mdbuf_masked, pageserver_masked, BLCKSZ) != 0)
 				{
 					neon_log(PANIC, "btree buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n",
 						 blkno,
 						 RelFileInfoFmt(InfoFromSMgrRel(reln)),
 						 forkNum,
 						 (uint32) (request_lsn >> 32), (uint32) request_lsn,
-						 hexdump_page(mdbuf_masked.data),
+						 hexdump_page(mdbuf_masked),
 						 hexdump_page(pageserver_masked));
 				}
 			}
@@ -3547,85 +3542,77 @@ neon_readv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
 	prefetch_pump_state(false);
 
 #ifdef DEBUG_COMPARE_LOCAL
-	if (forknum == MAIN_FORKNUM && IS_LOCAL_REL(reln))
+	if (forkNum == MAIN_FORKNUM && IS_LOCAL_REL(reln))
 	{
 		char		pageserver_masked[BLCKSZ];
-		PGIOAlignedBlock mdbuf;
-		PGIOAlignedBlock mdbuf_masked;
-		XLogRecPtr  request_lsn = request_lsns->request_lsn;
+		char		mdbuf[BLCKSZ];
+		char		mdbuf_masked[BLCKSZ];
 
 		for (int i = 0; i < nblocks; i++)
 		{
-			BlockNumber blkno = blocknum + i;
-			if (!BITMAP_ISSET(read, i))
-				continue;
-
 #if PG_MAJORVERSION_NUM >= 17
-			{
-				void* mdbuffers[1] = { mdbuf.data };
-				mdreadv(reln, forknum, blkno, mdbuffers, 1);
-			}
+			mdreadv(reln, forkNum, blkno + i, &mdbuf, 1);
 #else
-			mdread(reln, forknum, blkno, mdbuf.data);
+			mdread(reln, forkNum, blkno + i, mdbuf);
 #endif
 
-			memcpy(pageserver_masked, buffers[i], BLCKSZ);
-			memcpy(mdbuf_masked.data, mdbuf.data, BLCKSZ);
+			memcpy(pageserver_masked, buffer, BLCKSZ);
+			memcpy(mdbuf_masked, mdbuf, BLCKSZ);
 
-			if (PageIsNew((Page) mdbuf.data))
+			if (PageIsNew((Page) mdbuf))
 			{
 				if (!PageIsNew((Page) pageserver_masked))
 				{
 					neon_log(PANIC, "page is new in MD but not in Page Server at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n",
 						 blkno,
 						 RelFileInfoFmt(InfoFromSMgrRel(reln)),
-						 forknum,
+						 forkNum,
 						 (uint32) (request_lsn >> 32), (uint32) request_lsn,
-						 hexdump_page(buffers[i]));
+						 hexdump_page(buffer));
 				}
 			}
-			else if (PageIsNew((Page) buffers[i]))
+			else if (PageIsNew((Page) buffer))
 			{
 				neon_log(PANIC, "page is new in Page Server but not in MD at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n",
 					 blkno,
 					 RelFileInfoFmt(InfoFromSMgrRel(reln)),
-					 forknum,
+					 forkNum,
 					 (uint32) (request_lsn >> 32), (uint32) request_lsn,
-					 hexdump_page(mdbuf.data));
+					 hexdump_page(mdbuf));
 			}
-			else if (PageGetSpecialSize(mdbuf.data) == 0)
+			else if (PageGetSpecialSize(mdbuf) == 0)
 			{
 				/* assume heap */
-				RmgrTable[RM_HEAP_ID].rm_mask(mdbuf_masked.data, blkno);
+				RmgrTable[RM_HEAP_ID].rm_mask(mdbuf_masked, blkno);
 				RmgrTable[RM_HEAP_ID].rm_mask(pageserver_masked, blkno);
 
-				if (memcmp(mdbuf_masked.data, pageserver_masked, BLCKSZ) != 0)
+				if (memcmp(mdbuf_masked, pageserver_masked, BLCKSZ) != 0)
 				{
 					neon_log(PANIC, "heap buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n",
 						 blkno,
 						 RelFileInfoFmt(InfoFromSMgrRel(reln)),
-						 forknum,
+						 forkNum,
 						 (uint32) (request_lsn >> 32), (uint32) request_lsn,
-						 hexdump_page(mdbuf_masked.data),
+						 hexdump_page(mdbuf_masked),
 						 hexdump_page(pageserver_masked));
 				}
 			}
-			else if (PageGetSpecialSize(mdbuf.data) == MAXALIGN(sizeof(BTPageOpaqueData)))
+			else if (PageGetSpecialSize(mdbuf) == MAXALIGN(sizeof(BTPageOpaqueData)))
 			{
-				if (((BTPageOpaqueData *) PageGetSpecialPointer(mdbuf.data))->btpo_cycleid < MAX_BT_CYCLE_ID)
+				if (((BTPageOpaqueData *) PageGetSpecialPointer(mdbuf))->btpo_cycleid < MAX_BT_CYCLE_ID)
 				{
 					/* assume btree */
-					RmgrTable[RM_BTREE_ID].rm_mask(mdbuf_masked.data, blkno);
+					RmgrTable[RM_BTREE_ID].rm_mask(mdbuf_masked, blkno);
 					RmgrTable[RM_BTREE_ID].rm_mask(pageserver_masked, blkno);
 	
-					if (memcmp(mdbuf_masked.data, pageserver_masked, BLCKSZ) != 0)
+					if (memcmp(mdbuf_masked, pageserver_masked, BLCKSZ) != 0)
 					{
 						neon_log(PANIC, "btree buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n",
 							 blkno,
 							 RelFileInfoFmt(InfoFromSMgrRel(reln)),
-							 forknum,
+							 forkNum,
 							 (uint32) (request_lsn >> 32), (uint32) request_lsn,
-							 hexdump_page(mdbuf_masked.data),
+							 hexdump_page(mdbuf_masked),
 							 hexdump_page(pageserver_masked));
 					}
 				}
@@ -3677,7 +3664,6 @@ neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const vo
 	switch (reln->smgr_relpersistence)
 	{
 		case 0:
-#ifndef DEBUG_COMPARE_LOCAL
 			/* This is a bit tricky. Check if the relation exists locally */
 			if (mdexists(reln, forknum))
 			{
@@ -3696,7 +3682,6 @@ neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const vo
 				 */
 				return;
 			}
-#endif
 			break;
 
 		case RELPERSISTENCE_PERMANENT:
@@ -3747,7 +3732,6 @@ neon_writev(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno,
 	switch (reln->smgr_relpersistence)
 	{
 		case 0:
-#ifndef DEBUG_COMPARE_LOCAL
 			/* This is a bit tricky. Check if the relation exists locally */
 			if (mdexists(reln, forknum))
 			{
@@ -3763,7 +3747,6 @@ neon_writev(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno,
 				 */
 				return;
 			}
-#endif
 			break;
 
 		case RELPERSISTENCE_PERMANENT:
@@ -3785,7 +3768,7 @@ neon_writev(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno,
 
 #ifdef DEBUG_COMPARE_LOCAL
 	if (IS_LOCAL_REL(reln))
-		mdwritev(reln, forknum, blkno, buffers, nblocks, skipFsync);
+		mdwritev(reln, forknum, blocknum, &buffer, 1, skipFsync);
 #endif
 }