pageserver: add DeletionNotify to wait for deletion queue execution

.github/workflows/_push-to-acr.yml

@@ -0,0 +1,56 @@
+name: Push images to ACR
+on:
+  workflow_call:
+    inputs:
+      client_id:
+        description: Client ID of Azure managed identity or Entra app
+        required: true
+        type: string
+      image_tag:
+        description: Tag for the container image
+        required: true
+        type: string
+      images:
+        description: Images to push
+        required: true
+        type: string
+      registry_name:
+        description: Name of the container registry
+        required: true
+        type: string
+      subscription_id:
+        description: Azure subscription ID
+        required: true
+        type: string
+      tenant_id:
+        description: Azure tenant ID
+        required: true
+        type: string
+
+jobs:
+  push-to-acr:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read  # This is required for actions/checkout
+      id-token: write # This is required for Azure Login to work.
+
+    steps:
+      - name: Azure login
+        uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a  # @v2.1.1
+        with:
+          client-id: ${{ inputs.client_id }}
+          subscription-id: ${{ inputs.subscription_id }}
+          tenant-id: ${{ inputs.tenant_id }}
+
+      - name: Login to ACR
+        run: |
+          az acr login --name=${{ inputs.registry_name }}
+
+      - name: Copy docker images to ACR ${{ inputs.registry_name }}
+        run: |
+          images='${{ inputs.images }}'
+          for image in ${images}; do
+            docker buildx imagetools create \
+              -t ${{ inputs.registry_name }}.azurecr.io/neondatabase/${image}:${{ inputs.image_tag }} \
+                                                        neondatabase/${image}:${{ inputs.image_tag }}
+          done

.github/workflows/_push-to-container-registry.yml

@@ -1,101 +0,0 @@
-name: Push images to Container Registry
-on:
-  workflow_call:
-    inputs:
-      # Example: {"docker.io/neondatabase/neon:13196061314":["369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:13196061314","neoneastus2.azurecr.io/neondatabase/neon:13196061314"]}
-      image-map:
-        description: JSON map of images, mapping from a source image to an array of target images that should be pushed.
-        required: true
-        type: string
-      aws-region:
-        description: AWS region to log in to. Required when pushing to ECR.
-        required: false
-        type: string
-      aws-account-ids:
-        description: Comma separated AWS account IDs to log in to for pushing to ECR. Required when pushing to ECR.
-        required: false
-        type: string
-      azure-client-id:
-        description: Client ID of Azure managed identity or Entra app. Required when pushing to ACR.
-        required: false
-        type: string
-      azure-subscription-id:
-        description: Azure subscription ID. Required when pushing to ACR.
-        required: false
-        type: string
-      azure-tenant-id:
-        description: Azure tenant ID. Required when pushing to ACR.
-        required: false
-        type: string
-      acr-registry-name:
-        description: ACR registry name. Required when pushing to ACR.
-        required: false
-        type: string
-    secrets:
-      docker-hub-username:
-        description: Docker Hub username. Required when pushing to Docker Hub.
-        required: false
-      docker-hub-password:
-        description: Docker Hub password. Required when pushing to Docker Hub.
-        required: false
-      aws-role-to-assume:
-        description: AWS role to assume. Required when pushing to ECR.
-        required: false
-
-permissions: {}
-
-defaults:
-  run:
-    shell: bash -euo pipefail {0}
-
-jobs:
-  push-to-container-registry:
-    runs-on: ubuntu-22.04
-    permissions:
-      id-token: write  # Required for aws/azure login
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          sparse-checkout: scripts/push_with_image_map.py
-          sparse-checkout-cone-mode: false
-
-      - name: Print image-map
-        run: echo '${{ inputs.image-map }}' | jq
-
-      - name: Configure AWS credentials
-        if: contains(inputs.image-map, 'amazonaws.com/')
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: "${{ inputs.aws-region }}"
-          role-to-assume: "${{ secrets.aws-role-to-assume }}"
-          role-duration-seconds: 3600
-
-      - name: Login to ECR
-        if: contains(inputs.image-map, 'amazonaws.com/')
-        uses: aws-actions/amazon-ecr-login@v2
-        with:
-          registries: "${{ inputs.aws-account-ids }}"
-
-      - name: Configure Azure credentials
-        if: contains(inputs.image-map, 'azurecr.io/')
-        uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a  # @v2.1.1
-        with:
-          client-id: ${{ inputs.azure-client-id }}
-          subscription-id: ${{ inputs.azure-subscription-id }}
-          tenant-id: ${{ inputs.azure-tenant-id }}
-
-      - name: Login to ACR
-        if: contains(inputs.image-map, 'azurecr.io/')
-        run: |
-          az acr login --name=${{ inputs.acr-registry-name }}
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.docker-hub-username }}
-          password: ${{ secrets.docker-hub-password }}
-
-      - name: Copy docker images to target registries
-        run: python scripts/push_with_image_map.py
-        env:
-          IMAGE_MAP: ${{ inputs.image-map }}

.github/workflows/build_and_test.yml

@@ -497,7 +497,7 @@ jobs:
 
   trigger-e2e-tests:
     if: ${{ !github.event.pull_request.draft || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') || github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' }}
-    needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, tag ]
+    needs: [ check-permissions, promote-images-dev, tag ]
     uses: ./.github/workflows/trigger-e2e-tests.yml
     secrets: inherit
 
@@ -571,6 +571,21 @@ jobs:
                                              neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-x64 \
                                              neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-arm64
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Push multi-arch image to ECR
+        run: |
+          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{ needs.tag.outputs.build-tag }} \
+                                                                                neondatabase/neon:${{ needs.tag.outputs.build-tag }}
+
   compute-node-image-arch:
     needs: [ check-permissions, build-build-tools-image, tag ]
     permissions:
@@ -617,6 +632,16 @@ jobs:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
       - uses: docker/login-action@v3
         with:
           registry: cache.neon.build
@@ -704,6 +729,21 @@ jobs:
                                              neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
                                              neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Push multi-arch compute-node-${{ matrix.version.pg }} image to ECR
+        run: |
+          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
+                                                                                neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}
+
   vm-compute-node-image:
     needs: [ check-permissions, tag, compute-node-image ]
     runs-on: [ self-hosted, large ]
@@ -836,109 +876,133 @@ jobs:
           docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml logs || true
           docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml down
 
-  generate-image-maps:
-    needs: [ tag ]
+  promote-images-dev:
+    needs: [ check-permissions, tag, vm-compute-node-image, neon-image ]
     runs-on: ubuntu-22.04
-    outputs:
-      neon-dev: ${{ steps.generate.outputs.neon-dev }}
-      neon-prod: ${{ steps.generate.outputs.neon-prod }}
-      compute-dev: ${{ steps.generate.outputs.compute-dev }}
-      compute-prod: ${{ steps.generate.outputs.compute-prod }}
+
+    permissions:
+      id-token: write # aws-actions/configure-aws-credentials
+      statuses: write
+      contents: read
+
+    env:
+      VERSIONS: v14 v15 v16 v17
+
     steps:
-      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
         with:
-          sparse-checkout: scripts/generate_image_maps.py
-          sparse-checkout-cone-mode: false
+          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
-      - name: Generate Image Maps
-        id: generate
-        run: python scripts/generate_image_maps.py
-        env:
-          BUILD_TAG: "${{ needs.tag.outputs.build-tag }}"
-          BRANCH: "${{ github.ref_name }}"
-          DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}"
-          PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}"
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
 
-  push-neon-image-dev:
-    needs: [ generate-image-maps, neon-image ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.neon-dev }}'
-      aws-region: eu-central-1
-      aws-account-ids: "369495373322"
-      azure-client-id: ${{ vars.AZURE_DEV_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_DEV_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ vars.DEV_AWS_OIDC_ROLE_ARN }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
 
-  push-compute-image-dev:
-    needs: [ generate-image-maps, vm-compute-node-image ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.compute-dev }}'
-      aws-region: eu-central-1
-      aws-account-ids: "369495373322"
-      azure-client-id: ${{ vars.AZURE_DEV_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_DEV_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ vars.DEV_AWS_OIDC_ROLE_ARN }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+      - name: Copy vm-compute-node images to ECR
+        run: |
+          for version in ${VERSIONS}; do
+            docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} \
+                                               neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+          done
 
-  push-neon-image-prod:
-    if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
-    needs: [ generate-image-maps, neon-image, test-images ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.neon-prod }}'
-      aws-region: eu-central-1
-      aws-account-ids: "093970136003"
-      azure-client-id: ${{ vars.AZURE_PROD_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_PROD_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ secrets.PROD_GHA_OIDC_ROLE }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+  promote-images-prod:
+    needs: [ check-permissions, tag, test-images, promote-images-dev ]
+    runs-on: ubuntu-22.04
+    if: github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
 
-  push-compute-image-prod:
-    if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
-    needs: [ generate-image-maps, vm-compute-node-image, test-images ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.compute-prod }}'
-      aws-region: eu-central-1
-      aws-account-ids: "093970136003"
-      azure-client-id: ${{ vars.AZURE_PROD_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_PROD_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ secrets.PROD_GHA_OIDC_ROLE }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+    permissions:
+      id-token: write # aws-actions/configure-aws-credentials
+      statuses: write
+      contents: read
 
-  # This is a bit of a special case so we're not using a generated image map.
-  add-latest-tag-to-neon-extensions-test-image:
+    env:
+      VERSIONS: v14 v15 v16 v17
+
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+
+      - name: Add latest tag to images
+        if: github.ref_name == 'main'
+        run: |
+          for repo in neondatabase 369495373322.dkr.ecr.eu-central-1.amazonaws.com; do
+            docker buildx imagetools create -t $repo/neon:latest \
+                                               $repo/neon:${{ needs.tag.outputs.build-tag }}
+
+            for version in ${VERSIONS}; do
+              docker buildx imagetools create -t $repo/compute-node-${version}:latest \
+                                                 $repo/compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+
+              docker buildx imagetools create -t $repo/vm-compute-node-${version}:latest \
+                                                 $repo/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+            done
+          done
+          docker buildx imagetools create -t neondatabase/neon-test-extensions-v16:latest \
+                                              neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }}
+
+      - name: Configure AWS-prod credentials
+        if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          mask-aws-account-id: true
+          role-to-assume: ${{ secrets.PROD_GHA_OIDC_ROLE }}
+
+      - name: Login to prod ECR
+        uses: docker/login-action@v3
+        if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+        with:
+          registry: 093970136003.dkr.ecr.eu-central-1.amazonaws.com
+
+      - name: Copy all images to prod ECR
+        if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+        run: |
+          for image in neon {vm-,}compute-node-{v14,v15,v16,v17}; do
+            docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} \
+                                               369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }}
+          done
+
+  push-to-acr-dev:
     if: github.ref_name == 'main'
-    needs: [ tag, compute-node-image ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
+    needs: [ tag, promote-images-dev ]
+    uses: ./.github/workflows/_push-to-acr.yml
     with:
-      image-map: |
-        {
-          "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v16:latest"],
-          "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.tag.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v17:latest"]
-        }
-    secrets:
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+      client_id: ${{ vars.AZURE_DEV_CLIENT_ID }}
+      image_tag: ${{ needs.tag.outputs.build-tag }}
+      images: neon vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17
+      registry_name: ${{ vars.AZURE_DEV_REGISTRY_NAME }}
+      subscription_id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }}
+      tenant_id: ${{ vars.AZURE_TENANT_ID }}
+
+  push-to-acr-prod:
+    if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+    needs: [ tag, promote-images-prod ]
+    uses: ./.github/workflows/_push-to-acr.yml
+    with:
+      client_id: ${{ vars.AZURE_PROD_CLIENT_ID }}
+      image_tag: ${{ needs.tag.outputs.build-tag }}
+      images: neon vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17
+      registry_name: ${{ vars.AZURE_PROD_REGISTRY_NAME }}
+      subscription_id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }}
+      tenant_id: ${{ vars.AZURE_TENANT_ID }}
 
   trigger-custom-extensions-build-and-wait:
     needs: [ check-permissions, tag ]
@@ -1020,7 +1084,7 @@ jobs:
           exit 1
 
   deploy:
-    needs: [ check-permissions, push-neon-image-prod, push-compute-image-prod, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait ]
+    needs: [ check-permissions, promote-images-prod, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait, push-to-acr-dev, push-to-acr-prod ]
     # `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod`
     if: (github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute') && !failure() && !cancelled()
     permissions:
@@ -1273,7 +1337,7 @@ jobs:
           done
 
   pin-build-tools-image:
-    needs: [ build-build-tools-image, push-compute-image-prod, push-neon-image-prod, build-and-test-locally ]
+    needs: [ build-build-tools-image, promote-images-prod, build-and-test-locally ]
     if: github.ref_name == 'main'
     uses: ./.github/workflows/pin-build-tools-image.yml
     with:
@@ -1298,8 +1362,7 @@ jobs:
       - check-codestyle-rust
       - check-dependencies-rust
       - files-changed
-      - push-compute-image-dev
-      - push-neon-image-dev
+      - promote-images-dev
       - test-images
       - trigger-custom-extensions-build-and-wait
     runs-on: ubuntu-22.04
@@ -1316,7 +1379,6 @@ jobs:
           || needs.check-codestyle-python.result == 'skipped'
           || needs.check-codestyle-rust.result == 'skipped'
           || needs.files-changed.result == 'skipped'
-          || needs.push-compute-image-dev.result == 'skipped'
-          || needs.push-neon-image-dev.result == 'skipped'
+          || needs.promote-images-dev.result == 'skipped'
           || needs.test-images.result == 'skipped'
           || needs.trigger-custom-extensions-build-and-wait.result == 'skipped'

.github/workflows/trigger-e2e-tests.yml

@@ -15,14 +15,7 @@ env:
   E2E_CONCURRENCY_GROUP: ${{ github.repository }}-e2e-tests-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
 
 jobs:
-  check-permissions:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }}
-    uses: ./.github/workflows/check-permissions.yml
-    with:
-      github-event-name: ${{ github.event_name }}
-
   cancel-previous-e2e-tests:
-    needs: [ check-permissions ]
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-22.04
 
@@ -36,7 +29,6 @@ jobs:
               --field concurrency_group="${{ env.E2E_CONCURRENCY_GROUP }}"
 
   tag:
-    needs: [ check-permissions ]
     runs-on: ubuntu-22.04
     outputs:
       build-tag: ${{ steps.build-tag.outputs.tag }}
@@ -76,7 +68,7 @@ jobs:
       GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
       TAG: ${{ needs.tag.outputs.build-tag }}
     steps:
-      - name: Wait for `push-{neon,compute}-image-dev` job to finish
+      - name: Wait for `promote-images-dev` job to finish
         # It's important to have a timeout here, the script in the step can run infinitely
         timeout-minutes: 60
         run: |
@@ -87,20 +79,20 @@ jobs:
           # For PRs we use the run id as the tag
           BUILD_AND_TEST_RUN_ID=${TAG}
           while true; do
-            gh run --repo ${GITHUB_REPOSITORY} view ${BUILD_AND_TEST_RUN_ID} --json jobs --jq '[.jobs[] | select((.name | startswith("push-neon-image-dev")) or (.name | startswith("push-compute-image-dev"))) | {"name": .name, "conclusion": .conclusion, "url": .url}]' > jobs.json
-            if [ $(jq '[.[] | select(.conclusion == "success")]' jobs.json) -eq 2 ]; then
-              break
-            fi
-            jq -c '.[]' jobs.json | while read -r job; do
-              case $(echo $job | jq .conclusion) in
-                failure | cancelled | skipped)
-                  echo "The '$(echo $job | jq .name)' job didn't succeed: '$(echo $job | jq .conclusion)'. See log in '$(echo $job | jq .url)' Exiting..."
-                  exit 1
-                  ;;
-              esac
-            done
-            echo "The 'push-{neon,compute}-image-dev' jobs haven't succeeded yet. Waiting..."
-            sleep 60
+            conclusion=$(gh run --repo ${GITHUB_REPOSITORY} view ${BUILD_AND_TEST_RUN_ID} --json jobs --jq '.jobs[] | select(.name == "promote-images-dev") | .conclusion')
+            case "$conclusion" in
+              success)
+                break
+                ;;
+              failure | cancelled | skipped)
+                echo "The 'promote-images-dev' job didn't succeed: '${conclusion}'. Exiting..."
+                exit 1
+                ;;
+              *)
+                echo "The 'promote-images-dev' hasn't succeed yet. Waiting..."
+                sleep 60
+                ;;
+            esac
           done
 
       - name: Set e2e-platforms

Cargo.lock

@@ -786,7 +786,7 @@ dependencies = [
 [[package]]
 name = "azure_core"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#985db729824be324ed11527e45de722250028d9e"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -815,7 +815,7 @@ dependencies = [
 [[package]]
 name = "azure_identity"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#985db729824be324ed11527e45de722250028d9e"
 dependencies = [
  "async-lock",
  "async-trait",
@@ -834,7 +834,7 @@ dependencies = [
 [[package]]
 name = "azure_storage"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#985db729824be324ed11527e45de722250028d9e"
 dependencies = [
  "RustyXML",
  "async-lock",
@@ -852,7 +852,7 @@ dependencies = [
 [[package]]
 name = "azure_storage_blobs"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#985db729824be324ed11527e45de722250028d9e"
 dependencies = [
  "RustyXML",
  "azure_core",
@@ -872,7 +872,7 @@ dependencies = [
 [[package]]
 name = "azure_svc_blobstorage"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#985db729824be324ed11527e45de722250028d9e"
 dependencies = [
  "azure_core",
  "bytes",

Dockerfile

@@ -10,28 +10,6 @@ ARG STABLE_PG_VERSION=16
 ARG DEBIAN_VERSION=bookworm
 ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim
 
-# Here are the INDEX DIGESTS for the images we use.
-# You can get them following next steps for now:
-# 1. Get an authentication token from DockerHub:
-#    TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/debian:pull" | jq -r .token)
-# 2. Using that token, query index for the given tag:
-#    curl -s -H "Authorization: Bearer $TOKEN" \
-#       -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
-#       "https://registry.hub.docker.com/v2/library/debian/manifests/bullseye-slim" \
-#       -I | grep -i docker-content-digest
-# 3. As a next step, TODO(fedordikarev): create script and schedule workflow to run these checks
-#    and updates on regular bases and in automated way.
-ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-
-# Here we use ${var/search/replace} syntax, to check
-# if base image is one of the images, we pin image index for.
-# If var will match one the known images, we will replace it with the known sha.
-# If no match, than value will be unaffected, and will process with no-pinned image.
-ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA}
-
 # Build Postgres
 FROM $REPOSITORY/$IMAGE:$TAG AS pg-build
 WORKDIR /home/nonroot
@@ -81,7 +59,7 @@ RUN set -e \
 
 # Build final image
 #
-FROM $BASE_IMAGE_SHA
+FROM debian:${DEBIAN_FLAVOR}
 ARG DEFAULT_PG_VERSION
 WORKDIR /data
 
@@ -134,3 +112,4 @@ EXPOSE 6400
 EXPOSE 9898
 
 CMD ["/usr/local/bin/pageserver", "-D", "/data/.neon"]
+

build-tools.Dockerfile

@@ -1,29 +1,6 @@
 ARG DEBIAN_VERSION=bookworm
-ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim
 
-# Here are the INDEX DIGESTS for the images we use.
-# You can get them following next steps for now:
-# 1. Get an authentication token from DockerHub:
-#    TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/debian:pull" | jq -r .token)
-# 2. Using that token, query index for the given tag:
-#    curl -s -H "Authorization: Bearer $TOKEN" \
-#       -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
-#       "https://registry.hub.docker.com/v2/library/debian/manifests/bullseye-slim" \
-#       -I | grep -i docker-content-digest
-# 3. As a next step, TODO(fedordikarev): create script and schedule workflow to run these checks
-#    and updates on regular bases and in automated way.
-ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-
-# Here we use ${var/search/replace} syntax, to check
-# if base image is one of the images, we pin image index for.
-# If var will match one the known images, we will replace it with the known sha.
-# If no match, than value will be unaffected, and will process with no-pinned image.
-ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA}
-
-FROM $BASE_IMAGE_SHA AS pgcopydb_builder
+FROM debian:bookworm-slim AS pgcopydb_builder
 ARG DEBIAN_VERSION
 
 # Use strict mode for bash to catch errors early
@@ -32,7 +9,7 @@ SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 # By default, /bin/sh used in debian images will treat '\n' as eol,
 # but as we use bash as SHELL, and built-in echo in bash requires '-e' flag for that.
 RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused=on\ntimeout=15\ntries=5\nretry-on-host-error=on\n" > /root/.wgetrc && \
+    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc && \
     echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc
 
 COPY build_tools/patches/pgcopydbv017.patch /pgcopydbv017.patch
@@ -81,7 +58,7 @@ RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
         mkdir -p mkdir -p /pgcopydb/lib && touch /pgcopydb/lib/libpq.so.5; \
     fi
 
-FROM $BASE_IMAGE_SHA AS build_tools
+FROM debian:${DEBIAN_VERSION}-slim AS build_tools
 ARG DEBIAN_VERSION
 
 # Add nonroot user
@@ -98,7 +75,7 @@ COPY --from=pgcopydb_builder /usr/lib/postgresql/16/bin/pgcopydb /pgcopydb/bin/p
 COPY --from=pgcopydb_builder /pgcopydb/lib/libpq.so.5 /pgcopydb/lib/libpq.so.5
 
 RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused=on\ntimeout=15\ntries=5\nretry-on-host-error=on\n" > /root/.wgetrc && \
+    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc && \
     echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc
 
 # System deps
@@ -161,8 +138,7 @@ RUN curl -fsSL \
     --output sql_exporter.tar.gz \
     && mkdir /tmp/sql_exporter \
     && tar xzvf sql_exporter.tar.gz -C /tmp/sql_exporter --strip-components=1 \
-    && mv /tmp/sql_exporter/sql_exporter /usr/local/bin/sql_exporter \
-    && rm sql_exporter.tar.gz
+    && mv /tmp/sql_exporter/sql_exporter /usr/local/bin/sql_exporter
 
 # protobuf-compiler (protoc)
 ENV PROTOC_VERSION=25.1

compute/compute-node.Dockerfile

@@ -83,28 +83,7 @@ ARG TAG=pinned
 ARG BUILD_TAG
 ARG DEBIAN_VERSION=bookworm
 ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim
-
-# Here are the INDEX DIGESTS for the images we use.
-# You can get them following next steps for now:
-# 1. Get an authentication token from DockerHub:
-#    TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/debian:pull" | jq -r .token)
-# 2. Using that token, query index for the given tag:
-#    curl -s -H "Authorization: Bearer $TOKEN" \
-#       -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
-#       "https://registry.hub.docker.com/v2/library/debian/manifests/bullseye-slim" \
-#       -I | grep -i docker-content-digest
-# 3. As a next step, TODO(fedordikarev): create script and schedule workflow to run these checks
-#    and updates on regular bases and in automated way.
-ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-
-# Here we use ${var/search/replace} syntax, to check
-# if base image is one of the images, we pin image index for.
-# If var will match one the known images, we will replace it with the known sha.
-# If no match, than value will be unaffected, and will process with no-pinned image.
-ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA}
+ARG ALPINE_CURL_VERSION=8.11.1
 
 # By default, build all PostgreSQL extensions. For quick local testing when you don't
 # care about the extensions, pass EXTENSIONS=none or EXTENSIONS=minimal
@@ -115,7 +94,7 @@ ARG EXTENSIONS=all
 # Layer "build-deps"
 #
 #########################################################################################
-FROM $BASE_IMAGE_SHA AS build-deps
+FROM debian:$DEBIAN_FLAVOR AS build-deps
 ARG DEBIAN_VERSION
 
 # Use strict mode for bash to catch errors early
@@ -124,7 +103,7 @@ SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 # By default, /bin/sh used in debian images will treat '\n' as eol,
 # but as we use bash as SHELL, and built-in echo in bash requires '-e' flag for that.
 RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\nretry-on-host-error=on\n" > /root/.wgetrc && \
+    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc && \
     echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc
 
 RUN case $DEBIAN_VERSION in \
@@ -148,7 +127,7 @@ RUN case $DEBIAN_VERSION in \
     apt install --no-install-recommends --no-install-suggests -y \
     ninja-build git autoconf automake libtool build-essential bison flex libreadline-dev \
     zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget ca-certificates pkg-config libssl-dev \
-    libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd curl unzip \
+    libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd \
     $VERSION_INSTALLS \
     && apt clean && rm -rf /var/lib/apt/lists/*
 
@@ -160,11 +139,11 @@ RUN case $DEBIAN_VERSION in \
 #########################################################################################
 FROM build-deps AS pg-build
 ARG PG_VERSION
-COPY vendor/postgres-${PG_VERSION:?} postgres
+COPY vendor/postgres-${PG_VERSION} postgres
 RUN cd postgres && \
     export CONFIGURE_CMD="./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp \
     --with-icu --with-libxml --with-libxslt --with-lz4" && \
-    if [ "${PG_VERSION:?}" != "v14" ]; then \
+    if [ "${PG_VERSION}" != "v14" ]; then \
         # zstd is available only from PG15
         export CONFIGURE_CMD="${CONFIGURE_CMD} --with-zstd"; \
     fi && \
@@ -258,7 +237,7 @@ RUN case "${DEBIAN_VERSION}" in \
 
 # Postgis 3.5.0 supports v17
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
     "v17") \
         export POSTGIS_VERSION=3.5.0 \
         export POSTGIS_CHECKSUM=ca698a22cc2b2b3467ac4e063b43a28413f3004ddd505bdccdd74c56a647f510 \
@@ -333,7 +312,7 @@ FROM build-deps AS pgrouting-src
 ARG DEBIAN_VERSION
 ARG PG_VERSION
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
     "v17") \
         export PGROUTING_VERSION=3.6.2 \
         export PGROUTING_CHECKSUM=f4a1ed79d6f714e52548eca3bb8e5593c6745f1bde92eb5fb858efd8984dffa2 \
@@ -379,7 +358,7 @@ COPY compute/patches/plv8-3.1.10.patch .
 #
 # Use new version only for v17
 # because since v3.2, plv8 doesn't include plcoffee and plls extensions
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
     "v17") \
         export PLV8_TAG=v3.2.3 \
     ;; \
@@ -393,7 +372,7 @@ RUN case "${PG_VERSION:?}" in \
     git clone --recurse-submodules --depth 1 --branch ${PLV8_TAG} https://github.com/plv8/plv8.git plv8-src && \
     tar -czf plv8.tar.gz --exclude .git plv8-src && \
     cd plv8-src && \
-    if [[ "${PG_VERSION:?}" < "v17" ]]; then patch -p1 < /ext-src/plv8-3.1.10.patch; fi
+    if [[ "${PG_VERSION}" < "v17" ]]; then patch -p1 < /ext-src/plv8-3.1.10.patch; fi
 
 FROM pg-build AS plv8-build
 ARG PG_VERSION
@@ -413,7 +392,7 @@ RUN \
     find /usr/local/pgsql/ -name "plv8-*.so" | xargs strip && \
     # don't break computes with installed old version of plv8
     cd /usr/local/pgsql/lib/ && \
-    case "${PG_VERSION:?}" in \
+    case "${PG_VERSION}" in \
     "v17") \
         ln -s plv8-3.2.3.so plv8-3.1.8.so && \
         ln -s plv8-3.2.3.so plv8-3.1.5.so && \
@@ -750,7 +729,7 @@ FROM build-deps AS timescaledb-src
 ARG PG_VERSION
 
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
       "v14" | "v15") \
         export TIMESCALEDB_VERSION=2.10.1 \
         export TIMESCALEDB_CHECKSUM=6fca72a6ed0f6d32d2b3523951ede73dc5f9b0077b38450a029a5f411fdb8c73 \
@@ -788,7 +767,7 @@ ARG PG_VERSION
 
 # version-specific, has separate releases for each version
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
       "v14") \
         export PG_HINT_PLAN_VERSION=14_1_4_1 \
         export PG_HINT_PLAN_CHECKSUM=c3501becf70ead27f70626bce80ea401ceac6a77e2083ee5f3ff1f1444ec1ad1 \
@@ -864,7 +843,7 @@ ARG PG_VERSION
 # https://github.com/rdkit/rdkit/releases/tag/Release_2024_09_1
 
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
     "v17") \
         export RDKIT_VERSION=Release_2024_09_1 \
         export RDKIT_CHECKSUM=034c00d6e9de323506834da03400761ed8c3721095114369d06805409747a60f \
@@ -991,7 +970,7 @@ ARG PG_VERSION
 #
 # last release v0.40.0 - Jul 22, 2024
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
     "v17") \
         export SEMVER_VERSION=0.40.0 \
         export SEMVER_CHECKSUM=3e50bcc29a0e2e481e7b6d2bc937cadc5f5869f55d983b5a1aafeb49f5425cfc \
@@ -1027,7 +1006,7 @@ ARG PG_VERSION
 # This is our extension, support stopped in favor of pgvector
 # TODO: deprecate it
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
       "v14" | "v15") \
         export PG_EMBEDDING_VERSION=0.3.5 \
         export PG_EMBEDDING_CHECKSUM=0e95b27b8b6196e2cf0a0c9ec143fe2219b82e54c5bb4ee064e76398cbe69ae9 \
@@ -1060,7 +1039,7 @@ ARG PG_VERSION
 # This is an experimental extension, never got to real production.
 # !Do not remove! It can be present in shared_preload_libraries and compute will fail to start if library is not found.
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in "v17") \
+RUN case "${PG_VERSION}" in "v17") \
     echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \
     esac && \
     wget  https://github.com/neondatabase/postgresql_anonymizer/archive/refs/tags/neon_1.1.1.tar.gz -O pg_anon.tar.gz && \
@@ -1112,7 +1091,7 @@ RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux
 FROM pg-build-nonroot-with-cargo AS rust-extensions-build
 ARG PG_VERSION
 
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
         'v17') \
             echo 'v17 is not supported yet by pgrx. Quit' && exit 0;; \
     esac && \
@@ -1291,7 +1270,7 @@ FROM build-deps AS pgx_ulid-src
 ARG PG_VERSION
 
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
     "v14" | "v15" | "v16") \
         ;; \
     *) \
@@ -1323,7 +1302,7 @@ FROM build-deps AS pgx_ulid-pgrx12-src
 ARG PG_VERSION
 
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
     "v17") \
         ;; \
     *) \
@@ -1451,8 +1430,8 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) && \
 FROM build-deps AS pg_mooncake-src
 ARG PG_VERSION
 WORKDIR /ext-src
-RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.2/pg_mooncake-0.1.2.tar.gz -O pg_mooncake.tar.gz && \
-    echo "4550473784fcdd2e1e18062bc01eb9c286abd27cdf5e11a4399be6c0a426ba90 pg_mooncake.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.1/pg_mooncake-0.1.1.tar.gz -O pg_mooncake.tar.gz && \
+    echo "a2d16eff7948dde64f072609ca5d2962d6b4d07cb89d45952add473529c55f55 pg_mooncake.tar.gz" | sha256sum --check && \
     mkdir pg_mooncake-src && cd pg_mooncake-src && tar xzf ../pg_mooncake.tar.gz --strip-components=1 -C . && \
     echo "make -f pg_mooncake-src/Makefile.build installcheck TEST_DIR=./test SQL_DIR=./sql SRC_DIR=./src" > neon-test.sh && \
     chmod a+x neon-test.sh
@@ -1615,7 +1594,7 @@ RUN --mount=type=cache,uid=1000,target=/home/nonroot/.cargo/registry \
 #
 #########################################################################################
 
-FROM $BASE_IMAGE_SHA AS pgbouncer
+FROM debian:$DEBIAN_FLAVOR AS pgbouncer
 RUN set -e \
     && echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries \
     && apt update \
@@ -1645,12 +1624,13 @@ RUN set -e \
 # Layer "exporters"
 #
 #########################################################################################
-FROM build-deps AS exporters
+FROM alpine/curl:${ALPINE_CURL_VERSION} AS exporters
 ARG TARGETARCH
 # Keep sql_exporter version same as in build-tools.Dockerfile and
 # test_runner/regress/test_compute_metrics.py
 # See comment on the top of the file regading `echo`, `-e` and `\n`
-RUN if [ "$TARGETARCH" = "amd64" ]; then\
+RUN echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc; \
+    if [ "$TARGETARCH" = "amd64" ]; then\
         postgres_exporter_sha256='027e75dda7af621237ff8f5ac66b78a40b0093595f06768612b92b1374bd3105';\
         pgbouncer_exporter_sha256='c9f7cf8dcff44f0472057e9bf52613d93f3ffbc381ad7547a959daa63c5e84ac';\
         sql_exporter_sha256='38e439732bbf6e28ca4a94d7bc3686d3fa1abdb0050773d5617a9efdb9e64d08';\
@@ -1674,7 +1654,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then\
 # Layer "awscli"
 #
 #########################################################################################
-FROM build-deps AS awscli
+FROM alpine/curl:${ALPINE_CURL_VERSION} AS awscli
 ARG TARGETARCH
 RUN set -ex; \
     if [ "${TARGETARCH}" = "amd64" ]; then \
@@ -1724,7 +1704,7 @@ USER nonroot
 
 COPY --chown=nonroot compute compute
 
-RUN make PG_VERSION="${PG_VERSION:?}" -C compute
+RUN make PG_VERSION="${PG_VERSION}" -C compute
 
 #########################################################################################
 #
@@ -1757,8 +1737,8 @@ COPY --from=hll-src /ext-src/ /ext-src/
 COPY --from=plpgsql_check-src /ext-src/ /ext-src/
 #COPY --from=timescaledb-src /ext-src/ /ext-src/
 COPY --from=pg_hint_plan-src /ext-src/ /ext-src/
-COPY compute/patches/pg_hint_plan_${PG_VERSION:?}.patch /ext-src
-RUN cd /ext-src/pg_hint_plan-src && patch -p1 < /ext-src/pg_hint_plan_${PG_VERSION:?}.patch
+COPY compute/patches/pg_hint_plan_${PG_VERSION}.patch /ext-src
+RUN cd /ext-src/pg_hint_plan-src && patch -p1 < /ext-src/pg_hint_plan_${PG_VERSION}.patch
 COPY --from=pg_cron-src /ext-src/ /ext-src/
 #COPY --from=pgx_ulid-src /ext-src/ /ext-src/
 #COPY --from=pgx_ulid-pgrx12-src /ext-src/ /ext-src/
@@ -1787,7 +1767,7 @@ ENV PGDATABASE=postgres
 # Put it all together into the final image
 #
 #########################################################################################
-FROM $BASE_IMAGE_SHA
+FROM debian:$DEBIAN_FLAVOR
 ARG DEBIAN_VERSION
 
 # Use strict mode for bash to catch errors early

compute/vm-image-spec-bookworm.yaml

@@ -74,8 +74,8 @@ build: |
   # At time of migration to bookworm (2024-10-09), debian has a version of libcgroup/cgroup-tools 2.0.2,
   # and it _probably_ can be used as-is. However, we'll build it ourselves to minimise the changeset
   # for debian version migration.
-  ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-  FROM debian@$BOOKWORM_SLIM_SHA as libcgroup-builder
+  #
+  FROM debian:bookworm-slim as libcgroup-builder
   ENV LIBCGROUP_VERSION=v2.0.3
 
   RUN set -exu \

compute/vm-image-spec-bullseye.yaml

@@ -68,8 +68,7 @@ build: |
   # At time of writing (2023-03-14), debian bullseye has a version of cgroup-tools (technically
   # libcgroup) that doesn't support cgroup v2 (version 0.41-11). Unfortunately, the vm-monitor
   # requires cgroup v2, so we'll build cgroup-tools ourselves.
-  ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-  FROM debian@$BULLSEYE_SLIM_SHA as libcgroup-builder
+  FROM debian:bullseye-slim as libcgroup-builder
   ENV LIBCGROUP_VERSION=v2.0.3
 
   RUN set -exu \

compute_tools/src/compute.rs

@@ -1400,27 +1400,26 @@ impl ComputeNode {
         let postgresql_conf_path = pgdata_path.join("postgresql.conf");
         config::write_postgres_conf(&postgresql_conf_path, &spec, self.internal_http_port)?;
 
-        if !spec.skip_pg_catalog_updates {
-            let max_concurrent_connections = spec.reconfigure_concurrency;
-            // Temporarily reset max_cluster_size in config
-            // to avoid the possibility of hitting the limit, while we are reconfiguring:
-            // creating new extensions, roles, etc.
-            config::with_compute_ctl_tmp_override(pgdata_path, "neon.max_cluster_size=-1", || {
-                self.pg_reload_conf()?;
+        let max_concurrent_connections = spec.reconfigure_concurrency;
 
-                if spec.mode == ComputeMode::Primary {
-                    let mut conf = tokio_postgres::Config::from_str(self.connstr.as_str()).unwrap();
-                    conf.application_name("apply_config");
-                    let conf = Arc::new(conf);
+        // Temporarily reset max_cluster_size in config
+        // to avoid the possibility of hitting the limit, while we are reconfiguring:
+        // creating new extensions, roles, etc.
+        config::with_compute_ctl_tmp_override(pgdata_path, "neon.max_cluster_size=-1", || {
+            self.pg_reload_conf()?;
 
-                    let spec = Arc::new(spec.clone());
+            if spec.mode == ComputeMode::Primary {
+                let mut conf = tokio_postgres::Config::from_str(self.connstr.as_str()).unwrap();
+                conf.application_name("apply_config");
+                let conf = Arc::new(conf);
 
-                    self.apply_spec_sql(spec, conf, max_concurrent_connections)?;
-                }
+                let spec = Arc::new(spec.clone());
 
-                Ok(())
-            })?;
-        }
+                self.apply_spec_sql(spec, conf, max_concurrent_connections)?;
+            }
+
+            Ok(())
+        })?;
 
         self.pg_reload_conf()?;
 

control_plane/src/pageserver.rs

@@ -362,11 +362,6 @@ impl PageServerNode {
                 .map(|x| x.parse::<bool>())
                 .transpose()
                 .context("Failed to parse 'compaction_l0_first' as a bool")?,
-            compaction_l0_semaphore: settings
-                .remove("compaction_l0_semaphore")
-                .map(|x| x.parse::<bool>())
-                .transpose()
-                .context("Failed to parse 'compaction_l0_semaphore' as a bool")?,
             l0_flush_delay_threshold: settings
                 .remove("l0_flush_delay_threshold")
                 .map(|x| x.parse::<usize>())

libs/pageserver_api/src/config.rs

@@ -94,6 +94,7 @@ pub struct ConfigToml {
     pub ondemand_download_behavior_treat_error_as_warn: bool,
     #[serde(with = "humantime_serde")]
     pub background_task_maximum_delay: Duration,
+    pub use_compaction_semaphore: bool,
     pub control_plane_api: Option<reqwest::Url>,
     pub control_plane_api_token: Option<String>,
     pub control_plane_emergency_mode: bool,
@@ -265,9 +266,6 @@ pub struct TenantConfigToml {
     pub compaction_algorithm: crate::models::CompactionAlgorithmSettings,
     /// If true, compact down L0 across all tenant timelines before doing regular compaction.
     pub compaction_l0_first: bool,
-    /// If true, use a separate semaphore (i.e. concurrency limit) for the L0 compaction pass. Only
-    /// has an effect if `compaction_l0_first` is `true`.
-    pub compaction_l0_semaphore: bool,
     /// Level0 delta layer threshold at which to delay layer flushes for compaction backpressure,
     /// such that they take 2x as long, and start waiting for layer flushes during ephemeral layer
     /// rolls. This helps compaction keep up with WAL ingestion, and avoids read amplification
@@ -476,6 +474,7 @@ impl Default for ConfigToml {
                 DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY,
             )
             .unwrap()),
+            use_compaction_semaphore: false,
 
             control_plane_api: (None),
             control_plane_api_token: (None),
@@ -549,7 +548,6 @@ pub mod tenant_conf_defaults {
     // calculation to avoid loading all keys into the memory). So with this config, we can get a maximum peak compaction usage of 18GB.
     pub const DEFAULT_COMPACTION_UPPER_LIMIT: usize = 50;
     pub const DEFAULT_COMPACTION_L0_FIRST: bool = false;
-    pub const DEFAULT_COMPACTION_L0_SEMAPHORE: bool = true;
 
     pub const DEFAULT_COMPACTION_ALGORITHM: crate::models::CompactionAlgorithm =
         crate::models::CompactionAlgorithm::Legacy;
@@ -600,7 +598,6 @@ impl Default for TenantConfigToml {
                 kind: DEFAULT_COMPACTION_ALGORITHM,
             },
             compaction_l0_first: DEFAULT_COMPACTION_L0_FIRST,
-            compaction_l0_semaphore: DEFAULT_COMPACTION_L0_SEMAPHORE,
             l0_flush_delay_threshold: None,
             l0_flush_stall_threshold: None,
             l0_flush_wait_upload: DEFAULT_L0_FLUSH_WAIT_UPLOAD,

libs/pageserver_api/src/models.rs

@@ -466,8 +466,6 @@ pub struct TenantConfigPatch {
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub compaction_l0_first: FieldPatch<bool>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
-    pub compaction_l0_semaphore: FieldPatch<bool>,
-    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub l0_flush_delay_threshold: FieldPatch<usize>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub l0_flush_stall_threshold: FieldPatch<usize>,
@@ -534,7 +532,6 @@ pub struct TenantConfig {
     // defer parsing compaction_algorithm, like eviction_policy
     pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
     pub compaction_l0_first: Option<bool>,
-    pub compaction_l0_semaphore: Option<bool>,
     pub l0_flush_delay_threshold: Option<usize>,
     pub l0_flush_stall_threshold: Option<usize>,
     pub l0_flush_wait_upload: Option<bool>,
@@ -574,7 +571,6 @@ impl TenantConfig {
             mut compaction_upper_limit,
             mut compaction_algorithm,
             mut compaction_l0_first,
-            mut compaction_l0_semaphore,
             mut l0_flush_delay_threshold,
             mut l0_flush_stall_threshold,
             mut l0_flush_wait_upload,
@@ -615,9 +611,6 @@ impl TenantConfig {
             .apply(&mut compaction_upper_limit);
         patch.compaction_algorithm.apply(&mut compaction_algorithm);
         patch.compaction_l0_first.apply(&mut compaction_l0_first);
-        patch
-            .compaction_l0_semaphore
-            .apply(&mut compaction_l0_semaphore);
         patch
             .l0_flush_delay_threshold
             .apply(&mut l0_flush_delay_threshold);
@@ -682,7 +675,6 @@ impl TenantConfig {
             compaction_upper_limit,
             compaction_algorithm,
             compaction_l0_first,
-            compaction_l0_semaphore,
             l0_flush_delay_threshold,
             l0_flush_stall_threshold,
             l0_flush_wait_upload,

pageserver/src/config.rs

@@ -140,6 +140,10 @@ pub struct PageServerConf {
     /// not terrible.
     pub background_task_maximum_delay: Duration,
 
+    /// If true, use a separate semaphore for compaction tasks instead of the common background task
+    /// semaphore. Defaults to false.
+    pub use_compaction_semaphore: bool,
+
     pub control_plane_api: Option<Url>,
 
     /// JWT token for use with the control plane API.
@@ -336,6 +340,7 @@ impl PageServerConf {
             test_remote_failures,
             ondemand_download_behavior_treat_error_as_warn,
             background_task_maximum_delay,
+            use_compaction_semaphore,
             control_plane_api,
             control_plane_api_token,
             control_plane_emergency_mode,
@@ -390,6 +395,7 @@ impl PageServerConf {
             test_remote_failures,
             ondemand_download_behavior_treat_error_as_warn,
             background_task_maximum_delay,
+            use_compaction_semaphore,
             control_plane_api,
             control_plane_emergency_mode,
             heatmap_upload_concurrency,

pageserver/src/deletion_queue.rs

@@ -150,10 +150,30 @@ impl FlushOp {
     }
 }
 
+/// A DeletionNotify can be used to wait for a deletion to have been executed.
+#[derive(Debug)]
+pub struct DeletionNotify {
+    /// Receives the `DeletionListSeq` from `ListWriter` when scheduled in a `DeletionList`.
+    seq_rx: tokio::sync::oneshot::Receiver<DeletionListSeq>,
+    /// Watches the last executed `DeletionListSeq`.
+    executed_rx: tokio::sync::watch::Receiver<DeletionListSeq>,
+}
+
+impl DeletionNotify {
+    /// Waits for the deletion to have been executed.
+    pub async fn notify(mut self) {
+        let Ok(wait_seq) = self.seq_rx.await else {
+            return; // TODO return error
+        };
+        self.executed_rx.wait_for(|&seq| seq >= wait_seq).await.ok();
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct DeletionQueueClient {
     tx: tokio::sync::mpsc::UnboundedSender<ListWriterQueueMessage>,
     executor_tx: tokio::sync::mpsc::Sender<DeleterMessage>,
+    executed_rx: tokio::sync::watch::Receiver<DeletionListSeq>,
 
     lsn_table: Arc<std::sync::RwLock<VisibleLsnUpdates>>,
 }
@@ -176,6 +196,9 @@ impl TenantDeletionList {
     }
 }
 
+/// Deletion list sequence number. Monotonically increasing, even across restarts.
+type DeletionListSeq = u64;
+
 /// Files ending with this suffix will be ignored and erased
 /// during recovery as startup.
 const TEMP_SUFFIX: &str = "tmp";
@@ -185,8 +208,9 @@ struct DeletionList {
     /// Serialization version, for future use
     version: u8,
 
-    /// Used for constructing a unique key for each deletion list we write out.
-    sequence: u64,
+    /// Used for constructing a unique key for each deletion list we write out, and to notify
+    /// callers when a deletion has been executed (and will not be retried later).
+    sequence: DeletionListSeq,
 
     /// To avoid repeating tenant/timeline IDs in every key, we store keys in
     /// nested HashMaps by TenantTimelineID.  Each Tenant only appears once
@@ -214,13 +238,13 @@ struct DeletionHeader {
 
     /// The highest sequence number (inclusive) that has been validated.  All deletion
     /// lists on disk with a sequence <= this value are safe to execute.
-    validated_sequence: u64,
+    validated_sequence: DeletionListSeq,
 }
 
 impl DeletionHeader {
     const VERSION_LATEST: u8 = 1;
 
-    fn new(validated_sequence: u64) -> Self {
+    fn new(validated_sequence: DeletionListSeq) -> Self {
         Self {
             version: Self::VERSION_LATEST,
             validated_sequence,
@@ -242,7 +266,7 @@ impl DeletionHeader {
 
 impl DeletionList {
     const VERSION_LATEST: u8 = 1;
-    fn new(sequence: u64) -> Self {
+    fn new(sequence: DeletionListSeq) -> Self {
         Self {
             version: Self::VERSION_LATEST,
             sequence,
@@ -460,6 +484,8 @@ impl DeletionQueueClient {
     /// layers until you're sure they can be deleted safely (i.e. remote metadata no longer
     /// references them).
     ///
+    /// The returned `DeletionNotify` can be used to wait for the deletion to execute.
+    ///
     /// The `current_generation` is the generation of this pageserver's current attachment.  The
     /// generations in `layers` are the generations in which those layers were written.
     pub(crate) fn push_layers(
@@ -468,12 +494,14 @@ impl DeletionQueueClient {
         timeline_id: TimelineId,
         current_generation: Generation,
         layers: Vec<(LayerName, LayerFileMetadata)>,
-    ) -> Result<(), DeletionQueueError> {
+    ) -> Result<DeletionNotify, DeletionQueueError> {
         // None generations are not valid for attached tenants: they must always be attached in
         // a known generation.  None generations are still permitted for layers in the index because
         // they may be historical.
         assert!(!current_generation.is_none());
 
+        let (seq_tx, seq_rx) = tokio::sync::oneshot::channel();
+
         metrics::DELETION_QUEUE
             .keys_submitted
             .inc_by(layers.len() as u64);
@@ -485,8 +513,14 @@ impl DeletionQueueClient {
                 layers,
                 generation: current_generation,
                 objects: Vec::new(),
+                seq_tx,
             }),
-        )
+        )?;
+
+        Ok(DeletionNotify {
+            seq_rx,
+            executed_rx: self.executed_rx.clone(),
+        })
     }
 
     /// This is cancel-safe.  If you drop the future the flush may still happen in the background.
@@ -610,6 +644,10 @@ impl DeletionQueue {
         // happen in the backend (persistent), not in this queue.
         let (executor_tx, executor_rx) = tokio::sync::mpsc::channel(16);
 
+        // Notifies clients about executed deletions.
+        // TODO: recover the last sequence number on startup.
+        let (executed_tx, executed_rx) = tokio::sync::watch::channel(0);
+
         let lsn_table = Arc::new(std::sync::RwLock::new(VisibleLsnUpdates::new()));
 
         // The deletion queue has an independent cancellation token to
@@ -622,6 +660,7 @@ impl DeletionQueue {
                 client: DeletionQueueClient {
                     tx,
                     executor_tx: executor_tx.clone(),
+                    executed_rx,
                     lsn_table: lsn_table.clone(),
                 },
                 cancel: cancel.clone(),
@@ -632,6 +671,7 @@ impl DeletionQueue {
                     conf,
                     backend_rx,
                     executor_tx,
+                    executed_tx,
                     controller_upcall_client,
                     lsn_table.clone(),
                     cancel.clone(),
@@ -1228,6 +1268,7 @@ pub(crate) mod mock {
             DeletionQueueClient {
                 tx: self.tx.clone(),
                 executor_tx: self.executor_tx.clone(),
+                executed_rx: todo!(),
                 lsn_table: self.lsn_table.clone(),
             }
         }

pageserver/src/deletion_queue/list_writer.rs

@@ -12,6 +12,7 @@
 
 use super::DeletionHeader;
 use super::DeletionList;
+use super::DeletionListSeq;
 use super::FlushOp;
 use super::ValidatorQueueMessage;
 
@@ -65,6 +66,9 @@ pub(super) struct DeletionOp {
     /// The _current_ generation of the Tenant shard attachment in which we are enqueuing
     /// this deletion.
     pub(super) generation: Generation,
+
+    /// Return channel for the `DeletionList::sequence` this deletion is included in.
+    pub(super) seq_tx: tokio::sync::oneshot::Sender<DeletionListSeq>,
 }
 
 #[derive(Debug)]
@@ -175,7 +179,7 @@ impl ListWriter {
     ///
     /// It is not an error for the header to not exist: we return None, and
     /// the caller should act as if validated_sequence is 0
-    async fn load_validated_sequence(&self) -> Result<Option<u64>, anyhow::Error> {
+    async fn load_validated_sequence(&self) -> Result<Option<DeletionListSeq>, anyhow::Error> {
         let header_path = self.conf.deletion_header_path();
         match tokio::fs::read(&header_path).await {
             Ok(header_bytes) => {
@@ -228,7 +232,7 @@ impl ListWriter {
 
         let temp_extension = format!(".{TEMP_SUFFIX}");
         let header_path = self.conf.deletion_header_path();
-        let mut seqs: Vec<u64> = Vec::new();
+        let mut seqs: Vec<DeletionListSeq> = Vec::new();
         while let Some(dentry) = dir.next_entry().await.fatal_err("read deletion dentry") {
             let file_name = dentry.file_name();
             let dentry_str = file_name.to_string_lossy();
@@ -433,6 +437,9 @@ impl ListWriter {
                             metrics::DELETION_QUEUE.unexpected_errors.inc();
                         }
                     }
+
+                    // Notify the client about the sequence number of this deletion.
+                    op.seq_tx.send(self.pending.sequence).ok();
                 }
                 ListWriterQueueMessage::Flush(op) => {
                     if self.pending.is_empty() {

pageserver/src/deletion_queue/validator.rs

@@ -33,6 +33,7 @@ use crate::virtual_file::MaybeFatalIo;
 use super::deleter::DeleterMessage;
 use super::DeletionHeader;
 use super::DeletionList;
+use super::DeletionListSeq;
 use super::DeletionQueueError;
 use super::FlushOp;
 use super::VisibleLsnUpdates;
@@ -60,6 +61,9 @@ where
     rx: tokio::sync::mpsc::Receiver<ValidatorQueueMessage>,
     tx: tokio::sync::mpsc::Sender<DeleterMessage>,
 
+    /// Notifies clients about the last executed DeletionList sequence number.
+    executed_tx: tokio::sync::watch::Sender<DeletionListSeq>,
+
     // Client for calling into control plane API for validation of deletes
     controller_upcall_client: Option<C>,
 
@@ -94,6 +98,7 @@ where
         conf: &'static PageServerConf,
         rx: tokio::sync::mpsc::Receiver<ValidatorQueueMessage>,
         tx: tokio::sync::mpsc::Sender<DeleterMessage>,
+        executed_tx: tokio::sync::watch::Sender<DeletionListSeq>,
         controller_upcall_client: Option<C>,
         lsn_table: Arc<std::sync::RwLock<VisibleLsnUpdates>>,
         cancel: CancellationToken,
@@ -102,6 +107,7 @@ where
             conf,
             rx,
             tx,
+            executed_tx,
             controller_upcall_client,
             lsn_table,
             pending_lists: Vec::new(),
@@ -161,7 +167,7 @@ where
             tenant_generations.keys().map(|k| (*k, true)).collect()
         };
 
-        let mut validated_sequence: Option<u64> = None;
+        let mut validated_sequence: Option<DeletionListSeq> = None;
 
         // Apply the validation results to the pending LSN updates
         for (tenant_id, tenant_lsn_state) in pending_lsn_updates.tenants {
@@ -295,6 +301,7 @@ where
     async fn cleanup_lists(&mut self, list_paths: Vec<Utf8PathBuf>) {
         for list_path in list_paths {
             debug!("Removing deletion list {list_path}");
+            // TODO: this needs to fsync the removal.
             tokio::fs::remove_file(&list_path)
                 .await
                 .fatal_err("remove deletion list");
@@ -324,6 +331,7 @@ where
         }
 
         // Drain `validated_lists` into the executor
+        let executed_seq = self.validated_lists.iter().map(|l| l.sequence).max();
         let mut executing_lists = Vec::new();
         for list in self.validated_lists.drain(..) {
             let list_path = self.conf.deletion_list_path(list.sequence);
@@ -340,6 +348,14 @@ where
         // Erase the deletion lists whose keys have all be deleted from remote storage
         self.cleanup_lists(executing_lists).await;
 
+        // Notify any waiters that the deletions have been executed.
+        //
+        // TODO: this will wait for all pending lists to be deleted. Consider making it more
+        // responsive by processing lists one by one.
+        if let Some(executed_seq) = executed_seq {
+            self.executed_tx.send_replace(executed_seq);
+        }
+
         Ok(())
     }
 

pageserver/src/http/routes.rs

@@ -2151,7 +2151,6 @@ async fn timeline_compact_handler(
     let state = get_state(&request);
 
     let mut flags = EnumSet::empty();
-    flags |= CompactFlags::NoYield; // run compaction to completion
 
     if Some(true) == parse_query_param::<_, bool>(&request, "force_l0_compaction")? {
         flags |= CompactFlags::ForceL0Compaction;

pageserver/src/tenant.rs

@@ -5542,7 +5542,6 @@ pub(crate) mod harness {
                 compaction_upper_limit: Some(tenant_conf.compaction_upper_limit),
                 compaction_algorithm: Some(tenant_conf.compaction_algorithm),
                 compaction_l0_first: Some(tenant_conf.compaction_l0_first),
-                compaction_l0_semaphore: Some(tenant_conf.compaction_l0_semaphore),
                 l0_flush_delay_threshold: tenant_conf.l0_flush_delay_threshold,
                 l0_flush_stall_threshold: tenant_conf.l0_flush_stall_threshold,
                 l0_flush_wait_upload: Some(tenant_conf.l0_flush_wait_upload),

pageserver/src/tenant/config.rs

@@ -289,10 +289,6 @@ pub struct TenantConfOpt {
     #[serde(default)]
     pub compaction_l0_first: Option<bool>,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
-    pub compaction_l0_semaphore: Option<bool>,
-
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(default)]
     pub l0_flush_delay_threshold: Option<usize>,
@@ -427,9 +423,6 @@ impl TenantConfOpt {
             compaction_l0_first: self
                 .compaction_l0_first
                 .unwrap_or(global_conf.compaction_l0_first),
-            compaction_l0_semaphore: self
-                .compaction_l0_semaphore
-                .unwrap_or(global_conf.compaction_l0_semaphore),
             l0_flush_delay_threshold: self
                 .l0_flush_delay_threshold
                 .or(global_conf.l0_flush_delay_threshold),
@@ -508,7 +501,6 @@ impl TenantConfOpt {
             mut compaction_upper_limit,
             mut compaction_algorithm,
             mut compaction_l0_first,
-            mut compaction_l0_semaphore,
             mut l0_flush_delay_threshold,
             mut l0_flush_stall_threshold,
             mut l0_flush_wait_upload,
@@ -555,9 +547,6 @@ impl TenantConfOpt {
             .apply(&mut compaction_upper_limit);
         patch.compaction_algorithm.apply(&mut compaction_algorithm);
         patch.compaction_l0_first.apply(&mut compaction_l0_first);
-        patch
-            .compaction_l0_semaphore
-            .apply(&mut compaction_l0_semaphore);
         patch
             .l0_flush_delay_threshold
             .apply(&mut l0_flush_delay_threshold);
@@ -640,7 +629,6 @@ impl TenantConfOpt {
             compaction_upper_limit,
             compaction_algorithm,
             compaction_l0_first,
-            compaction_l0_semaphore,
             l0_flush_delay_threshold,
             l0_flush_stall_threshold,
             l0_flush_wait_upload,
@@ -704,7 +692,6 @@ impl From<TenantConfOpt> for models::TenantConfig {
             compaction_threshold: value.compaction_threshold,
             compaction_upper_limit: value.compaction_upper_limit,
             compaction_l0_first: value.compaction_l0_first,
-            compaction_l0_semaphore: value.compaction_l0_semaphore,
             l0_flush_delay_threshold: value.l0_flush_delay_threshold,
             l0_flush_stall_threshold: value.l0_flush_stall_threshold,
             l0_flush_wait_upload: value.l0_flush_wait_upload,

pageserver/src/tenant/remote_timeline_client.rs

@@ -2151,6 +2151,7 @@ impl RemoteTimelineClient {
                                 self.generation,
                                 delete.layers.clone(),
                             )
+                            .map(|_| ())
                             .map_err(|e| anyhow::anyhow!(e))
                     }
                 }

pageserver/src/tenant/tasks.rs

@@ -4,7 +4,7 @@ use std::cmp::max;
 use std::future::Future;
 use std::ops::{ControlFlow, RangeInclusive};
 use std::pin::pin;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 
 use once_cell::sync::Lazy;
@@ -15,7 +15,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::*;
 
 use crate::context::{DownloadBehavior, RequestContext};
-use crate::metrics::{self, BackgroundLoopSemaphoreMetricsRecorder, TENANT_TASK_EVENTS};
+use crate::metrics::{BackgroundLoopSemaphoreMetricsRecorder, TENANT_TASK_EVENTS};
 use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME, TOKIO_WORKER_THREADS};
 use crate::tenant::throttle::Stats;
 use crate::tenant::timeline::compaction::CompactionOutcome;
@@ -25,6 +25,7 @@ use pageserver_api::config::tenant_conf_defaults::DEFAULT_COMPACTION_PERIOD;
 use utils::backoff::exponential_backoff_duration;
 use utils::completion::Barrier;
 use utils::pausable_failpoint;
+use utils::rate_limit::RateLimit;
 
 /// Semaphore limiting concurrent background tasks (across all tenants).
 ///
@@ -37,17 +38,17 @@ static CONCURRENT_BACKGROUND_TASKS: Lazy<Semaphore> = Lazy::new(|| {
     Semaphore::new(permits)
 });
 
-/// Semaphore limiting concurrent L0 compaction tasks (across all tenants). This is only used if
-/// both `compaction_l0_semaphore` and `compaction_l0_first` are enabled.
+/// Semaphore limiting concurrent compaction tasks (across all tenants). This is disabled by
+/// default, see `use_compaction_semaphore`.
+///
+/// We use 3/4 Tokio threads, to avoid blocking all threads in case we do any CPU-heavy work.
 ///
 /// This is a separate semaphore from background tasks, because L0 compaction needs to be responsive
-/// to avoid high read amp during heavy write workloads. Regular image/GC compaction is less
-/// important (e.g. due to page images in delta layers) and can wait for other background tasks.
+/// to avoid high read amp during heavy write workloads.
 ///
-/// We use 3/4 Tokio threads, to avoid blocking all threads in case we do any CPU-heavy work. Note
-/// that this runs on the same Tokio runtime as `CONCURRENT_BACKGROUND_TASKS`, and shares the same
-/// thread pool.
-static CONCURRENT_L0_COMPACTION_TASKS: Lazy<Semaphore> = Lazy::new(|| {
+/// TODO: split image compaction and L0 compaction, and move image compaction to background tasks.
+/// Only L0 compaction needs to be responsive, and it shouldn't block on image compaction.
+static CONCURRENT_COMPACTION_TASKS: Lazy<Semaphore> = Lazy::new(|| {
     let total_threads = TOKIO_WORKER_THREADS.get();
     let permits = max(1, (total_threads * 3).checked_div(4).unwrap_or(0));
     assert_ne!(permits, 0, "we will not be adding in permits later");
@@ -58,7 +59,7 @@ static CONCURRENT_L0_COMPACTION_TASKS: Lazy<Semaphore> = Lazy::new(|| {
 /// Background jobs.
 ///
 /// NB: not all of these acquire a CONCURRENT_BACKGROUND_TASKS semaphore permit, only the ones that
-/// do any significant IO or CPU work.
+/// do any significant IO.
 #[derive(
     Debug,
     PartialEq,
@@ -71,9 +72,6 @@ static CONCURRENT_L0_COMPACTION_TASKS: Lazy<Semaphore> = Lazy::new(|| {
 )]
 #[strum(serialize_all = "snake_case")]
 pub(crate) enum BackgroundLoopKind {
-    /// L0Compaction runs as a separate pass within the Compaction loop, not a separate loop. It is
-    /// used to request the `CONCURRENT_L0_COMPACTION_TASKS` semaphore and associated metrics.
-    L0Compaction,
     Compaction,
     Gc,
     Eviction,
@@ -93,22 +91,37 @@ pub struct BackgroundLoopSemaphorePermit<'a> {
 /// Acquires a semaphore permit, to limit concurrent background jobs.
 pub(crate) async fn acquire_concurrency_permit(
     loop_kind: BackgroundLoopKind,
+    use_compaction_semaphore: bool,
     _ctx: &RequestContext,
 ) -> BackgroundLoopSemaphorePermit<'static> {
-    let mut recorder = metrics::BACKGROUND_LOOP_SEMAPHORE.record(loop_kind);
+    // TODO: use a lower threshold and remove the pacer once we resolve some blockage.
+    const WARN_THRESHOLD: Duration = Duration::from_secs(600);
+    static WARN_PACER: Lazy<Mutex<RateLimit>> =
+        Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
+
+    let mut recorder = crate::metrics::BACKGROUND_LOOP_SEMAPHORE.record(loop_kind);
 
     if loop_kind == BackgroundLoopKind::InitialLogicalSizeCalculation {
         pausable_failpoint!("initial-size-calculation-permit-pause");
     }
 
     // TODO: assert that we run on BACKGROUND_RUNTIME; requires tokio_unstable Handle::id();
-    let semaphore = match loop_kind {
-        BackgroundLoopKind::L0Compaction => &CONCURRENT_L0_COMPACTION_TASKS,
-        _ => &CONCURRENT_BACKGROUND_TASKS,
-    };
-    let permit = semaphore.acquire().await.expect("should never close");
+    let permit = if loop_kind == BackgroundLoopKind::Compaction && use_compaction_semaphore {
+        CONCURRENT_COMPACTION_TASKS.acquire().await
+    } else {
+        assert!(!use_compaction_semaphore);
+        CONCURRENT_BACKGROUND_TASKS.acquire().await
+    }
+    .expect("should never close");
 
-    recorder.acquired();
+    let waited = recorder.acquired();
+    if waited >= WARN_THRESHOLD {
+        let waited = waited.as_secs_f64();
+        WARN_PACER
+            .lock()
+            .unwrap()
+            .call(|| warn!("{loop_kind} task waited {waited:.3}s for semaphore permit"));
+    }
 
     BackgroundLoopSemaphorePermit {
         _permit: permit,
@@ -576,7 +589,7 @@ pub(crate) fn warn_when_period_overrun(
             ?task,
             "task iteration took longer than the configured period"
         );
-        metrics::BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT
+        crate::metrics::BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT
             .with_label_values(&[task.into(), &format!("{}", period.as_secs())])
             .inc();
     }

pageserver/src/tenant/timeline.rs

@@ -879,9 +879,6 @@ pub(crate) enum CompactFlags {
     OnlyL0Compaction,
     EnhancedGcBottomMostCompaction,
     DryRun,
-    /// Disables compaction yielding e.g. due to high L0 count. This is set e.g. when requesting
-    /// compaction via HTTP API.
-    NoYield,
 }
 
 #[serde_with::serde_as]
@@ -1557,7 +1554,6 @@ impl Timeline {
             let lsn = xlog_utils::normalize_lsn(lsn, WAL_SEGMENT_SIZE);
 
             let mut gc_info = self.gc_info.write().unwrap();
-            let planned_cutoff = gc_info.min_cutoff();
 
             let valid_until = SystemTime::now() + length;
 
@@ -1578,7 +1574,7 @@ impl Timeline {
                     existing_lease.clone()
                 }
                 Entry::Vacant(vacant) => {
-                    // Reject already GC-ed LSN if we are in AttachedSingle and
+                    // Reject already GC-ed LSN (lsn < latest_gc_cutoff) if we are in AttachedSingle and
                     // not blocked by the lsn lease deadline.
                     let validate = {
                         let conf = self.tenant_conf.load();
@@ -1589,10 +1585,7 @@ impl Timeline {
                     if init || validate {
                         let latest_gc_cutoff_lsn = self.get_latest_gc_cutoff_lsn();
                         if lsn < *latest_gc_cutoff_lsn {
-                            bail!("tried to request an lsn lease for an lsn below the latest gc cutoff. requested at {} gc cutoff {}", lsn, *latest_gc_cutoff_lsn);
-                        }
-                        if lsn < planned_cutoff {
-                            bail!("tried to request an lsn lease for an lsn below the planned gc cutoff. requested at {} planned gc cutoff {}", lsn, planned_cutoff);
+                            bail!("tried to request a page version that was garbage collected. requested at {} gc cutoff {}", lsn, *latest_gc_cutoff_lsn);
                         }
                     }
 
@@ -1794,46 +1787,34 @@ impl Timeline {
         .await
     }
 
-    /// Outermost timeline compaction operation; downloads needed layers.
-    ///
-    /// NB: the cancellation token is usually from a background task, but can also come from a
-    /// request task.
+    /// Outermost timeline compaction operation; downloads needed layers. Returns whether we have pending
+    /// compaction tasks.
     pub(crate) async fn compact_with_options(
         self: &Arc<Self>,
         cancel: &CancellationToken,
         options: CompactOptions,
         ctx: &RequestContext,
     ) -> Result<CompactionOutcome, CompactionError> {
-        // Acquire the compaction lock and task semaphore.
-        //
-        // L0-only compaction uses a separate semaphore (if enabled) to make sure it isn't starved
-        // out by other background tasks (including image compaction). We request this via
-        // `BackgroundLoopKind::L0Compaction`.
-        //
-        // If this is a regular compaction pass, and L0-only compaction is enabled in the config,
-        // then we should yield for immediate L0 compaction if necessary while we're waiting for the
-        // background task semaphore. There's no point yielding otherwise, since we'd just end up
-        // right back here.
-        let is_l0_only = options.flags.contains(CompactFlags::OnlyL0Compaction);
-        let semaphore_kind = match is_l0_only && self.get_compaction_l0_semaphore() {
-            true => BackgroundLoopKind::L0Compaction,
-            false => BackgroundLoopKind::Compaction,
-        };
-        let yield_for_l0 = !is_l0_only
-            && self.get_compaction_l0_first()
-            && !options.flags.contains(CompactFlags::NoYield);
+        // most likely the cancellation token is from background task, but in tests it could be the
+        // request task as well.
 
-        let acquire = async move {
+        let prepare = async move {
             let guard = self.compaction_lock.lock().await;
-            let permit = super::tasks::acquire_concurrency_permit(semaphore_kind, ctx).await;
+
+            let permit = super::tasks::acquire_concurrency_permit(
+                BackgroundLoopKind::Compaction,
+                self.conf.use_compaction_semaphore,
+                ctx,
+            )
+            .await;
+
             (guard, permit)
         };
 
+        // this wait probably never needs any "long time spent" logging, because we already nag if
+        // compaction task goes over it's period (20s) which is quite often in production.
         let (_guard, _permit) = tokio::select! {
-            (guard, permit) = acquire => (guard, permit),
-            _ = self.l0_compaction_trigger.notified(), if yield_for_l0 => {
-                return Ok(CompactionOutcome::YieldForL0);
-            }
+            tuple = prepare => { tuple },
             _ = self.cancel.cancelled() => return Ok(CompactionOutcome::Skipped),
             _ = cancel.cancelled() => return Ok(CompactionOutcome::Skipped),
         };
@@ -2345,20 +2326,6 @@ impl Timeline {
             .unwrap_or(self.conf.default_tenant_conf.compaction_upper_limit)
     }
 
-    pub fn get_compaction_l0_first(&self) -> bool {
-        let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
-        tenant_conf
-            .compaction_l0_first
-            .unwrap_or(self.conf.default_tenant_conf.compaction_l0_first)
-    }
-
-    pub fn get_compaction_l0_semaphore(&self) -> bool {
-        let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
-        tenant_conf
-            .compaction_l0_semaphore
-            .unwrap_or(self.conf.default_tenant_conf.compaction_l0_semaphore)
-    }
-
     fn get_l0_flush_delay_threshold(&self) -> Option<usize> {
         // Disable L0 flushes by default. This and compaction needs further tuning.
         const DEFAULT_L0_FLUSH_DELAY_FACTOR: usize = 0; // TODO: default to e.g. 3
@@ -3176,6 +3143,7 @@ impl Timeline {
             async move {
                 let wait_for_permit = super::tasks::acquire_concurrency_permit(
                     BackgroundLoopKind::InitialLogicalSizeCalculation,
+                    false,
                     background_ctx,
                 );
 
@@ -4220,7 +4188,6 @@ impl Timeline {
                     ImageLayerCreationMode::Initial,
                     ctx,
                     LastImageLayerCreationStatus::Initial,
-                    false, // don't yield for L0, we're flushing L0
                 )
                 .await?;
             debug_assert!(
@@ -4793,7 +4760,6 @@ impl Timeline {
         mode: ImageLayerCreationMode,
         ctx: &RequestContext,
         last_status: LastImageLayerCreationStatus,
-        yield_for_l0: bool,
     ) -> Result<(Vec<ResidentLayer>, LastImageLayerCreationStatus), CreateImageLayersError> {
         let timer = self.metrics.create_images_time_histo.start_timer();
 
@@ -4990,7 +4956,7 @@ impl Timeline {
 
             if let ImageLayerCreationMode::Try = mode {
                 // We have at least made some progress
-                if yield_for_l0 && batch_image_writer.pending_layer_num() >= 1 {
+                if batch_image_writer.pending_layer_num() >= 1 {
                     // The `Try` mode is currently only used on the compaction path. We want to avoid
                     // image layer generation taking too long time and blocking L0 compaction. So in this
                     // mode, we also inspect the current number of L0 layers and skip image layer generation

pageserver/src/tenant/timeline/compaction.rs

@@ -726,9 +726,7 @@ impl Timeline {
         }
 
         // Yield if we have pending L0 compaction. The scheduler will do another pass.
-        if (l0_outcome == CompactionOutcome::Pending || l0_outcome == CompactionOutcome::YieldForL0)
-            && !options.flags.contains(CompactFlags::NoYield)
-        {
+        if l0_outcome == CompactionOutcome::Pending || l0_outcome == CompactionOutcome::YieldForL0 {
             info!("image/ancestor compaction yielding for L0 compaction");
             return Ok(CompactionOutcome::YieldForL0);
         }
@@ -776,7 +774,6 @@ impl Timeline {
                                 .load()
                                 .as_ref()
                                 .clone(),
-                            !options.flags.contains(CompactFlags::NoYield),
                         )
                         .await
                         .inspect_err(|err| {

pageserver/src/tenant/timeline/eviction_task.rs

@@ -332,8 +332,11 @@ impl Timeline {
         cancel: &CancellationToken,
         ctx: &RequestContext,
     ) -> ControlFlow<(), BackgroundLoopSemaphorePermit<'static>> {
-        let acquire_permit =
-            crate::tenant::tasks::acquire_concurrency_permit(BackgroundLoopKind::Eviction, ctx);
+        let acquire_permit = crate::tenant::tasks::acquire_concurrency_permit(
+            BackgroundLoopKind::Eviction,
+            false,
+            ctx,
+        );
 
         tokio::select! {
             permit = acquire_permit => ControlFlow::Continue(permit),

scripts/generate_image_maps.py

@@ -1,58 +0,0 @@
-import itertools
-import json
-import os
-
-build_tag = os.environ["BUILD_TAG"]
-branch = os.environ["BRANCH"]
-dev_acr = os.environ["DEV_ACR"]
-prod_acr = os.environ["PROD_ACR"]
-
-components = {
-    "neon": ["neon"],
-    "compute": [
-        "compute-node-v14",
-        "compute-node-v15",
-        "compute-node-v16",
-        "compute-node-v17",
-        "vm-compute-node-v14",
-        "vm-compute-node-v15",
-        "vm-compute-node-v16",
-        "vm-compute-node-v17",
-    ],
-}
-
-registries = {
-    "dev": [
-        "docker.io/neondatabase",
-        "369495373322.dkr.ecr.eu-central-1.amazonaws.com",
-        f"{dev_acr}.azurecr.io/neondatabase",
-    ],
-    "prod": [
-        "093970136003.dkr.ecr.eu-central-1.amazonaws.com",
-        f"{prod_acr}.azurecr.io/neondatabase",
-    ],
-}
-
-outputs: dict[str, dict[str, list[str]]] = {}
-
-target_tags = [build_tag, "latest"] if branch == "main" else [build_tag]
-target_stages = ["dev", "prod"] if branch.startswith("release") else ["dev"]
-
-for component_name, component_images in components.items():
-    for stage in target_stages:
-        outputs[f"{component_name}-{stage}"] = dict(
-            [
-                (
-                    f"docker.io/neondatabase/{component_image}:{build_tag}",
-                    [
-                        f"{combo[0]}/{component_image}:{combo[1]}"
-                        for combo in itertools.product(registries[stage], target_tags)
-                    ],
-                )
-                for component_image in component_images
-            ]
-        )
-
-with open(os.environ["GITHUB_OUTPUT"], "a") as f:
-    for key, value in outputs.items():
-        f.write(f"{key}={json.dumps(value)}\n")

scripts/push_with_image_map.py

@@ -1,22 +0,0 @@
-import json
-import os
-import subprocess
-
-image_map = os.getenv("IMAGE_MAP")
-if not image_map:
-    raise ValueError("IMAGE_MAP environment variable is not set")
-
-try:
-    parsed_image_map: dict[str, list[str]] = json.loads(image_map)
-except json.JSONDecodeError as e:
-    raise ValueError("Failed to parse IMAGE_MAP as JSON") from e
-
-for source, targets in parsed_image_map.items():
-    for target in targets:
-        cmd = ["docker", "buildx", "imagetools", "create", "-t", target, source]
-        print(f"Running: {' '.join(cmd)}")
-        result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-        if result.returncode != 0:
-            print(f"Error: {result.stdout}")
-            raise RuntimeError(f"Command failed: {' '.join(cmd)}")

test_runner/regress/test_attach_tenant_config.py

@@ -141,7 +141,6 @@ def test_fully_custom_config(positive_env: NeonEnv):
         "compaction_threshold": 13,
         "compaction_upper_limit": 100,
         "compaction_l0_first": False,
-        "compaction_l0_semaphore": False,
         "l0_flush_delay_threshold": 25,
         "l0_flush_stall_threshold": 42,
         "l0_flush_wait_upload": False,

test_runner/regress/test_compute_reconfigure.py

@@ -1,62 +0,0 @@
-from __future__ import annotations
-
-from fixtures.neon_fixtures import NeonEnv
-from fixtures.utils import wait_until
-
-
-def test_compute_reconfigure(neon_simple_env: NeonEnv):
-    """
-    Test that we can change postgresql.conf settings even if
-    skip_pg_catalog_updates=True is set.
-    """
-    env = neon_simple_env
-
-    TEST_LOG_LINE_PREFIX = "%m [%p] [test_compute_reconfigure]: "
-
-    endpoint = env.endpoints.create_start("main")
-
-    # Check that the log line prefix is not set
-    # or different from TEST_LOG_LINE_PREFIX
-    with endpoint.cursor() as cursor:
-        cursor.execute("SHOW log_line_prefix;")
-        row = cursor.fetchone()
-        assert row is not None
-        assert row[0] != TEST_LOG_LINE_PREFIX
-
-    endpoint.respec_deep(
-        **{
-            "skip_pg_catalog_updates": True,
-            "cluster": {
-                "settings": [
-                    {
-                        "name": "log_line_prefix",
-                        "vartype": "string",
-                        "value": TEST_LOG_LINE_PREFIX,
-                    }
-                ]
-            },
-        }
-    )
-    endpoint.reconfigure()
-
-    # Check that in logs we see that it was actually reconfigured,
-    # not restarted or something else.
-    endpoint.log_contains("INFO request{method=POST uri=/configure")
-
-    # In /configure we only send SIGHUP at the end, so in theory
-    # it doesn't necessarily mean that Postgres already reloaded
-    # the new config; and it may race in some envs.
-    # So we wait until we see the log line that the config was changed.
-    def check_logs():
-        endpoint.log_contains(
-            f'[test_compute_reconfigure]: LOG:  parameter "log_line_prefix" changed to "{TEST_LOG_LINE_PREFIX}"'
-        )
-
-    wait_until(check_logs)
-
-    # Check that the log line prefix is set
-    with endpoint.cursor() as cursor:
-        cursor.execute("SHOW log_line_prefix;")
-        row = cursor.fetchone()
-        assert row is not None
-        assert row[0] == TEST_LOG_LINE_PREFIX