use serde_ignored to warn about ignored unknown fields

refs
- refs https://github.com/neondatabase/neon/issues/8915

.github/scripts/generate_image_maps.py

@@ -49,10 +49,10 @@ target_stages = (
 for component_name, component_images in components.items():
     for stage in target_stages:
         outputs[f"{component_name}-{stage}"] = {
-            f"docker.io/neondatabase/{component_image}:{source_tag}": [
+            f"ghcr.io/neondatabase/{component_image}:{source_tag}": [
                 f"{registry}/{component_image}:{tag}"
                 for registry, tag in itertools.product(registries[stage], target_tags)
-                if not (registry == "docker.io/neondatabase" and tag == source_tag)
+                if not (registry == "ghcr.io/neondatabase" and tag == source_tag)
             ]
             for component_image in component_images
         }

.github/workflows/_benchmarking_preparation.yml

@@ -8,6 +8,9 @@ defaults:
   run:
     shell: bash -euxo pipefail {0}
 
+permissions:
+  contents: read
+
 jobs:
   setup-databases:
     permissions:
@@ -27,13 +30,18 @@ jobs:
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
     - name: Set up Connection String
       id: set-up-prep-connstr
       run: |
@@ -58,10 +66,10 @@ jobs:
 
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}

.github/workflows/_build-and-test-locally.yml

@@ -37,17 +37,20 @@ env:
   RUST_BACKTRACE: 1
   COPT: '-Werror'
 
+permissions:
+  contents: read
+
 jobs:
   build-neon:
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }}
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }}
     permissions:
       id-token: write # aws-actions/configure-aws-credentials
       contents: read
     container:
       image: ${{ inputs.build-tools-image }}
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       # Raise locked memory limit for tokio-epoll-uring.
       # On 5.10 LTS kernels < 5.10.162 (and generally mainline kernels < 5.12),
       # io_uring will account the memory of the CQ and SQ as locked.
@@ -59,7 +62,12 @@ jobs:
       BUILD_TAG: ${{ inputs.build-tag }}
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
@@ -120,28 +128,28 @@ jobs:
 
       - name: Cache postgres v14 build
         id: cache_pg_14
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v14
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
 
       - name: Cache postgres v15 build
         id: cache_pg_15
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v15
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
 
       - name: Cache postgres v16 build
         id: cache_pg_16
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v16
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
 
       - name: Cache postgres v17 build
         id: cache_pg_17
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v17
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
@@ -221,7 +229,7 @@ jobs:
           fi
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
           aws-region: eu-central-1
           role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -318,19 +326,24 @@ jobs:
       contents: read
       statuses: write
     needs: [ build-neon ]
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }}
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }}
     container:
       image: ${{ inputs.build-tools-image }}
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       # for changed limits, see comments on `options:` earlier in this file
       options: --init --shm-size=512mb --ulimit memlock=67108864:67108864
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(format('{{"include":{0}}}', inputs.test-cfg)) }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 

.github/workflows/_check-codestyle-python.yml

@@ -12,20 +12,32 @@ defaults:
   run:
     shell: bash -euxo pipefail {0}
 
+permissions:
+  contents: read
+
 jobs:
   check-codestyle-python:
     runs-on: [ self-hosted, small ]
+
+    permissions:
+      packages: read
+
     container:
       image: ${{ inputs.build-tools-image }}
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
 
-      - uses: actions/cache@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/pypoetry/virtualenvs
           key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }}

.github/workflows/_check-codestyle-rust.yml

@@ -23,24 +23,32 @@ jobs:
   check-codestyle-rust:
     strategy:
       matrix:
-        arch: ${{ fromJson(inputs.archs) }}
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }}
+        arch: ${{ fromJSON(inputs.archs) }}
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }}
+
+    permissions:
+      packages: read
 
     container:
       image: ${{ inputs.build-tools-image }}
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
       - name: Cache cargo deps
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: |
             ~/.cargo/registry

.github/workflows/_create-release-pr.yml

@@ -20,6 +20,9 @@ defaults:
   run:
     shell: bash -euo pipefail {0}
 
+permissions:
+  contents: read
+
 jobs:
   create-release-branch:
     runs-on: ubuntu-22.04
@@ -28,7 +31,12 @@ jobs:
       contents: write # for `git push`
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         ref: ${{ inputs.source-branch }}
         fetch-depth: 0

.github/workflows/_meta.yml

@@ -9,6 +9,9 @@ on:
       build-tag:
         description: "Tag for the current workflow run"
         value: ${{ jobs.tags.outputs.build-tag }}
+      release-tag:
+        description: "Tag for the release if this is an RC PR run"
+        value: ${{ jobs.tags.outputs.release-tag }}
       previous-storage-release:
         description: "Tag of the last storage release"
         value: ${{ jobs.tags.outputs.storage }}
@@ -35,7 +38,8 @@ jobs:
   tags:
     runs-on: ubuntu-22.04
     outputs:
-      build-tag: ${{ steps.build-tag.outputs.tag }}
+      build-tag: ${{ steps.build-tag.outputs.build-tag }}
+      release-tag: ${{ steps.build-tag.outputs.release-tag }}
       compute: ${{ steps.previous-releases.outputs.compute }}
       proxy: ${{ steps.previous-releases.outputs.proxy }}
       storage: ${{ steps.previous-releases.outputs.storage }}
@@ -45,7 +49,12 @@ jobs:
       contents: read
     steps:
       # Need `fetch-depth: 0` to count the number of commits in the branch
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
 
@@ -79,16 +88,16 @@ jobs:
         run: |
           case $RUN_KIND in
           push-main)
-            echo "tag=$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
+            echo "build-tag=$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
             ;;
           storage-release)
-            echo "tag=release-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
+            echo "build-tag=release-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
             ;;
           proxy-release)
-            echo "tag=release-proxy-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
+            echo "build-tag=release-proxy-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
             ;;
           compute-release)
-            echo "tag=release-compute-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
+            echo "build-tag=release-compute-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
             ;;
           pr|storage-rc-pr|compute-rc-pr|proxy-rc-pr)
             BUILD_AND_TEST_RUN_ID=$(gh api --paginate \
@@ -96,10 +105,21 @@ jobs:
               -H "X-GitHub-Api-Version: 2022-11-28" \
               "/repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${CURRENT_SHA}&branch=${CURRENT_BRANCH}" \
               | jq '[.workflow_runs[] | select(.name == "Build and Test")][0].id // ("Error: No matching workflow run found." | halt_error(1))')
-            echo "tag=$BUILD_AND_TEST_RUN_ID" | tee -a $GITHUB_OUTPUT
+            echo "build-tag=$BUILD_AND_TEST_RUN_ID" | tee -a $GITHUB_OUTPUT
+            case $RUN_KIND in
+            storage-rc-pr)
+              echo "release-tag=release-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
+              ;;
+            proxy-rc-pr)
+              echo "release-tag=release-proxy-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
+              ;;
+            compute-rc-pr)
+              echo "release-tag=release-compute-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
+              ;;
+            esac
             ;;
           workflow-dispatch)
-            echo "tag=$GITHUB_RUN_ID" | tee -a $GITHUB_OUTPUT
+            echo "build-tag=$GITHUB_RUN_ID" | tee -a $GITHUB_OUTPUT
             ;;
           *)
             echo "Unexpected RUN_KIND ('${RUN_KIND}'), failing to assign build-tag!"
@@ -120,7 +140,7 @@ jobs:
 
       - name: Get the release PR run ID
         id: release-pr-run-id
-        if: ${{ contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), steps.run-kind.outputs.run-kind) }}
+        if: ${{ contains(fromJSON('["storage-release", "compute-release", "proxy-release"]'), steps.run-kind.outputs.run-kind) }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}

.github/workflows/_push-to-container-registry.yml

@@ -49,7 +49,12 @@ jobs:
       id-token: write  # Required for aws/azure login
       packages: write  # required for pushing to GHCR
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/scripts/push_with_image_map.py
           sparse-checkout-cone-mode: false
@@ -59,7 +64,7 @@ jobs:
 
       - name: Configure AWS credentials
         if: contains(inputs.image-map, 'amazonaws.com/')
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
           aws-region: "${{ inputs.aws-region }}"
           role-to-assume: "arn:aws:iam::${{ inputs.aws-account-id }}:role/${{ inputs.aws-role-to-assume }}"
@@ -67,7 +72,7 @@ jobs:
 
       - name: Login to ECR
         if: contains(inputs.image-map, 'amazonaws.com/')
-        uses: aws-actions/amazon-ecr-login@v2
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
         with:
           registries: "${{ inputs.aws-account-id }}"
 
@@ -86,14 +91,14 @@ jobs:
 
       - name: Login to GHCR
         if: contains(inputs.image-map, 'ghcr.io/')
-        uses: docker/login-action@v3
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
-          username: ${{ github.repository_owner }}
+          username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

.github/workflows/actionlint.yml

@@ -26,8 +26,13 @@ jobs:
     needs: [ check-permissions ]
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
-      - uses: reviewdog/action-actionlint@v1
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: reviewdog/action-actionlint@a5524e1c19e62881d79c1f1b9b6f09f16356e281 # v1.65.2
         env:
           # SC2046 - Quote this to prevent word splitting. - https://www.shellcheck.net/wiki/SC2046
           # SC2086 - Double quote to prevent globbing and word splitting. - https://www.shellcheck.net/wiki/SC2086

.github/workflows/approved-for-ci-run.yml

@@ -47,6 +47,11 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - run: gh pr --repo "${GITHUB_REPOSITORY}" edit "${PR_NUMBER}" --remove-label "approved-for-ci-run"
 
   create-or-update-pr-for-ci-run:
@@ -63,9 +68,14 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - run: gh pr --repo "${GITHUB_REPOSITORY}" edit "${PR_NUMBER}" --remove-label "approved-for-ci-run"
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           token: ${{ secrets.CI_ACCESS_TOKEN }}
@@ -153,6 +163,11 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Close PR and delete `ci-run/pr-${{ env.PR_NUMBER }}` branch
         run: |
           CLOSED="$(gh pr --repo ${GITHUB_REPOSITORY} list --head ${BRANCH} --json 'closed' --jq '.[].closed')"

.github/workflows/benchmarking.yml

@@ -87,17 +87,22 @@ jobs:
 
     runs-on: ${{ matrix.RUNNER }}
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials # necessary on Azure runners
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -164,7 +169,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |
@@ -190,17 +195,22 @@ jobs:
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -245,17 +255,22 @@ jobs:
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -314,7 +329,7 @@ jobs:
     # Post both success and failure to the Slack channel
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && !cancelled() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06T9AMNDQQ" # on-call-compute-staging-stream
         slack-message: |
@@ -346,13 +361,18 @@ jobs:
       tpch-compare-matrix: ${{ steps.tpch-compare-matrix.outputs.matrix }}
 
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
     - name: Generate matrix for pgbench benchmark
       id: pgbench-compare-matrix
       run: |
         region_id_default=${{ env.DEFAULT_REGION_ID }}
         runner_default='["self-hosted", "us-east-2", "x64"]'
         runner_azure='["self-hosted", "eastus2", "x64"]'
-        image_default="neondatabase/build-tools:pinned-bookworm"
+        image_default="ghcr.io/neondatabase/build-tools:pinned-bookworm"
         matrix='{
           "pg_version" : [
             16
@@ -368,18 +388,18 @@ jobs:
           "db_size": [ "10gb" ],
           "runner": ['"$runner_default"'],
           "image": [ "'"$image_default"'" ],
-          "include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier",       "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"',   "image": "neondatabase/build-tools:pinned-bookworm" },
-                      { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-new",      "db_size": "10gb","runner": '"$runner_azure"',   "image": "neondatabase/build-tools:pinned-bookworm" },
-                      { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-new",      "db_size": "50gb","runner": '"$runner_azure"',   "image": "neondatabase/build-tools:pinned-bookworm" },
-                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier",       "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]
+          "include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier",       "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"',   "image": "ghcr.io/neondatabase/build-tools:pinned-bookworm" },
+                      { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-new",      "db_size": "10gb","runner": '"$runner_azure"',   "image": "ghcr.io/neondatabase/build-tools:pinned-bookworm" },
+                      { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-new",      "db_size": "50gb","runner": '"$runner_azure"',   "image": "ghcr.io/neondatabase/build-tools:pinned-bookworm" },
+                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier",       "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
+                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               }]
         }'
 
         if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
@@ -441,7 +461,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix: ${{fromJson(needs.generate-matrices.outputs.pgbench-compare-matrix)}}
+      matrix: ${{fromJSON(needs.generate-matrices.outputs.pgbench-compare-matrix)}}
 
     env:
       TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
@@ -457,18 +477,23 @@ jobs:
     container:
       image: ${{ matrix.image }}
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     # Increase timeout to 8h, default timeout is 6h
     timeout-minutes: 480
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -483,7 +508,7 @@ jobs:
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
 
     - name: Create Neon Project
-      if: contains(fromJson('["neonvm-captest-new", "neonvm-captest-new-many-tables", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform)
+      if: contains(fromJSON('["neonvm-captest-new", "neonvm-captest-new-many-tables", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform)
       id: create-neon-project
       uses: ./.github/actions/neon-project-create
       with:
@@ -523,7 +548,7 @@ jobs:
     # without (neonvm-captest-new)
     # and with (neonvm-captest-new-many-tables) many relations in the database
     - name: Create many relations before the run
-      if: contains(fromJson('["neonvm-captest-new-many-tables"]'), matrix.platform)
+      if: contains(fromJSON('["neonvm-captest-new-many-tables"]'), matrix.platform)
       uses: ./.github/actions/run-python-test-set
       with:
         build_type: ${{ env.BUILD_TYPE }}
@@ -600,7 +625,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |
@@ -642,17 +667,22 @@ jobs:
 
     runs-on: ${{ matrix.RUNNER }}
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -726,7 +756,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |
@@ -753,7 +783,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }}
+      matrix: ${{ fromJSON(needs.generate-matrices.outputs.olap-compare-matrix) }}
 
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
@@ -767,10 +797,10 @@ jobs:
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     # Increase timeout to 12h, default timeout is 6h
@@ -778,10 +808,15 @@ jobs:
     timeout-minutes: 720
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -854,7 +889,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |
@@ -880,7 +915,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrices.outputs.tpch-compare-matrix) }}
+      matrix: ${{ fromJSON(needs.generate-matrices.outputs.tpch-compare-matrix) }}
 
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
@@ -892,17 +927,22 @@ jobs:
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -979,7 +1019,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |
@@ -999,7 +1039,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }}
+      matrix: ${{ fromJSON(needs.generate-matrices.outputs.olap-compare-matrix) }}
 
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
@@ -1011,17 +1051,22 @@ jobs:
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -1091,7 +1136,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |

.github/workflows/build-build-tools-image.yml

@@ -19,7 +19,7 @@ on:
         value: ${{ jobs.check-image.outputs.tag }}
       image:
         description: "build-tools image"
-        value: neondatabase/build-tools:${{ jobs.check-image.outputs.tag }}
+        value: ghcr.io/neondatabase/build-tools:${{ jobs.check-image.outputs.tag }}
 
 defaults:
   run:
@@ -49,8 +49,22 @@ jobs:
       everything: ${{ steps.set-more-variables.outputs.everything }}
       found: ${{ steps.set-more-variables.outputs.found }}
 
+    permissions:
+      packages: read
+
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set variables
         id: set-variables
@@ -70,12 +84,12 @@ jobs:
         env:
           IMAGE_TAG: ${{ steps.set-variables.outputs.image-tag }}
           EVERYTHING: |
-            ${{ contains(fromJson(steps.set-variables.outputs.archs), 'x64') &&
-                contains(fromJson(steps.set-variables.outputs.archs), 'arm64') &&
-                contains(fromJson(steps.set-variables.outputs.debians), 'bullseye') &&
-                contains(fromJson(steps.set-variables.outputs.debians), 'bookworm') }}
+            ${{ contains(fromJSON(steps.set-variables.outputs.archs), 'x64') &&
+                contains(fromJSON(steps.set-variables.outputs.archs), 'arm64') &&
+                contains(fromJSON(steps.set-variables.outputs.debians), 'bullseye') &&
+                contains(fromJSON(steps.set-variables.outputs.debians), 'bookworm') }}
         run: |
-          if docker manifest inspect neondatabase/build-tools:${IMAGE_TAG}; then
+          if docker manifest inspect ghcr.io/neondatabase/build-tools:${IMAGE_TAG}; then
             found=true
           else
             found=false
@@ -90,31 +104,45 @@ jobs:
 
     strategy:
       matrix:
-        arch: ${{ fromJson(needs.check-image.outputs.archs) }}
-        debian: ${{ fromJson(needs.check-image.outputs.debians) }}
+        arch: ${{ fromJSON(needs.check-image.outputs.archs) }}
+        debian: ${{ fromJSON(needs.check-image.outputs.debians) }}
 
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
+    permissions:
+      packages: write
+
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
         with:
           cache-binary: false
 
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: cache.neon.build
           username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }}
           password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }}
 
-      - uses: docker/build-push-action@v6
+      - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           file: build-tools.Dockerfile
           context: .
@@ -126,35 +154,49 @@ jobs:
           cache-from: type=registry,ref=cache.neon.build/build-tools:cache-${{ matrix.debian }}-${{ matrix.arch }}
           cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/build-tools:cache-{0}-{1},mode=max', matrix.debian, matrix.arch) || '' }}
           tags: |
-            neondatabase/build-tools:${{ needs.check-image.outputs.tag }}-${{ matrix.debian }}-${{ matrix.arch }}
+            ghcr.io/neondatabase/build-tools:${{ needs.check-image.outputs.tag }}-${{ matrix.debian }}-${{ matrix.arch }}
 
   merge-images:
     needs: [ check-image, build-image ]
     runs-on: ubuntu-22.04
 
+    permissions:
+      packages: write
+
     steps:
-      - uses: docker/login-action@v3
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Create multi-arch image
         env:
           DEFAULT_DEBIAN_VERSION: bookworm
-          ARCHS: ${{ join(fromJson(needs.check-image.outputs.archs), ' ') }}
-          DEBIANS: ${{ join(fromJson(needs.check-image.outputs.debians), ' ') }}
+          ARCHS: ${{ join(fromJSON(needs.check-image.outputs.archs), ' ') }}
+          DEBIANS: ${{ join(fromJSON(needs.check-image.outputs.debians), ' ') }}
           EVERYTHING: ${{ needs.check-image.outputs.everything }}
           IMAGE_TAG: ${{ needs.check-image.outputs.tag }}
         run: |
           for debian in ${DEBIANS}; do
-            tags=("-t" "neondatabase/build-tools:${IMAGE_TAG}-${debian}")
+            tags=("-t" "ghcr.io/neondatabase/build-tools:${IMAGE_TAG}-${debian}")
 
             if [ "${EVERYTHING}" == "true" ] && [ "${debian}" == "${DEFAULT_DEBIAN_VERSION}" ]; then
-              tags+=("-t" "neondatabase/build-tools:${IMAGE_TAG}")
+              tags+=("-t" "ghcr.io/neondatabase/build-tools:${IMAGE_TAG}")
             fi
 
             for arch in ${ARCHS}; do
-              tags+=("neondatabase/build-tools:${IMAGE_TAG}-${debian}-${arch}")
+              tags+=("ghcr.io/neondatabase/build-tools:${IMAGE_TAG}-${debian}-${arch}")
             done
 
             docker buildx imagetools create "${tags[@]}"

.github/workflows/build-macos.yml

@@ -28,6 +28,9 @@ env:
 # - You can connect up to four levels of workflows
 # - You can call a maximum of 20 unique reusable workflows from a single workflow file.
 # https://docs.github.com/en/actions/sharing-automations/reusing-workflows#limitations
+permissions:
+  contents: read
+
 jobs:
   build-pgxn:
     if: |
@@ -40,14 +43,19 @@ jobs:
     runs-on: macos-15
     strategy:
       matrix:
-        postgres-version: ${{ inputs.rebuild_everything && fromJson('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }}
+        postgres-version: ${{ inputs.rebuild_everything && fromJSON('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }}
     env:
       # Use release build only, to have less debug info around
       # Hence keeping target/ (and general cache size) smaller
       BUILD_TYPE: release
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout main repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Set pg ${{ matrix.postgres-version }} for caching
         id: pg_rev
@@ -55,7 +63,7 @@ jobs:
 
       - name: Cache postgres ${{ matrix.postgres-version }} build
         id: cache_pg
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/${{ matrix.postgres-version }}
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ matrix.postgres-version }}-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
@@ -107,8 +115,13 @@ jobs:
       # Hence keeping target/ (and general cache size) smaller
       BUILD_TYPE: release
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout main repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Set pg v17 for caching
         id: pg_rev
@@ -116,14 +129,14 @@ jobs:
 
       - name: Cache postgres v17 build
         id: cache_pg
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v17
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
 
       - name: Cache walproposer-lib
         id: cache_walproposer_lib
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/build/walproposer-lib
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
@@ -165,8 +178,13 @@ jobs:
       # Hence keeping target/ (and general cache size) smaller
       BUILD_TYPE: release
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout main repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
@@ -185,31 +203,31 @@ jobs:
 
       - name: Cache postgres v14 build
         id: cache_pg
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v14
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v14-${{ steps.pg_rev_v14.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
       - name: Cache postgres v15 build
         id: cache_pg_v15
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v15
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v15-${{ steps.pg_rev_v15.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
       - name: Cache postgres v16 build
         id: cache_pg_v16
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v16
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v16-${{ steps.pg_rev_v16.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
       - name: Cache postgres v17 build
         id: cache_pg_v17
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/v17
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
 
       - name: Cache cargo deps (only for v17)
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: |
             ~/.cargo/registry
@@ -220,7 +238,7 @@ jobs:
 
       - name: Cache walproposer-lib
         id: cache_walproposer_lib
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: pg_install/build/walproposer-lib
           key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

.github/workflows/build_and_test.yml

@@ -37,6 +37,11 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Cancel previous e2e-tests runs for this PR
         env:
           GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
@@ -53,8 +58,13 @@ jobs:
       check-rust-dependencies: ${{ steps.files-changed.outputs.rust_dependencies }}
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
@@ -77,25 +87,33 @@ jobs:
     secrets: inherit
 
   check-codestyle-python:
-    needs: [ check-permissions, build-build-tools-image ]
+    needs: [ meta, check-permissions, build-build-tools-image ]
+    # No need to run on `main` because we this in the merge queue
+    if: ${{ needs.meta.outputs.run-kind == 'pr' }}
     uses: ./.github/workflows/_check-codestyle-python.yml
     with:
       build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
     secrets: inherit
 
   check-codestyle-jsonnet:
-    needs: [ check-permissions, build-build-tools-image ]
+    needs: [ meta, check-permissions, build-build-tools-image ]
+    if: ${{ contains(fromJSON('["pr", "push-main"]'), needs.meta.outputs.run-kind) }}
     runs-on: [ self-hosted, small ]
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Check Jsonnet code formatting
         run: |
@@ -107,12 +125,17 @@ jobs:
     needs: [ check-permissions ]
     runs-on: ubuntu-22.04
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
-      - uses: dorny/paths-filter@v3
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
         id: check-if-submodules-changed
         with:
           filters: |
@@ -121,7 +144,7 @@ jobs:
 
       - name: Check vendor/postgres-v14 submodule reference
         if: steps.check-if-submodules-changed.outputs.vendor == 'true'
-        uses: jtmullen/submodule-branch-check-action@v1
+        uses: jtmullen/submodule-branch-check-action@ab0d3a69278e3fa0a2d4f3be3199d2514b676e13 # v1.3.0
         with:
           path: "vendor/postgres-v14"
           fetch_depth: "50"
@@ -130,7 +153,7 @@ jobs:
 
       - name: Check vendor/postgres-v15 submodule reference
         if: steps.check-if-submodules-changed.outputs.vendor == 'true'
-        uses: jtmullen/submodule-branch-check-action@v1
+        uses: jtmullen/submodule-branch-check-action@ab0d3a69278e3fa0a2d4f3be3199d2514b676e13 # v1.3.0
         with:
           path: "vendor/postgres-v15"
           fetch_depth: "50"
@@ -139,7 +162,7 @@ jobs:
 
       - name: Check vendor/postgres-v16 submodule reference
         if: steps.check-if-submodules-changed.outputs.vendor == 'true'
-        uses: jtmullen/submodule-branch-check-action@v1
+        uses: jtmullen/submodule-branch-check-action@ab0d3a69278e3fa0a2d4f3be3199d2514b676e13 # v1.3.0
         with:
           path: "vendor/postgres-v16"
           fetch_depth: "50"
@@ -148,7 +171,7 @@ jobs:
 
       - name: Check vendor/postgres-v17 submodule reference
         if: steps.check-if-submodules-changed.outputs.vendor == 'true'
-        uses: jtmullen/submodule-branch-check-action@v1
+        uses: jtmullen/submodule-branch-check-action@ab0d3a69278e3fa0a2d4f3be3199d2514b676e13 # v1.3.0
         with:
           path: "vendor/postgres-v17"
           fetch_depth: "50"
@@ -156,7 +179,9 @@ jobs:
           pass_if_unchanged: true
 
   check-codestyle-rust:
-    needs: [ check-permissions, build-build-tools-image ]
+    needs: [ meta, check-permissions, build-build-tools-image ]
+    # No need to run on `main` because we this in the merge queue
+    if: ${{ needs.meta.outputs.run-kind == 'pr' }}
     uses: ./.github/workflows/_check-codestyle-rust.yml
     with:
       build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
@@ -164,8 +189,8 @@ jobs:
     secrets: inherit
 
   check-dependencies-rust:
-    needs: [ files-changed, build-build-tools-image ]
-    if: ${{ needs.files-changed.outputs.check-rust-dependencies == 'true' }}
+    needs: [ meta, files-changed, build-build-tools-image ]
+    if: ${{ needs.files-changed.outputs.check-rust-dependencies == 'true' && needs.meta.outputs.run-kind == 'pr' }}
     uses: ./.github/workflows/cargo-deny.yml
     with:
       build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
@@ -173,12 +198,13 @@ jobs:
 
   build-and-test-locally:
     needs: [ meta, build-build-tools-image ]
+    if: ${{ contains(fromJSON('["pr", "push-main"]'), needs.meta.outputs.run-kind) }}
     strategy:
       fail-fast: false
       matrix:
         arch: [ x64, arm64 ]
         # Do not build or run tests in debug for release branches
-        build-type: ${{ fromJson((startsWith(github.ref_name, 'release') && github.event_name == 'push') && '["release"]' || '["debug", "release"]') }}
+        build-type: ${{ fromJSON((startsWith(github.ref_name, 'release') && github.event_name == 'push') && '["release"]' || '["debug", "release"]') }}
         include:
           - build-type: release
             arch: arm64
@@ -209,15 +235,20 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Cache poetry deps
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/pypoetry/virtualenvs
           key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }}
@@ -248,8 +279,8 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       # for changed limits, see comments on `options:` earlier in this file
       options: --init --shm-size=512mb --ulimit memlock=67108864:67108864
     strategy:
@@ -259,8 +290,13 @@ jobs:
         pytest_split_group: [ 1, 2, 3, 4, 5 ]
         build_type: [ release ]
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Pytest benchmarks
         uses: ./.github/actions/run-python-test-set
@@ -288,7 +324,12 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-    - uses: slackapi/slack-github-action@v2
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
       with:
         method: chat.postMessage
         token: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -314,12 +355,17 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Create Allure report
         if: ${{ !cancelled() }}
@@ -331,7 +377,7 @@ jobs:
         env:
           REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
 
-      - uses: actions/github-script@v7
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         if: ${{ !cancelled() }}
         with:
           # Retry script for 5XX server errors: https://github.com/actions/github-script#retries
@@ -367,8 +413,8 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
     strategy:
       fail-fast: false
@@ -379,7 +425,12 @@ jobs:
         coverage-json: ${{ steps.upload-coverage-report-new.outputs.summary-json }}
     steps:
       # Need `fetch-depth: 0` for differential coverage (to get diff between two commits)
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
           fetch-depth: 0
@@ -450,7 +501,7 @@ jobs:
           REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/lcov/summary.json
           echo "summary-json=${REPORT_URL}" >> $GITHUB_OUTPUT
 
-      - uses: actions/github-script@v7
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         env:
           REPORT_URL_NEW: ${{ steps.upload-coverage-report-new.outputs.report-url }}
           COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
@@ -470,14 +521,20 @@ jobs:
             })
 
   trigger-e2e-tests:
-    # Depends on jobs that can get skipped
+    # !failure() && !cancelled() because it depends on jobs that can get skipped
     if: >-
       ${{
         (
-          !github.event.pull_request.draft
-          || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft')
-          || needs.meta.outputs.run-kind == 'push-main'
-        ) && !failure() && !cancelled()
+          (
+            needs.meta.outputs.run-kind == 'pr'
+            && (
+              !github.event.pull_request.draft
+              || contains(github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft')
+            )
+          )
+          || contains(fromJSON('["push-main", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind)
+        )
+        && !failure() && !cancelled()
       }}
     needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, meta ]
     uses: ./.github/workflows/trigger-e2e-tests.yml
@@ -492,30 +549,44 @@ jobs:
       matrix:
         arch: [ x64, arm64 ]
 
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
+
+    permissions:
+      packages: write
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
       - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
         with:
           cache-binary: false
 
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: cache.neon.build
           username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }}
           password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }}
 
-      - uses: docker/build-push-action@v6
+      - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: .
           # ARM-specific flags are recommended for Graviton ≥ 2, these flags are also supported by Ampere Altra (Azure)
@@ -523,7 +594,7 @@ jobs:
           build-args: |
             ADDITIONAL_RUSTFLAGS=${{ matrix.arch == 'arm64' && '-Ctarget-feature=+lse -Ctarget-cpu=neoverse-n1' || '' }}
             GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }}
-            BUILD_TAG=${{ needs.meta.outputs.build-tag }}
+            BUILD_TAG=${{ needs.meta.outputs.release-tag || needs.meta.outputs.build-tag }}
             TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-bookworm
             DEBIAN_VERSION=bookworm
           provenance: false
@@ -533,7 +604,7 @@ jobs:
           cache-from: type=registry,ref=cache.neon.build/neon:cache-bookworm-${{ matrix.arch }}
           cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/neon:cache-{0}-{1},mode=max', 'bookworm', matrix.arch) || '' }}
           tags: |
-            neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-${{ matrix.arch }}
+            ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-${{ matrix.arch }}
 
   neon-image:
     needs: [ neon-image-arch, meta ]
@@ -543,19 +614,26 @@ jobs:
       id-token: write # aws-actions/configure-aws-credentials
       statuses: write
       contents: read
+      packages: write
 
     steps:
-      - uses: docker/login-action@v3
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
         with:
-          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+          egress-policy: audit
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create multi-arch image
         run: |
-          docker buildx imagetools create -t neondatabase/neon:${{ needs.meta.outputs.build-tag }} \
-                                          -t neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm \
-                                             neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-x64 \
-                                             neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-arm64
+          docker buildx imagetools create -t ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }} \
+                                          -t ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm \
+                                             ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-x64 \
+                                             ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-arm64
 
   compute-node-image-arch:
     needs: [ check-permissions, build-build-tools-image, meta ]
@@ -564,6 +642,7 @@ jobs:
       id-token: write # aws-actions/configure-aws-credentials
       statuses: write
       contents: read
+      packages: write
     strategy:
       fail-fast: false
       matrix:
@@ -582,15 +661,20 @@ jobs:
             debian: bookworm
         arch: [ x64, arm64 ]
 
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
       - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193
-      - uses: docker/setup-buildx-action@v3
+      - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
         with:
           cache-binary: false
           # Disable parallelism for docker buildkit.
@@ -599,25 +683,31 @@ jobs:
             [worker.oci]
               max-parallelism = 1
 
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: cache.neon.build
           username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }}
           password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }}
 
       - name: Build compute-node image
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: .
           build-args: |
             GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }}
             PG_VERSION=${{ matrix.version.pg }}
-            BUILD_TAG=${{ needs.meta.outputs.build-tag }}
+            BUILD_TAG=${{ needs.meta.outputs.release-tag || needs.meta.outputs.build-tag }}
             TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-${{ matrix.version.debian }}
             DEBIAN_VERSION=${{ matrix.version.debian }}
           provenance: false
@@ -627,17 +717,17 @@ jobs:
           cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }}
           cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-node-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }}
           tags: |
-            neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-${{ matrix.arch }}
+            ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-${{ matrix.arch }}
 
       - name: Build neon extensions test image
         if: matrix.version.pg >= 'v16'
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: .
           build-args: |
             GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }}
             PG_VERSION=${{ matrix.version.pg }}
-            BUILD_TAG=${{ needs.meta.outputs.build-tag }}
+            BUILD_TAG=${{ needs.meta.outputs.release-tag || needs.meta.outputs.build-tag }}
             TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-${{ matrix.version.debian }}
             DEBIAN_VERSION=${{ matrix.version.debian }}
           provenance: false
@@ -647,7 +737,7 @@ jobs:
           target: extension-tests
           cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }}
           tags: |
-            neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{needs.meta.outputs.build-tag}}-${{ matrix.version.debian }}-${{ matrix.arch }}
+            ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{needs.meta.outputs.build-tag}}-${{ matrix.version.debian }}-${{ matrix.arch }}
 
   compute-node-image:
     needs: [ compute-node-image-arch, meta ]
@@ -656,6 +746,7 @@ jobs:
       id-token: write # aws-actions/configure-aws-credentials
       statuses: write
       contents: read
+      packages: write
     runs-on: ubuntu-22.04
 
     strategy:
@@ -672,30 +763,39 @@ jobs:
             debian: bookworm
 
     steps:
-      - uses: docker/login-action@v3
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
         with:
-          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+          egress-policy: audit
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create multi-arch compute-node image
         run: |
-          docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-                                          -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \
-                                             neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
-                                             neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
+          docker buildx imagetools create -t ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
+                                          -t ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \
+                                             ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
+                                             ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
 
       - name: Create multi-arch neon-test-extensions image
         if: matrix.version.pg >= 'v16'
         run: |
-          docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-                                          -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \
-                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
-                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
+          docker buildx imagetools create -t ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
+                                          -t ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \
+                                             ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
+                                             ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
 
   vm-compute-node-image-arch:
     needs: [ check-permissions, meta, compute-node-image ]
     if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }}
+    permissions:
+      contents: read
+      packages: write
     strategy:
       fail-fast: false
       matrix:
@@ -713,7 +813,12 @@ jobs:
       VM_BUILDER_VERSION: v0.42.2
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Downloading vm-builder
         run: |
@@ -721,33 +826,36 @@ jobs:
           chmod +x vm-builder
 
       - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193
-      - uses: docker/login-action@v3
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
-          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       # Note: we need a separate pull step here because otherwise vm-builder will try to pull, and
       # it won't have the proper authentication (written at v0.6.0)
       - name: Pulling compute-node image
         run: |
-          docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}
+          docker pull ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}
 
       - name: Build vm image
         run: |
           ./vm-builder \
             -size=2G \
             -spec=compute/vm-image-spec-${{ matrix.version.debian }}.yaml \
-            -src=neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-            -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }} \
+            -src=ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
+            -dst=ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }} \
             -target-arch=linux/${{ matrix.arch }}
 
       - name: Pushing vm-compute-node image
         run: |
-          docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }}
+          docker push ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }}
 
   vm-compute-node-image:
     needs: [ vm-compute-node-image-arch, meta ]
     if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
+    permissions:
+      packages: write
     runs-on: ubuntu-22.04
     strategy:
       matrix:
@@ -758,16 +866,22 @@ jobs:
           - pg: v16
           - pg: v17
     steps:
-      - uses: docker/login-action@v3
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
         with:
-          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+          egress-policy: audit
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create multi-arch compute-node image
         run: |
-          docker buildx imagetools create -t neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-                                             neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-amd64 \
-                                             neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-arm64
+          docker buildx imagetools create -t ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
+                                             ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-amd64 \
+                                             ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-arm64
 
 
   test-images:
@@ -785,18 +899,33 @@ jobs:
         arch: [ x64, arm64 ]
         pg_version: [v16, v17]
 
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }}
+    permissions:
+      packages: read
+
+    runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }}
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193
-      - uses: docker/login-action@v3
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
-      # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # `ghcr.io/neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
       # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
       # Regular pageserver version string looks like
       #   Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: []
@@ -807,7 +936,7 @@ jobs:
         shell: bash # ensure no set -e for better error messages
         if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
         run: |
-          pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
+          pageserver_version=$(docker run --rm ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
 
           echo "Pageserver version string: $pageserver_version"
 
@@ -881,7 +1010,12 @@ jobs:
       compute-dev: ${{ steps.generate.outputs.compute-dev }}
       compute-prod: ${{ steps.generate.outputs.compute-prod }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/scripts/generate_image_maps.py
           sparse-checkout-cone-mode: false
@@ -892,7 +1026,7 @@ jobs:
         env:
           SOURCE_TAG: >-
             ${{
-              contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), needs.meta.outputs.run-kind)
+              contains(fromJSON('["storage-release", "compute-release", "proxy-release"]'), needs.meta.outputs.run-kind)
               && needs.meta.outputs.release-pr-run-id
               || needs.meta.outputs.build-tag
             }}
@@ -978,18 +1112,21 @@ jobs:
       acr-registry-name: ${{ vars.AZURE_PROD_REGISTRY_NAME }}
     secrets: inherit
 
-  push-neon-test-extensions-image-ghcr:
+  push-neon-test-extensions-image-dockerhub:
     if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
     needs: [ meta, compute-node-image ]
     uses: ./.github/workflows/_push-to-container-registry.yml
+    permissions:
+      packages: write
+      id-token: write
     with:
       image-map: |
         {
-          "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [
-            "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}"
+          "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [
+            "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}"
           ],
-          "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [
-            "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}"
+          "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [
+            "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}"
           ]
         }
     secrets: inherit
@@ -998,14 +1135,17 @@ jobs:
     if: ${{ needs.meta.outputs.run-kind == 'push-main' }}
     needs: [ meta, compute-node-image ]
     uses: ./.github/workflows/_push-to-container-registry.yml
+    permissions:
+      packages: write
+      id-token: write
     with:
       image-map: |
         {
-          "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [
+          "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [
             "docker.io/neondatabase/neon-test-extensions-v16:latest",
             "ghcr.io/neondatabase/neon-test-extensions-v16:latest"
           ],
-          "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [
+          "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [
             "docker.io/neondatabase/neon-test-extensions-v17:latest",
             "ghcr.io/neondatabase/neon-test-extensions-v17:latest"
           ]
@@ -1014,16 +1154,19 @@ jobs:
 
   add-release-tag-to-neon-test-extensions-image:
     if: ${{ needs.meta.outputs.run-kind == 'compute-release' }}
-    needs: [ meta, compute-node-image ]
+    needs: [ meta ]
     uses: ./.github/workflows/_push-to-container-registry.yml
+    permissions:
+      packages: write
+      id-token: write
     with:
       image-map: |
         {
-          "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.release-pr-run-id }}": [
+          "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.release-pr-run-id }}": [
             "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}",
             "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}"
           ],
-          "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.release-pr-run-id }}": [
+          "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.release-pr-run-id }}": [
             "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}",
             "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}"
           ]
@@ -1040,6 +1183,11 @@ jobs:
       contents: write
       pull-requests: write
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Set PR's status to pending and request a remote CI test
         run: |
           COMMIT_SHA=${{ github.event.pull_request.head.sha || github.sha }}
@@ -1121,11 +1269,16 @@ jobs:
     runs-on: [ self-hosted, small ]
     container: ${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/ansible:latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Create git tag and GitHub release
         if: ${{ contains(fromJSON('["storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind) }}
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         env:
           TAG: "${{ needs.meta.outputs.build-tag }}"
           BRANCH: "${{ github.ref_name }}"
@@ -1273,8 +1426,13 @@ jobs:
     if: github.ref_name == 'release' && needs.deploy.result != 'success' && always()
     runs-on: ubuntu-22.04
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Post release-deploy failure to team-storage slack channel
-        uses: slackapi/slack-github-action@v2
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           method: chat.postMessage
           token: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -1295,7 +1453,12 @@ jobs:
 
     runs-on: ubuntu-22.04
     steps:
-      - uses: aws-actions/configure-aws-credentials@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
           aws-region: eu-central-1
           role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -1383,15 +1546,20 @@ jobs:
     steps:
       # The list of possible results:
       # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Fail the job if any of the dependencies do not succeed
         run: exit 1
         if: |
           contains(needs.*.result, 'failure')
           || contains(needs.*.result, 'cancelled')
-          || (needs.check-dependencies-rust.result == 'skipped' && needs.files-changed.outputs.check-rust-dependencies == 'true')
-          || needs.build-and-test-locally.result == 'skipped'
-          || needs.check-codestyle-python.result == 'skipped'
-          || needs.check-codestyle-rust.result == 'skipped'
+          || (needs.check-dependencies-rust.result == 'skipped' && needs.files-changed.outputs.check-rust-dependencies == 'true' && needs.meta.outputs.run-kind == 'pr')
+          || (needs.build-and-test-locally.result == 'skipped' && needs.meta.outputs.run-kind == 'pr')
+          || (needs.check-codestyle-python.result == 'skipped' && needs.meta.outputs.run-kind == 'pr')
+          || (needs.check-codestyle-rust.result == 'skipped' && needs.meta.outputs.run-kind == 'pr')
           || needs.files-changed.result == 'skipped'
           || (needs.push-compute-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind))
           || (needs.push-neon-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind))

.github/workflows/build_and_test_with_sanitizers.yml

@@ -33,7 +33,12 @@ jobs:
 
     steps:
       # Need `fetch-depth: 0` to count the number of commits in the branch
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0
 
@@ -94,12 +99,17 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Create Allure report
         if: ${{ !cancelled() }}
@@ -111,7 +121,7 @@ jobs:
         env:
           REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
 
-      - uses: actions/github-script@v7
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         if: ${{ !cancelled() }}
         with:
           # Retry script for 5XX server errors: https://github.com/actions/github-script#retries

.github/workflows/cargo-deny.yml

@@ -9,6 +9,9 @@ on:
   schedule:
     - cron: '0 10 * * *'
 
+permissions:
+  contents: read
+
 jobs:
   cargo-deny:
     strategy:
@@ -24,16 +27,24 @@ jobs:
 
     runs-on: [self-hosted, small]
 
+    permissions:
+      packages: read
+
     container:
-      image: ${{ inputs.build-tools-image || 'neondatabase/build-tools:pinned' }}
+      image: ${{ inputs.build-tools-image || 'ghcr.io/neondatabase/build-tools:pinned' }}
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ matrix.ref }}
 
@@ -45,7 +56,7 @@ jobs:
 
       - name: Post to a Slack channel
         if: ${{ github.event_name == 'schedule' && failure() }}
-        uses: slackapi/slack-github-action@v2
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           method: chat.postMessage
           token: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/check-permissions.yml

@@ -18,6 +18,11 @@ jobs:
   check-permissions:
     runs-on: ubuntu-22.04
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@v2
+      with:
+        egress-policy: audit
+
     - name: Disallow CI runs on PRs from forks
       if: |
         inputs.github-event-name  == 'pull_request' &&

.github/workflows/cleanup-caches-by-a-branch.yml

@@ -11,6 +11,11 @@ jobs:
   cleanup:
     runs-on: ubuntu-22.04
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
       - name: Cleanup
         run: |
           gh extension install actions/gh-actions-cache

.github/workflows/cloud-regress.yml

@@ -37,14 +37,19 @@ jobs:
 
     runs-on: us-east-2
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
@@ -121,7 +126,7 @@ jobs:
 
       - name: Post to a Slack channel
         if: ${{ github.event.schedule && failure() }}
-        uses: slackapi/slack-github-action@v1
+        uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
         with:
           channel-id: ${{ vars.SLACK_ON_CALL_QA_STAGING_STREAM }}
           slack-message: |

.github/workflows/fast-forward.yml

@@ -13,6 +13,11 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
       - name: Remove fast-forward label to PR
         env:
           GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}

.github/workflows/force-test-extensions-upgrade.yml

@@ -34,7 +34,12 @@ jobs:
     runs-on: small
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: false
 
@@ -67,7 +72,7 @@ jobs:
 
       - name: Post to the Slack channel
         if: ${{ github.event.schedule && failure() }}
-        uses: slackapi/slack-github-action@v1
+        uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
         with:
           channel-id: ${{ vars.SLACK_ON_CALL_QA_STAGING_STREAM }}
           slack-message: |

.github/workflows/ingest_benchmark.yml

@@ -23,6 +23,9 @@ concurrency:
   group: ingest-bench-workflow
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   ingest:
     strategy:
@@ -67,18 +70,23 @@ jobs:
       PGCOPYDB_LIB_PATH: /pgcopydb/lib
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
     timeout-minutes: 1440
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials # necessary to download artefacts
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}

.github/workflows/label-for-external-users.yml

@@ -27,6 +27,11 @@ jobs:
       is-member: ${{ steps.check-user.outputs.is-member }}
 
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@v2
+      with:
+        egress-policy: audit
+
     - name: Check whether `${{ github.actor }}` is a member of `${{ github.repository_owner }}`
       id: check-user
       env:
@@ -69,6 +74,11 @@ jobs:
       issues: write        # for `gh issue edit`
 
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@v2
+      with:
+        egress-policy: audit
+
     - name: Add `${{ env.LABEL }}` label
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/large_oltp_benchmark.yml

@@ -24,6 +24,9 @@ concurrency:
   group: large-oltp-bench-workflow
   cancel-in-progress: false
 
+permissions:
+  contents: read
+
 jobs:
   oltp:
     strategy:
@@ -50,10 +53,10 @@ jobs:
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     # Increase timeout to 2 days, default timeout is 6h - database maintenance can take a long time
@@ -62,10 +65,15 @@ jobs:
     timeout-minutes: 2880
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Configure AWS credentials # necessary to download artefacts
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -160,7 +168,7 @@ jobs:
         api_key: ${{ secrets.NEON_STAGING_API_KEY }}
 
     - name: Configure AWS credentials # again because prior steps could have exceeded 5 hours
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -175,7 +183,7 @@ jobs:
   
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |

.github/workflows/lint-release-pr.yml

@@ -7,12 +7,20 @@ on:
       - release-proxy
       - release-compute
 
+permissions:
+  contents: read
+
 jobs:
   lint-release-pr:
     runs-on: ubuntu-22.04
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout PR branch
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 0  # Fetch full history for git operations
           ref: ${{ github.event.pull_request.head.ref }}

.github/workflows/neon_extra_builds.yml

@@ -42,8 +42,13 @@ jobs:
       rebuild_everything: ${{ steps.files_changed.outputs.rebuild_neon_extra || steps.files_changed.outputs.rebuild_macos }}
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
@@ -71,8 +76,8 @@ jobs:
     uses: ./.github/workflows/build-macos.yml
     with:
       pg_versions: ${{ needs.files-changed.outputs.postgres_changes }}
-      rebuild_rust_code: ${{ fromJson(needs.files-changed.outputs.rebuild_rust_code) }}
-      rebuild_everything: ${{ fromJson(needs.files-changed.outputs.rebuild_everything) }}
+      rebuild_rust_code: ${{ fromJSON(needs.files-changed.outputs.rebuild_rust_code) }}
+      rebuild_everything: ${{ fromJSON(needs.files-changed.outputs.rebuild_everything) }}
 
   gather-rust-build-stats:
     needs: [ check-permissions, build-build-tools-image, files-changed ]
@@ -90,8 +95,8 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
 
     env:
@@ -101,8 +106,13 @@ jobs:
       CARGO_INCREMENTAL: 0
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
 
@@ -117,7 +127,7 @@ jobs:
         run: cargo build --all --release --timings -j$(nproc)
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
           aws-region: eu-central-1
           role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
@@ -134,7 +144,7 @@ jobs:
           echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT
 
       - name: Publish build stats report
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         env:
           REPORT_URL: ${{ steps.upload-stats.outputs.report-url }}
           SHA: ${{ github.event.pull_request.head.sha || github.sha }}

.github/workflows/periodic_pagebench.yml

@@ -25,6 +25,9 @@ concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: false
 
+permissions:
+  contents: read
+
 jobs:
   trigger_bench_on_ec2_machine_in_eu_central_1:
     permissions:
@@ -34,10 +37,10 @@ jobs:
       pull-requests: write
     runs-on: [ self-hosted, small ]
     container:
-      image: neondatabase/build-tools:pinned-bookworm
+      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init
     timeout-minutes: 360  # Set the timeout to 6 hours
     env:
@@ -48,13 +51,18 @@ jobs:
     steps:
     # we don't need the neon source code because we run everything remotely
     # however we still need the local github actions to run the allure step below
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Show my own (github runner) external IP address - usefull for IP allowlisting
       run: curl https://ifconfig.me
 
     - name: Assume AWS OIDC role that allows to manage (start/stop/describe... EC machine)
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_MANAGE_BENCHMARK_EC2_VMS_ARN }}
@@ -143,7 +151,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: "Periodic pagebench testing on dedicated hardware: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
@@ -161,7 +169,7 @@ jobs:
 
     - name: Assume AWS OIDC role that allows to manage (start/stop/describe... EC machine)
       if: always() && steps.poll_step.outputs.too_many_runs != 'true'
-      uses: aws-actions/configure-aws-credentials@v4
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-region: eu-central-1
         role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_MANAGE_BENCHMARK_EC2_VMS_ARN }}

.github/workflows/pg-clients.yml

@@ -53,8 +53,8 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init --user root
     services:
       clickhouse:
@@ -88,7 +88,12 @@ jobs:
         ports:
           - 8083:8083
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Download Neon artifact
         uses: ./.github/actions/download
@@ -138,7 +143,7 @@ jobs:
 
       - name: Post to a Slack channel
         if: github.event.schedule && failure()
-        uses: slackapi/slack-github-action@v1
+        uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
         with:
           channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
           slack-message: |
@@ -153,12 +158,17 @@ jobs:
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
       options: --init --user root
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
     - name: Download Neon artifact
       uses: ./.github/actions/download
@@ -206,7 +216,7 @@ jobs:
 
     - name: Post to a Slack channel
       if: github.event.schedule && failure()
-      uses: slackapi/slack-github-action@v1
+      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
       with:
         channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
         slack-message: |

.github/workflows/pin-build-tools-image.yml

@@ -40,14 +40,19 @@ jobs:
       skip: ${{ steps.check-manifests.outputs.skip }}
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
       - name: Check if we really need to pin the image
         id: check-manifests
         env:
           FROM_TAG: ${{ inputs.from-tag }}
           TO_TAG: pinned
         run: |
-          docker manifest inspect "docker.io/neondatabase/build-tools:${FROM_TAG}" > "${FROM_TAG}.json"
-          docker manifest inspect "docker.io/neondatabase/build-tools:${TO_TAG}"   > "${TO_TAG}.json"
+          docker manifest inspect "ghcr.io/neondatabase/build-tools:${FROM_TAG}" > "${FROM_TAG}.json"
+          docker manifest inspect "ghcr.io/neondatabase/build-tools:${TO_TAG}"   > "${TO_TAG}.json"
 
           if diff "${FROM_TAG}.json" "${TO_TAG}.json"; then
             skip=true
@@ -71,13 +76,13 @@ jobs:
     with:
       image-map: |
         {
-          "docker.io/neondatabase/build-tools:${{ inputs.from-tag }}-bullseye": [
+          "ghcr.io/neondatabase/build-tools:${{ inputs.from-tag }}-bullseye": [
             "docker.io/neondatabase/build-tools:pinned-bullseye",
             "ghcr.io/neondatabase/build-tools:pinned-bullseye",
             "${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/build-tools:pinned-bullseye",
             "${{ vars.AZURE_DEV_REGISTRY_NAME }}.azurecr.io/neondatabase/build-tools:pinned-bullseye"
           ],
-          "docker.io/neondatabase/build-tools:${{ inputs.from-tag }}-bookworm": [
+          "ghcr.io/neondatabase/build-tools:${{ inputs.from-tag }}-bookworm": [
             "docker.io/neondatabase/build-tools:pinned-bookworm",
             "docker.io/neondatabase/build-tools:pinned",
             "ghcr.io/neondatabase/build-tools:pinned-bookworm",

.github/workflows/pre-merge-checks.yml

@@ -19,13 +19,20 @@ permissions: {}
 jobs:
   meta:
     runs-on: ubuntu-22.04
+    permissions:
+      contents: read
     outputs:
       python-changed: ${{ steps.python-src.outputs.any_changed }}
       rust-changed: ${{ steps.rust-src.outputs.any_changed }}
       branch: ${{ steps.group-metadata.outputs.branch }}
       pr-number: ${{ steps.group-metadata.outputs.pr-number }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 # v45.0.1
         id: python-src
@@ -72,6 +79,9 @@ jobs:
       || needs.meta.outputs.python-changed == 'true'
       || needs.meta.outputs.rust-changed == 'true'
     needs: [ meta ]
+    permissions:
+      contents: read
+      packages: write
     uses: ./.github/workflows/build-build-tools-image.yml
     with:
       # Build only one combination to save time
@@ -82,6 +92,9 @@ jobs:
   check-codestyle-python:
     if: needs.meta.outputs.python-changed == 'true'
     needs: [ meta, build-build-tools-image ]
+    permissions:
+      contents: read
+      packages: read
     uses: ./.github/workflows/_check-codestyle-python.yml
     with:
       # `-bookworm-x64` suffix should match the combination in `build-build-tools-image`
@@ -91,6 +104,9 @@ jobs:
   check-codestyle-rust:
     if: needs.meta.outputs.rust-changed == 'true'
     needs: [ meta, build-build-tools-image ]
+    permissions:
+      contents: read
+      packages: read
     uses: ./.github/workflows/_check-codestyle-rust.yml
     with:
       # `-bookworm-x64` suffix should match the combination in `build-build-tools-image`
@@ -114,8 +130,13 @@ jobs:
       - check-codestyle-rust
     runs-on: ubuntu-22.04
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Create fake `neon-cloud-e2e` check
-        uses: actions/github-script@v7
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           # Retry script for 5XX server errors: https://github.com/actions/github-script#retries
           retries: 5
@@ -148,7 +169,7 @@ jobs:
           ${{
             always()
             && github.event_name == 'merge_group'
-            && contains(fromJson('["release", "release-proxy", "release-compute"]'), needs.meta.outputs.branch)
+            && contains(fromJSON('["release", "release-proxy", "release-compute"]'), needs.meta.outputs.branch)
           }}
         env:
           GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}

.github/workflows/regenerate-pg-setting.yml

@@ -23,8 +23,13 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
       - name: Add comment
-        uses: thollander/actions-comment-pull-request@v3
+        uses: thollander/actions-comment-pull-request@65f9e5c9a1f2cd378bd74b2e057c9736982a8e74 # v3
         with:
           comment-tag: ${{ github.job }}
           pr-number: ${{ github.event.number }}

.github/workflows/release-notify.yml

@@ -22,7 +22,12 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: neondatabase/dev-actions/release-pr-notify@main
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - uses: neondatabase/dev-actions/release-pr-notify@483a843f2a8bcfbdc4c69d27630528a3ddc4e14b # main
         with:
           slack-token: ${{ secrets.SLACK_BOT_TOKEN }}
           slack-channel-id: ${{ vars.SLACK_UPCOMING_RELEASE_CHANNEL_ID || 'C05QQ9J1BRC' }} # if not set, then `#test-release-notifications`

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Create Release Branch
 on:
   schedule:
     # It should be kept in sync with if-condition in jobs
-    - cron: '0 6 * * THU' # Proxy release
+    - cron: '0 6 * * TUE' # Proxy release
     - cron: '0 6 * * FRI' # Storage release
     - cron: '0 7 * * FRI' # Compute release
   workflow_dispatch:
@@ -43,7 +43,7 @@ jobs:
       ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }}
 
   create-proxy-release-branch:
-    if: ${{ github.event.schedule == '0 6 * * THU' || inputs.create-proxy-release-branch }}
+    if: ${{ github.event.schedule == '0 6 * * TUE' || inputs.create-proxy-release-branch }}
 
     permissions:
       contents: write

.github/workflows/report-workflow-stats-batch.yml

@@ -6,6 +6,9 @@ on:
     - cron: '25 0 * * *'
     - cron: '25 1 * * 6'
 
+permissions:
+  contents: read
+
 jobs:
   gh-workflow-stats-batch-2h:
     name: GitHub Workflow Stats Batch 2 hours
@@ -14,8 +17,13 @@ jobs:
     permissions:
       actions: read
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
     - name: Export Workflow Run for the past 2 hours
-      uses: neondatabase/gh-workflow-stats-action@v0.2.1
+      uses: neondatabase/gh-workflow-stats-action@4c998b25ab5cc6588b52a610b749531f6a566b6b # v0.2.1
       with:
         db_uri: ${{ secrets.GH_REPORT_STATS_DB_RW_CONNSTR }}
         db_table: "gh_workflow_stats_neon"
@@ -29,8 +37,13 @@ jobs:
     permissions:
       actions: read
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
     - name: Export Workflow Run for the past 48 hours
-      uses: neondatabase/gh-workflow-stats-action@v0.2.1
+      uses: neondatabase/gh-workflow-stats-action@4c998b25ab5cc6588b52a610b749531f6a566b6b # v0.2.1
       with:
         db_uri: ${{ secrets.GH_REPORT_STATS_DB_RW_CONNSTR }}
         db_table: "gh_workflow_stats_neon"
@@ -44,8 +57,13 @@ jobs:
     permissions:
       actions: read
     steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+      with:
+        egress-policy: audit
+
     - name: Export Workflow Run for the past 30 days
-      uses: neondatabase/gh-workflow-stats-action@v0.2.1
+      uses: neondatabase/gh-workflow-stats-action@4c998b25ab5cc6588b52a610b749531f6a566b6b # v0.2.1
       with:
         db_uri: ${{ secrets.GH_REPORT_STATS_DB_RW_CONNSTR }}
         db_table: "gh_workflow_stats_neon"

.github/workflows/trigger-e2e-tests.yml

@@ -31,6 +31,11 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
       - name: Cancel previous e2e-tests runs for this PR
         env:
           GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
@@ -63,6 +68,11 @@ jobs:
           || needs.meta.outputs.build-tag
         }}
     steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+
       - name: Wait for `push-{neon,compute}-image-dev` job to finish
         # It's important to have a timeout here, the script in the step can run infinitely
         timeout-minutes: 60

Cargo.lock

@@ -167,45 +167,6 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
-[[package]]
-name = "asn1-rs"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048"
-dependencies = [
- "asn1-rs-derive",
- "asn1-rs-impl",
- "displaydoc",
- "nom",
- "num-traits",
- "rusticata-macros",
- "thiserror 1.0.69",
- "time",
-]
-
-[[package]]
-name = "asn1-rs-derive"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
- "synstructure",
-]
-
-[[package]]
-name = "asn1-rs-impl"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
 [[package]]
 name = "assert-json-diff"
 version = "2.0.2"
@@ -1813,20 +1774,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "der-parser"
-version = "9.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"
-dependencies = [
- "asn1-rs",
- "displaydoc",
- "nom",
- "num-bigint",
- "num-traits",
- "rusticata-macros",
-]
-
 [[package]]
 name = "der_derive"
 version = "0.7.3"
@@ -4044,15 +3991,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "oid-registry"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9"
-dependencies = [
- "asn1-rs",
-]
-
 [[package]]
 name = "once_cell"
 version = "1.20.2"
@@ -4332,6 +4270,7 @@ dependencies = [
  "scopeguard",
  "send-future",
  "serde",
+ "serde_ignored",
  "serde_json",
  "serde_path_to_error",
  "serde_with",
@@ -4754,7 +4693,7 @@ dependencies = [
 [[package]]
 name = "postgres-protocol"
 version = "0.6.6"
-source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#1f21e7959a96a34dcfbfce1b14b73286cdadffe9"
+source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#394851e467755562b4173ff68f9eb0e7f737be13"
 dependencies = [
  "base64 0.22.1",
  "byteorder",
@@ -4788,7 +4727,7 @@ dependencies = [
 [[package]]
 name = "postgres-types"
 version = "0.2.6"
-source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#1f21e7959a96a34dcfbfce1b14b73286cdadffe9"
+source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#394851e467755562b4173ff68f9eb0e7f737be13"
 dependencies = [
  "bytes",
  "chrono",
@@ -5227,7 +5166,7 @@ dependencies = [
  "uuid",
  "walkdir",
  "workspace_hack",
- "x509-parser",
+ "x509-cert",
  "zerocopy",
 ]
 
@@ -5848,15 +5787,6 @@ dependencies = [
  "semver",
 ]
 
-[[package]]
-name = "rusticata-macros"
-version = "4.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
-dependencies = [
- "nom",
-]
-
 [[package]]
 name = "rustix"
 version = "0.38.41"
@@ -6358,6 +6288,15 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "serde_ignored"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "566da67d80e92e009728b3731ff0e5360cb181432b8ca73ea30bb1d170700d76"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_json"
 version = "1.0.125"
@@ -7241,7 +7180,7 @@ dependencies = [
 [[package]]
 name = "tokio-postgres"
 version = "0.7.10"
-source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#1f21e7959a96a34dcfbfce1b14b73286cdadffe9"
+source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#394851e467755562b4173ff68f9eb0e7f737be13"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -8440,7 +8379,6 @@ dependencies = [
  "der 0.7.8",
  "deranged",
  "digest",
- "displaydoc",
  "ecdsa 0.16.9",
  "either",
  "elliptic-curve 0.13.8",
@@ -8568,23 +8506,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "x509-parser"
-version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69"
-dependencies = [
- "asn1-rs",
- "data-encoding",
- "der-parser",
- "lazy_static",
- "nom",
- "oid-registry",
- "rusticata-macros",
- "thiserror 1.0.69",
- "time",
-]
-
 [[package]]
 name = "xattr"
 version = "1.0.0"

Cargo.toml

@@ -164,6 +164,7 @@ sd-notify = "0.4.1"
 send-future = "0.1.0"
 sentry = { version = "0.32", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
 serde = { version = "1.0", features = ["derive"] }
+serde_ignored = "0.1.11"
 serde_json = "1"
 serde_path_to_error = "0.1"
 serde_with = { version = "2.0", features = [ "base64" ] }
@@ -215,10 +216,10 @@ urlencoding = "2.1"
 uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
 walkdir = "2.3.2"
 rustls-native-certs = "0.8"
-x509-parser = "0.16"
 whoami = "1.5.1"
 zerocopy = { version = "0.7", features = ["derive"] }
 json-structural-diff = { version = "0.2.0" }
+x509-cert = { version = "0.2.5" }
 
 ## TODO replace this with tracing
 env_logger = "0.11"

Dockerfile

@@ -2,7 +2,7 @@
 ### The image itself is mainly used as a container for the binaries and for starting e2e tests with custom parameters.
 ### By default, the binaries inside the image have some mock parameters and can start, but are not intended to be used
 ### inside this image in the real deployments.
-ARG REPOSITORY=neondatabase
+ARG REPOSITORY=ghcr.io/neondatabase
 ARG IMAGE=build-tools
 ARG TAG=pinned
 ARG DEFAULT_PG_VERSION=17

compute/compute-node.Dockerfile

@@ -77,7 +77,7 @@
 # build_and_test.yml github workflow for how that's done.
 
 ARG PG_VERSION
-ARG REPOSITORY=neondatabase
+ARG REPOSITORY=ghcr.io/neondatabase
 ARG IMAGE=build-tools
 ARG TAG=pinned
 ARG BUILD_TAG

compute/patches/pg_anon.patch

@@ -0,0 +1,265 @@
+commit 00aa659afc9c7336ab81036edec3017168aabf40
+Author: Heikki Linnakangas <heikki@neon.tech>
+Date:   Tue Nov 12 16:59:19 2024 +0200
+
+    Temporarily disable test that depends on timezone
+
+diff --git a/tests/expected/generalization.out b/tests/expected/generalization.out
+index 23ef5fa..9e60deb 100644
+--- a/ext-src/pg_anon-src/tests/expected/generalization.out
++++ b/ext-src/pg_anon-src/tests/expected/generalization.out
+@@ -284,12 +284,9 @@ SELECT anon.generalize_tstzrange('19041107','century');
+  ["Tue Jan 01 00:00:00 1901 PST","Mon Jan 01 00:00:00 2001 PST")
+ (1 row)
+ 
+-SELECT anon.generalize_tstzrange('19041107','millennium');
+-                      generalize_tstzrange                       
+------------------------------------------------------------------
+- ["Thu Jan 01 00:00:00 1001 PST","Mon Jan 01 00:00:00 2001 PST")
+-(1 row)
+-
++-- temporarily disabled, see:
++-- https://gitlab.com/dalibo/postgresql_anonymizer/-/commit/199f0a392b37c59d92ae441fb8f037e094a11a52#note_2148017485
++--SELECT anon.generalize_tstzrange('19041107','millennium');
+ -- generalize_daterange
+ SELECT anon.generalize_daterange('19041107');
+   generalize_daterange   
+diff --git a/tests/sql/generalization.sql b/tests/sql/generalization.sql
+index b868344..b4fc977 100644
+--- a/ext-src/pg_anon-src/tests/sql/generalization.sql
++++ b/ext-src/pg_anon-src/tests/sql/generalization.sql
+@@ -61,7 +61,9 @@ SELECT anon.generalize_tstzrange('19041107','month');
+ SELECT anon.generalize_tstzrange('19041107','year');
+ SELECT anon.generalize_tstzrange('19041107','decade');
+ SELECT anon.generalize_tstzrange('19041107','century');
+-SELECT anon.generalize_tstzrange('19041107','millennium');
++-- temporarily disabled, see:
++-- https://gitlab.com/dalibo/postgresql_anonymizer/-/commit/199f0a392b37c59d92ae441fb8f037e094a11a52#note_2148017485
++--SELECT anon.generalize_tstzrange('19041107','millennium');
+ 
+ -- generalize_daterange
+ SELECT anon.generalize_daterange('19041107');
+
+commit 7dd414ee75f2875cffb1d6ba474df1f135a6fc6f
+Author: Alexey Masterov <alexeymasterov@neon.tech>
+Date:   Fri May 31 06:34:26 2024 +0000
+
+    These alternative expected files were added to consider the neon features
+
+diff --git a/ext-src/pg_anon-src/tests/expected/permissions_masked_role_1.out b/ext-src/pg_anon-src/tests/expected/permissions_masked_role_1.out
+new file mode 100644
+index 0000000..2539cfd
+--- /dev/null
++++ b/ext-src/pg_anon-src/tests/expected/permissions_masked_role_1.out
+@@ -0,0 +1,101 @@
++BEGIN;
++CREATE EXTENSION anon CASCADE;
++NOTICE:  installing required extension "pgcrypto"
++SELECT anon.init();
++ init 
++------
++ t
++(1 row)
++
++CREATE ROLE mallory_the_masked_user;
++SECURITY LABEL FOR anon ON ROLE mallory_the_masked_user IS 'MASKED';
++CREATE TABLE t1(i INT);
++ALTER TABLE t1 ADD COLUMN t TEXT;
++SECURITY LABEL FOR anon ON COLUMN t1.t
++IS 'MASKED WITH VALUE NULL';
++INSERT INTO t1 VALUES (1,'test');
++--
++-- We're checking the owner's permissions
++--
++-- see
++-- https://postgresql-anonymizer.readthedocs.io/en/latest/SECURITY/#permissions
++--
++SET ROLE mallory_the_masked_user;
++SELECT anon.pseudo_first_name(0) IS NOT NULL;
++ ?column? 
++----------
++ t
++(1 row)
++
++-- SHOULD FAIL
++DO $$
++BEGIN
++  PERFORM anon.init();
++  EXCEPTION WHEN insufficient_privilege
++  THEN RAISE NOTICE 'insufficient_privilege';
++END$$;
++NOTICE:  insufficient_privilege
++-- SHOULD FAIL
++DO $$
++BEGIN
++  PERFORM anon.anonymize_table('t1');
++  EXCEPTION WHEN insufficient_privilege
++  THEN RAISE NOTICE 'insufficient_privilege';
++END$$;
++NOTICE:  insufficient_privilege
++-- SHOULD FAIL
++SAVEPOINT fail_start_engine;
++SELECT anon.start_dynamic_masking();
++ERROR:  Only supersusers can start the dynamic masking engine.
++CONTEXT:  PL/pgSQL function anon.start_dynamic_masking(boolean) line 18 at RAISE
++ROLLBACK TO fail_start_engine;
++RESET ROLE;
++SELECT anon.start_dynamic_masking();
++ start_dynamic_masking 
++-----------------------
++ t
++(1 row)
++
++SET ROLE mallory_the_masked_user;
++SELECT * FROM mask.t1;
++ i | t 
++---+---
++ 1 | 
++(1 row)
++
++-- SHOULD FAIL
++DO $$
++BEGIN
++  SELECT * FROM public.t1;
++  EXCEPTION WHEN insufficient_privilege
++  THEN RAISE NOTICE 'insufficient_privilege';
++END$$;
++NOTICE:  insufficient_privilege
++-- SHOULD FAIL
++SAVEPOINT fail_stop_engine;
++SELECT anon.stop_dynamic_masking();
++ERROR:  Only supersusers can stop the dynamic masking engine.
++CONTEXT:  PL/pgSQL function anon.stop_dynamic_masking() line 18 at RAISE
++ROLLBACK TO fail_stop_engine;
++RESET ROLE;
++SELECT anon.stop_dynamic_masking();
++NOTICE:  The previous priviledges of 'mallory_the_masked_user' are not restored. You need to grant them manually.
++ stop_dynamic_masking 
++----------------------
++ t
++(1 row)
++
++SET ROLE mallory_the_masked_user;
++SELECT COUNT(*)=1 FROM anon.pg_masking_rules;
++ ?column? 
++----------
++ t
++(1 row)
++
++-- SHOULD FAIL
++SAVEPOINT fail_seclabel_on_role;
++SECURITY LABEL FOR anon ON ROLE mallory_the_masked_user IS NULL;
++ERROR:  permission denied
++DETAIL:  The current user must have the CREATEROLE attribute.
++ROLLBACK TO fail_seclabel_on_role;
++ROLLBACK;
+diff --git a/ext-src/pg_anon-src/tests/expected/permissions_owner_1.out b/ext-src/pg_anon-src/tests/expected/permissions_owner_1.out
+new file mode 100644
+index 0000000..8b090fe
+--- /dev/null
++++ b/ext-src/pg_anon-src/tests/expected/permissions_owner_1.out
+@@ -0,0 +1,104 @@
++BEGIN;
++CREATE EXTENSION anon CASCADE;
++NOTICE:  installing required extension "pgcrypto"
++SELECT anon.init();
++ init 
++------
++ t
++(1 row)
++
++CREATE ROLE oscar_the_owner;
++ALTER DATABASE :DBNAME OWNER TO oscar_the_owner;
++CREATE ROLE mallory_the_masked_user;
++SECURITY LABEL FOR anon ON ROLE mallory_the_masked_user IS 'MASKED';
++--
++-- We're checking the owner's permissions
++--
++-- see
++-- https://postgresql-anonymizer.readthedocs.io/en/latest/SECURITY/#permissions
++--
++SET ROLE oscar_the_owner;
++SELECT anon.pseudo_first_name(0) IS NOT NULL;
++ ?column? 
++----------
++ t
++(1 row)
++
++-- SHOULD FAIL
++DO $$
++BEGIN
++  PERFORM anon.init();
++  EXCEPTION WHEN insufficient_privilege
++  THEN RAISE NOTICE 'insufficient_privilege';
++END$$;
++NOTICE:  insufficient_privilege
++CREATE TABLE t1(i INT);
++ALTER TABLE t1 ADD COLUMN t TEXT;
++SECURITY LABEL FOR anon ON COLUMN t1.t
++IS 'MASKED WITH VALUE NULL';
++INSERT INTO t1 VALUES (1,'test');
++SELECT anon.anonymize_table('t1');
++ anonymize_table 
++-----------------
++ t
++(1 row)
++
++SELECT * FROM t1;
++ i | t 
++---+---
++ 1 | 
++(1 row)
++
++UPDATE t1 SET t='test' WHERE i=1;
++-- SHOULD FAIL
++SAVEPOINT fail_start_engine;
++SELECT anon.start_dynamic_masking();
++ start_dynamic_masking 
++-----------------------
++ t
++(1 row)
++
++ROLLBACK TO fail_start_engine;
++RESET ROLE;
++SELECT anon.start_dynamic_masking();
++ start_dynamic_masking 
++-----------------------
++ t
++(1 row)
++
++SET ROLE oscar_the_owner;
++SELECT * FROM t1;
++ i |  t   
++---+------
++ 1 | test
++(1 row)
++
++--SELECT * FROM mask.t1;
++-- SHOULD FAIL
++SAVEPOINT fail_stop_engine;
++SELECT anon.stop_dynamic_masking();
++ERROR:  permission denied for schema mask
++CONTEXT:  SQL statement "DROP VIEW mask.t1;"
++PL/pgSQL function anon.mask_drop_view(oid) line 3 at EXECUTE
++SQL statement "SELECT anon.mask_drop_view(oid)
++  FROM pg_catalog.pg_class
++  WHERE relnamespace=quote_ident(pg_catalog.current_setting('anon.sourceschema'))::REGNAMESPACE
++  AND relkind IN ('r','p','f')"
++PL/pgSQL function anon.stop_dynamic_masking() line 22 at PERFORM
++ROLLBACK TO fail_stop_engine;
++RESET ROLE;
++SELECT anon.stop_dynamic_masking();
++NOTICE:  The previous priviledges of 'mallory_the_masked_user' are not restored. You need to grant them manually.
++ stop_dynamic_masking 
++----------------------
++ t
++(1 row)
++
++SET ROLE oscar_the_owner;
++-- SHOULD FAIL
++SAVEPOINT fail_seclabel_on_role;
++SECURITY LABEL FOR anon ON ROLE mallory_the_masked_user IS NULL;
++ERROR:  permission denied
++DETAIL:  The current user must have the CREATEROLE attribute.
++ROLLBACK TO fail_seclabel_on_role;
++ROLLBACK;

compute_tools/Cargo.toml

@@ -61,7 +61,7 @@ thiserror.workspace = true
 url.workspace = true
 uuid.workspace = true
 walkdir.workspace = true
-x509-cert = { version = "0.2.5" }
+x509-cert.workspace = true
 
 postgres_initdb.workspace = true
 compute_api.workspace = true

compute_tools/src/config.rs

@@ -158,53 +158,89 @@ pub fn write_postgres_conf(
         writeln!(file, "# Managed by compute_ctl: end")?;
     }
 
-    // If audit logging is enabled, configure pgaudit.
+    // If base audit logging is enabled, configure it.
+    // In this setup, the audit log will be written to the standard postgresql log.
+    //
+    // If compliance audit logging is enabled, configure pgaudit.
     //
     // Note, that this is called after the settings from spec are written.
     // This way we always override the settings from the spec
     // and don't allow the user or the control plane admin to change them.
-    if let ComputeAudit::Hipaa = spec.audit_log_level {
-        writeln!(file, "# Managed by compute_ctl audit settings: begin")?;
-        // This log level is very verbose
-        // but this is necessary for HIPAA compliance.
-        // Exclude 'misc' category, because it doesn't contain anythig relevant.
-        writeln!(file, "pgaudit.log='all, -misc'")?;
-        writeln!(file, "pgaudit.log_parameter=on")?;
-        // Disable logging of catalog queries
-        // The catalog doesn't contain sensitive data, so we don't need to audit it.
-        writeln!(file, "pgaudit.log_catalog=off")?;
-        // Set log rotation to 5 minutes
-        // TODO: tune this after performance testing
-        writeln!(file, "pgaudit.log_rotation_age=5")?;
+    match spec.audit_log_level {
+        ComputeAudit::Disabled => {}
+        ComputeAudit::Log => {
+            writeln!(file, "# Managed by compute_ctl base audit settings: start")?;
+            writeln!(file, "pgaudit.log='ddl,role'")?;
+            // Disable logging of catalog queries to reduce the noise
+            writeln!(file, "pgaudit.log_catalog=off")?;
 
-        // Add audit shared_preload_libraries, if they are not present.
-        //
-        // The caller who sets the flag is responsible for ensuring that the necessary
-        // shared_preload_libraries are present in the compute image,
-        // otherwise the compute start will fail.
-        if let Some(libs) = spec.cluster.settings.find("shared_preload_libraries") {
-            let mut extra_shared_preload_libraries = String::new();
-            if !libs.contains("pgaudit") {
-                extra_shared_preload_libraries.push_str(",pgaudit");
-            }
-            if !libs.contains("pgauditlogtofile") {
-                extra_shared_preload_libraries.push_str(",pgauditlogtofile");
+            if let Some(libs) = spec.cluster.settings.find("shared_preload_libraries") {
+                let mut extra_shared_preload_libraries = String::new();
+                if !libs.contains("pgaudit") {
+                    extra_shared_preload_libraries.push_str(",pgaudit");
+                }
+                writeln!(
+                    file,
+                    "shared_preload_libraries='{}{}'",
+                    libs, extra_shared_preload_libraries
+                )?;
+            } else {
+                // Typically, this should be unreacheable,
+                // because we always set at least some shared_preload_libraries in the spec
+                // but let's handle it explicitly anyway.
+                writeln!(file, "shared_preload_libraries='neon,pgaudit'")?;
             }
+            writeln!(file, "# Managed by compute_ctl base audit settings: end")?;
+        }
+        ComputeAudit::Hipaa => {
             writeln!(
                 file,
-                "shared_preload_libraries='{}{}'",
-                libs, extra_shared_preload_libraries
+                "# Managed by compute_ctl compliance audit settings: begin"
             )?;
-        } else {
-            // Typically, this should be unreacheable,
-            // because we always set at least some shared_preload_libraries in the spec
-            // but let's handle it explicitly anyway.
+            // This log level is very verbose
+            // but this is necessary for HIPAA compliance.
+            // Exclude 'misc' category, because it doesn't contain anythig relevant.
+            writeln!(file, "pgaudit.log='all, -misc'")?;
+            writeln!(file, "pgaudit.log_parameter=on")?;
+            // Disable logging of catalog queries
+            // The catalog doesn't contain sensitive data, so we don't need to audit it.
+            writeln!(file, "pgaudit.log_catalog=off")?;
+            // Set log rotation to 5 minutes
+            // TODO: tune this after performance testing
+            writeln!(file, "pgaudit.log_rotation_age=5")?;
+
+            // Add audit shared_preload_libraries, if they are not present.
+            //
+            // The caller who sets the flag is responsible for ensuring that the necessary
+            // shared_preload_libraries are present in the compute image,
+            // otherwise the compute start will fail.
+            if let Some(libs) = spec.cluster.settings.find("shared_preload_libraries") {
+                let mut extra_shared_preload_libraries = String::new();
+                if !libs.contains("pgaudit") {
+                    extra_shared_preload_libraries.push_str(",pgaudit");
+                }
+                if !libs.contains("pgauditlogtofile") {
+                    extra_shared_preload_libraries.push_str(",pgauditlogtofile");
+                }
+                writeln!(
+                    file,
+                    "shared_preload_libraries='{}{}'",
+                    libs, extra_shared_preload_libraries
+                )?;
+            } else {
+                // Typically, this should be unreacheable,
+                // because we always set at least some shared_preload_libraries in the spec
+                // but let's handle it explicitly anyway.
+                writeln!(
+                    file,
+                    "shared_preload_libraries='neon,pgaudit,pgauditlogtofile'"
+                )?;
+            }
             writeln!(
                 file,
-                "shared_preload_libraries='neon,pgaudit,pgauditlogtofile'"
+                "# Managed by compute_ctl compliance audit settings: end"
             )?;
         }
-        writeln!(file, "# Managed by compute_ctl audit settings: end")?;
     }
 
     writeln!(file, "neon.extension_server_port={}", extension_server_port)?;

compute_tools/src/spec_apply.rs

@@ -286,7 +286,10 @@ impl ComputeNode {
                     phases.push(CreatePgauditlogtofileExtension);
                     phases.push(DisablePostgresDBPgAudit);
                 }
-                ComputeAudit::Log => { /* not implemented yet */ }
+                ComputeAudit::Log => {
+                    phases.push(CreatePgauditExtension);
+                    phases.push(DisablePostgresDBPgAudit);
+                }
                 ComputeAudit::Disabled => {}
             }
 

compute_tools/src/tls.rs

@@ -3,7 +3,6 @@ use std::{io::Write, os::unix::fs::OpenOptionsExt, path::Path, time::Duration};
 use anyhow::{Context, Result, bail};
 use compute_api::responses::TlsConfig;
 use ring::digest;
-use spki::ObjectIdentifier;
 use spki::der::{Decode, PemReader};
 use x509_cert::Certificate;
 
@@ -91,13 +90,13 @@ fn try_update_key_path_blocking(pg_data: &Path, tls_config: &TlsConfig) -> Resul
 }
 
 fn verify_key_cert(key: &str, cert: &str) -> Result<()> {
-    const ECDSA_WITH_SHA256: ObjectIdentifier = ObjectIdentifier::new_unwrap("1.2.840.10045.4.3.2");
+    use x509_cert::der::oid::db::rfc5912::ECDSA_WITH_SHA_256;
 
     let cert = Certificate::decode(&mut PemReader::new(cert.as_bytes()).context("pem reader")?)
         .context("decode cert")?;
 
     match cert.signature_algorithm.oid {
-        ECDSA_WITH_SHA256 => {
+        ECDSA_WITH_SHA_256 => {
             let key = p256::SecretKey::from_sec1_pem(key).context("parse key")?;
 
             let a = key.public_key().to_sec1_bytes();

docker-compose/compute_wrapper/Dockerfile

@@ -1,4 +1,4 @@
-ARG REPOSITORY=neondatabase
+ARG REPOSITORY=ghcr.io/neondatabase
 ARG COMPUTE_IMAGE=compute-node-v14
 ARG TAG=latest
 

docker-compose/compute_wrapper/shell/compute.sh

@@ -67,6 +67,14 @@ else
   fi
 fi
 
+if [[ ${PG_VERSION} -ge 17 ]]; then
+  ulid_extension=pgx_ulid
+else
+  ulid_extension=ulid
+fi
+echo "Adding pgx_ulid"
+shared_libraries=$(jq -r '.cluster.settings[] | select(.name=="shared_preload_libraries").value' ${SPEC_FILE})
+sed -i "s/${shared_libraries}/${shared_libraries},${ulid_extension}/" ${SPEC_FILE}
 echo "Overwrite tenant id and timeline id in spec file"
 sed -i "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE}
 sed -i "s/TIMELINE_ID/${timeline_id}/" ${SPEC_FILE}

docker-compose/docker-compose.yml

@@ -29,7 +29,7 @@ services:
 
   pageserver:
     restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
+    image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
     environment:
       - AWS_ACCESS_KEY_ID=minio
       - AWS_SECRET_ACCESS_KEY=password
@@ -45,7 +45,7 @@ services:
 
   safekeeper1:
     restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
+    image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454
       - SAFEKEEPER_ID=1
@@ -75,7 +75,7 @@ services:
 
   safekeeper2:
     restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
+    image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454
       - SAFEKEEPER_ID=2
@@ -105,7 +105,7 @@ services:
 
   safekeeper3:
     restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
+    image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
     environment:
       - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454
       - SAFEKEEPER_ID=3
@@ -135,7 +135,7 @@ services:
 
   storage_broker:
     restart: always
-    image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest}
+    image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest}
     ports:
       - 50051:50051
     command:
@@ -147,7 +147,7 @@ services:
     build:
       context: ./compute_wrapper/
       args:
-        - REPOSITORY=${REPOSITORY:-neondatabase}
+        - REPOSITORY=${REPOSITORY:-ghcr.io/neondatabase}
         - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-16}
         - TAG=${COMPUTE_TAG:-${TAG:-latest}}
         - http_proxy=${http_proxy:-}
@@ -186,7 +186,7 @@ services:
 
   neon-test-extensions:
     profiles: ["test-extensions"]
-    image: ${REPOSITORY:-neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-16}:${TEST_EXTENSIONS_TAG:-${TAG:-latest}}
+    image: ${REPOSITORY:-ghcr.io/neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-16}:${TEST_EXTENSIONS_TAG:-${TAG:-latest}}
     environment:
       - PGPASSWORD=cloud_admin
     entrypoint:

docker-compose/docker_compose_test.sh

@@ -69,7 +69,7 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do
         cat ../compute/patches/contrib_pg${pg_version}.patch | docker exec -i $TEST_CONTAINER_NAME bash -c "(cd /postgres && patch -p1)"
         # We are running tests now
         rm -f testout.txt testout_contrib.txt
-        docker exec -e USE_PGXS=1 -e SKIP=timescaledb-src,rdkit-src,postgis-src,pgx_ulid-src,pg_tiktoken-src,pg_jsonschema-src,kq_imcx-src,wal2json_2_5-src \
+        docker exec -e USE_PGXS=1 -e SKIP=timescaledb-src,rdkit-src,postgis-src,pg_jsonschema-src,kq_imcx-src,wal2json_2_5-src,rag_jina_reranker_v1_tiny_en-src,rag_bge_small_en_v15-src \
         $TEST_CONTAINER_NAME /run-tests.sh /ext-src | tee testout.txt && EXT_SUCCESS=1 || EXT_SUCCESS=0
         docker exec -e SKIP=start-scripts,postgres_fdw,ltree_plpython,jsonb_plpython,jsonb_plperl,hstore_plpython,hstore_plperl,dblink,bool_plperl \
         $TEST_CONTAINER_NAME /run-tests.sh /postgres/contrib | tee testout_contrib.txt && CONTRIB_SUCCESS=1 || CONTRIB_SUCCESS=0

docker-compose/ext-src/pg_tiktoken-src/Makefile

@@ -0,0 +1,8 @@
+PG_CONFIG ?= pg_config
+PG_REGRESS = $(shell dirname $$($(PG_CONFIG) --pgxs))/../../src/test/regress/pg_regress
+REGRESS = pg_tiktoken
+
+installcheck: regression-test
+
+regression-test:
+	$(PG_REGRESS) --inputdir=. --outputdir=. --dbname=contrib_regression $(REGRESS)

docker-compose/ext-src/pg_tiktoken-src/expected/pg_tiktoken.out

@@ -0,0 +1,53 @@
+-- Load the extension
+CREATE EXTENSION IF NOT EXISTS pg_tiktoken;
+-- Test encoding function
+SELECT tiktoken_encode('cl100k_base', 'Hello world!');
+ tiktoken_encode 
+-----------------
+ {9906,1917,0}
+(1 row)
+
+-- Test token count function
+SELECT tiktoken_count('cl100k_base', 'Hello world!');
+ tiktoken_count 
+----------------
+              3
+(1 row)
+
+-- Test encoding function with a different model
+SELECT tiktoken_encode('r50k_base', 'PostgreSQL is amazing!');
+     tiktoken_encode     
+-------------------------
+ {6307,47701,318,4998,0}
+(1 row)
+
+-- Test token count function with the same model
+SELECT tiktoken_count('r50k_base', 'PostgreSQL is amazing!');
+ tiktoken_count 
+----------------
+              5
+(1 row)
+
+-- Edge cases: Empty string
+SELECT tiktoken_encode('cl100k_base', '');
+ tiktoken_encode 
+-----------------
+ {}
+(1 row)
+
+SELECT tiktoken_count('cl100k_base', '');
+ tiktoken_count 
+----------------
+              0
+(1 row)
+
+-- Edge cases: Long text
+SELECT tiktoken_count('cl100k_base', repeat('word ', 100));
+ tiktoken_count 
+----------------
+            101
+(1 row)
+
+-- Edge case: Invalid encoding
+SELECT tiktoken_encode('invalid_model', 'Test') AS should_fail;
+ERROR:  'invalid_model': unknown model or encoder

docker-compose/ext-src/pg_tiktoken-src/sql/pg_tiktoken.sql

@@ -0,0 +1,24 @@
+-- Load the extension
+CREATE EXTENSION IF NOT EXISTS pg_tiktoken;
+
+-- Test encoding function
+SELECT tiktoken_encode('cl100k_base', 'Hello world!');
+
+-- Test token count function
+SELECT tiktoken_count('cl100k_base', 'Hello world!');
+
+-- Test encoding function with a different model
+SELECT tiktoken_encode('r50k_base', 'PostgreSQL is amazing!');
+
+-- Test token count function with the same model
+SELECT tiktoken_count('r50k_base', 'PostgreSQL is amazing!');
+
+-- Edge cases: Empty string
+SELECT tiktoken_encode('cl100k_base', '');
+SELECT tiktoken_count('cl100k_base', '');
+
+-- Edge cases: Long text
+SELECT tiktoken_count('cl100k_base', repeat('word ', 100));
+
+-- Edge case: Invalid encoding
+SELECT tiktoken_encode('invalid_model', 'Test') AS should_fail;

docker-compose/ext-src/pgrag-src/Makefile

@@ -0,0 +1,10 @@
+EXTENSION = rag
+MODULE_big = rag
+OBJS = $(patsubst %.rs,%.o,$(wildcard src/*.rs))
+
+REGRESS = basic_functions text_processing api_keys chunking_functions document_processing embedding_api_functions voyageai_functions
+REGRESS_OPTS = --load-extension=vector --load-extension=rag
+
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)

docker-compose/ext-src/pgrag-src/expected/api_keys.out

@@ -0,0 +1,49 @@
+-- API key function tests
+SELECT rag.anthropic_set_api_key('test_key');
+ anthropic_set_api_key 
+-----------------------
+ 
+(1 row)
+
+SELECT rag.anthropic_get_api_key();
+ anthropic_get_api_key 
+-----------------------
+ test_key
+(1 row)
+
+SELECT rag.openai_set_api_key('test_key');
+ openai_set_api_key 
+--------------------
+ 
+(1 row)
+
+SELECT rag.openai_get_api_key();
+ openai_get_api_key 
+--------------------
+ test_key
+(1 row)
+
+SELECT rag.fireworks_set_api_key('test_key');
+ fireworks_set_api_key 
+-----------------------
+ 
+(1 row)
+
+SELECT rag.fireworks_get_api_key();
+ fireworks_get_api_key 
+-----------------------
+ test_key
+(1 row)
+
+SELECT rag.voyageai_set_api_key('test_key');
+ voyageai_set_api_key 
+----------------------
+ 
+(1 row)
+
+SELECT rag.voyageai_get_api_key();
+ voyageai_get_api_key 
+----------------------
+ test_key
+(1 row)
+

docker-compose/ext-src/pgrag-src/expected/basic_functions.out

@@ -0,0 +1,13 @@
+-- Basic function tests
+SELECT rag.markdown_from_html('<p>Hello</p>');
+ markdown_from_html 
+--------------------
+ Hello
+(1 row)
+
+SELECT array_length(rag.chunks_by_character_count('the cat sat on the mat', 10, 5), 1);
+ array_length 
+--------------
+            3
+(1 row)
+

docker-compose/ext-src/pgrag-src/expected/chunking_functions.out

@@ -0,0 +1,31 @@
+-- Chunking function tests
+SELECT rag.chunks_by_character_count('the cat sat on the mat', 10, 5);
+       chunks_by_character_count       
+---------------------------------------
+ {"the cat","cat sat on","on the mat"}
+(1 row)
+
+SELECT rag.chunks_by_character_count('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', 20, 10);
+                                                                                     chunks_by_character_count                                                                                      
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"Lorem ipsum dolor","dolor sit amet,","amet, consectetur","adipiscing elit.","Sed do eiusmod","do eiusmod tempor","tempor incididunt ut","ut labore et dolore","et dolore magna","magna aliqua."}
+(1 row)
+
+SELECT (rag.chunks_by_character_count('the cat', 10, 0))[1];
+ chunks_by_character_count 
+---------------------------
+ the cat
+(1 row)
+
+SELECT rag.chunks_by_character_count('', 10, 5);
+ chunks_by_character_count 
+---------------------------
+ {}
+(1 row)
+
+SELECT rag.chunks_by_character_count('a b c d e f g h i j k l m n o p', 5, 2);
+                    chunks_by_character_count                    
+-----------------------------------------------------------------
+ {"a b c","c d e","e f g","g h i","i j k","k l m","m n o","o p"}
+(1 row)
+

docker-compose/ext-src/pgrag-src/expected/document_processing.out

@@ -0,0 +1,56 @@
+-- HTML to Markdown conversion tests
+SELECT rag.markdown_from_html('<p>Hello</p>');
+ markdown_from_html 
+--------------------
+ Hello
+(1 row)
+
+SELECT rag.markdown_from_html('<p>Hello <i>world</i></p>');
+ markdown_from_html 
+--------------------
+ Hello _world_
+(1 row)
+
+SELECT rag.markdown_from_html('<h1>Title</h1><p>Paragraph</p>');
+ markdown_from_html 
+--------------------
+ # Title           +
+                   +
+ Paragraph
+(1 row)
+
+SELECT rag.markdown_from_html('<ul><li>Item 1</li><li>Item 2</li></ul>');
+ markdown_from_html 
+--------------------
+ *   Item 1        +
+ *   Item 2
+(1 row)
+
+SELECT rag.markdown_from_html('<a href="https://example.com">Link</a>');
+     markdown_from_html      
+-----------------------------
+ [Link](https://example.com)
+(1 row)
+
+-- Note: text_from_pdf and text_from_docx require binary input which is harder to test in regression tests
+-- We'll test that the functions exist and have the right signature
+SELECT 'text_from_pdf_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'text_from_pdf'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+      test_name       | result 
+----------------------+--------
+ text_from_pdf_exists | t
+(1 row)
+
+SELECT 'text_from_docx_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'text_from_docx'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+       test_name       | result 
+-----------------------+--------
+ text_from_docx_exists | t
+(1 row)
+

docker-compose/ext-src/pgrag-src/expected/embedding_api_functions.out

@@ -0,0 +1,103 @@
+-- Test embedding functions exist with correct signatures
+-- OpenAI embedding functions
+SELECT 'openai_text_embedding_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+          test_name           | result 
+------------------------------+--------
+ openai_text_embedding_exists | t
+(1 row)
+
+SELECT 'openai_text_embedding_3_small_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding_3_small'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+              test_name               | result 
+--------------------------------------+--------
+ openai_text_embedding_3_small_exists | t
+(1 row)
+
+SELECT 'openai_text_embedding_3_large_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding_3_large'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+              test_name               | result 
+--------------------------------------+--------
+ openai_text_embedding_3_large_exists | t
+(1 row)
+
+SELECT 'openai_text_embedding_ada_002_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding_ada_002'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+              test_name               | result 
+--------------------------------------+--------
+ openai_text_embedding_ada_002_exists | t
+(1 row)
+
+-- Fireworks embedding functions
+SELECT 'fireworks_nomic_embed_text_v1_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_nomic_embed_text_v1'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+              test_name               | result 
+--------------------------------------+--------
+ fireworks_nomic_embed_text_v1_exists | t
+(1 row)
+
+SELECT 'fireworks_nomic_embed_text_v15_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_nomic_embed_text_v15'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+               test_name               | result 
+---------------------------------------+--------
+ fireworks_nomic_embed_text_v15_exists | t
+(1 row)
+
+SELECT 'fireworks_text_embedding_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+            test_name            | result 
+---------------------------------+--------
+ fireworks_text_embedding_exists | t
+(1 row)
+
+SELECT 'fireworks_text_embedding_thenlper_gte_base_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding_thenlper_gte_base'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+                     test_name                     | result 
+---------------------------------------------------+--------
+ fireworks_text_embedding_thenlper_gte_base_exists | t
+(1 row)
+
+SELECT 'fireworks_text_embedding_thenlper_gte_large_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding_thenlper_gte_large'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+                     test_name                      | result 
+----------------------------------------------------+--------
+ fireworks_text_embedding_thenlper_gte_large_exists | t
+(1 row)
+
+SELECT 'fireworks_text_embedding_whereisai_uae_large_v1_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding_whereisai_uae_large_v1'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+                       test_name                        | result 
+--------------------------------------------------------+--------
+ fireworks_text_embedding_whereisai_uae_large_v1_exists | t
+(1 row)
+

docker-compose/ext-src/pgrag-src/expected/embedding_functions.out

@@ -0,0 +1,9 @@
+BEGIN
+CREATE EXTENSION IF NOT EXISTS vector;
+DROP EXTENSION IF EXISTS rag CASCADE;
+CREATE EXTENSION rag CASCADE;
+test_name|result
+openai_embedding_dimensions_test|t
+test_name|result
+fireworks_embedding_dimensions_test|t
+COMMIT

docker-compose/ext-src/pgrag-src/expected/text_processing.out

@@ -0,0 +1,13 @@
+-- Text processing function tests
+SELECT rag.markdown_from_html('<p>Hello <i>world</i></p>');
+ markdown_from_html 
+--------------------
+ Hello _world_
+(1 row)
+
+SELECT rag.chunks_by_character_count('the cat sat on the mat', 10, 5);
+       chunks_by_character_count       
+---------------------------------------
+ {"the cat","cat sat on","on the mat"}
+(1 row)
+

docker-compose/ext-src/pgrag-src/expected/voyageai_functions.out

@@ -0,0 +1,141 @@
+-- Test VoyageAI API key functions
+SELECT 'voyageai_api_key_test' AS test_name, 
+       (SELECT rag.voyageai_set_api_key('test_key') IS NULL) AS result;
+       test_name       | result 
+-----------------------+--------
+ voyageai_api_key_test | t
+(1 row)
+
+SELECT 'voyageai_get_api_key_test' AS test_name,
+       (SELECT rag.voyageai_get_api_key() = 'test_key') AS result;
+         test_name         | result 
+---------------------------+--------
+ voyageai_get_api_key_test | t
+(1 row)
+
+-- Test VoyageAI embedding functions exist
+SELECT 'voyageai_embedding_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+         test_name         | result 
+---------------------------+--------
+ voyageai_embedding_exists | t
+(1 row)
+
+SELECT 'voyageai_embedding_3_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_3'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+          test_name          | result 
+-----------------------------+--------
+ voyageai_embedding_3_exists | t
+(1 row)
+
+SELECT 'voyageai_embedding_3_lite_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_3_lite'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+            test_name             | result 
+----------------------------------+--------
+ voyageai_embedding_3_lite_exists | t
+(1 row)
+
+SELECT 'voyageai_embedding_code_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_code_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+            test_name             | result 
+----------------------------------+--------
+ voyageai_embedding_code_2_exists | t
+(1 row)
+
+SELECT 'voyageai_embedding_finance_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_finance_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+              test_name              | result 
+-------------------------------------+--------
+ voyageai_embedding_finance_2_exists | t
+(1 row)
+
+SELECT 'voyageai_embedding_law_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_law_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+            test_name            | result 
+---------------------------------+--------
+ voyageai_embedding_law_2_exists | t
+(1 row)
+
+SELECT 'voyageai_embedding_multilingual_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_multilingual_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+                test_name                 | result 
+------------------------------------------+--------
+ voyageai_embedding_multilingual_2_exists | t
+(1 row)
+
+-- Test VoyageAI reranking functions exist
+SELECT 'voyageai_rerank_distance_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_distance'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+            test_name            | result 
+---------------------------------+--------
+ voyageai_rerank_distance_exists | t
+(1 row)
+
+SELECT 'voyageai_rerank_score_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_score'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+          test_name           | result 
+------------------------------+--------
+ voyageai_rerank_score_exists | t
+(1 row)
+
+-- Test VoyageAI function signatures
+SELECT 'voyageai_embedding_signature' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag')
+  AND pronargs = 3;
+          test_name           | result 
+------------------------------+--------
+ voyageai_embedding_signature | t
+(1 row)
+
+SELECT 'voyageai_rerank_distance_signature' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_distance'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag')
+  AND pronargs IN (3, 4);
+             test_name              | result 
+------------------------------------+--------
+ voyageai_rerank_distance_signature | t
+(1 row)
+
+SELECT 'voyageai_rerank_score_signature' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_score'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag')
+  AND pronargs IN (3, 4);
+            test_name            | result 
+---------------------------------+--------
+ voyageai_rerank_score_signature | t
+(1 row)
+

docker-compose/ext-src/pgrag-src/sql/api_keys.sql

@@ -0,0 +1,16 @@
+-- API key function tests
+SELECT rag.anthropic_set_api_key('test_key');
+
+SELECT rag.anthropic_get_api_key();
+
+SELECT rag.openai_set_api_key('test_key');
+
+SELECT rag.openai_get_api_key();
+
+SELECT rag.fireworks_set_api_key('test_key');
+
+SELECT rag.fireworks_get_api_key();
+
+SELECT rag.voyageai_set_api_key('test_key');
+
+SELECT rag.voyageai_get_api_key();

docker-compose/ext-src/pgrag-src/sql/basic_functions.sql

@@ -0,0 +1,4 @@
+-- Basic function tests
+SELECT rag.markdown_from_html('<p>Hello</p>');
+
+SELECT array_length(rag.chunks_by_character_count('the cat sat on the mat', 10, 5), 1);

docker-compose/ext-src/pgrag-src/sql/chunking_functions.sql

@@ -0,0 +1,11 @@
+-- Chunking function tests
+SELECT rag.chunks_by_character_count('the cat sat on the mat', 10, 5);
+
+SELECT rag.chunks_by_character_count('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', 20, 10);
+
+SELECT (rag.chunks_by_character_count('the cat', 10, 0))[1];
+
+SELECT rag.chunks_by_character_count('', 10, 5);
+
+SELECT rag.chunks_by_character_count('a b c d e f g h i j k l m n o p', 5, 2);
+

docker-compose/ext-src/pgrag-src/sql/document_processing.sql

@@ -0,0 +1,24 @@
+-- HTML to Markdown conversion tests
+SELECT rag.markdown_from_html('<p>Hello</p>');
+
+SELECT rag.markdown_from_html('<p>Hello <i>world</i></p>');
+
+SELECT rag.markdown_from_html('<h1>Title</h1><p>Paragraph</p>');
+
+SELECT rag.markdown_from_html('<ul><li>Item 1</li><li>Item 2</li></ul>');
+
+SELECT rag.markdown_from_html('<a href="https://example.com">Link</a>');
+
+-- Note: text_from_pdf and text_from_docx require binary input which is harder to test in regression tests
+-- We'll test that the functions exist and have the right signature
+SELECT 'text_from_pdf_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'text_from_pdf'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'text_from_docx_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'text_from_docx'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');

docker-compose/ext-src/pgrag-src/sql/embedding_api_functions.sql

@@ -0,0 +1,62 @@
+-- Test embedding functions exist with correct signatures
+-- OpenAI embedding functions
+SELECT 'openai_text_embedding_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'openai_text_embedding_3_small_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding_3_small'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'openai_text_embedding_3_large_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding_3_large'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'openai_text_embedding_ada_002_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'openai_text_embedding_ada_002'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+-- Fireworks embedding functions
+SELECT 'fireworks_nomic_embed_text_v1_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_nomic_embed_text_v1'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'fireworks_nomic_embed_text_v15_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_nomic_embed_text_v15'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'fireworks_text_embedding_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'fireworks_text_embedding_thenlper_gte_base_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding_thenlper_gte_base'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'fireworks_text_embedding_thenlper_gte_large_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding_thenlper_gte_large'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'fireworks_text_embedding_whereisai_uae_large_v1_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'fireworks_text_embedding_whereisai_uae_large_v1'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');

docker-compose/ext-src/pgrag-src/sql/text_processing.sql

@@ -0,0 +1,4 @@
+-- Text processing function tests
+SELECT rag.markdown_from_html('<p>Hello <i>world</i></p>');
+
+SELECT rag.chunks_by_character_count('the cat sat on the mat', 10, 5);

docker-compose/ext-src/pgrag-src/sql/voyageai_functions.sql

@@ -0,0 +1,84 @@
+-- Test VoyageAI API key functions
+SELECT 'voyageai_api_key_test' AS test_name, 
+       (SELECT rag.voyageai_set_api_key('test_key') IS NULL) AS result;
+
+SELECT 'voyageai_get_api_key_test' AS test_name,
+       (SELECT rag.voyageai_get_api_key() = 'test_key') AS result;
+
+-- Test VoyageAI embedding functions exist
+SELECT 'voyageai_embedding_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'voyageai_embedding_3_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_3'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'voyageai_embedding_3_lite_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_3_lite'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'voyageai_embedding_code_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_code_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'voyageai_embedding_finance_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_finance_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'voyageai_embedding_law_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_law_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'voyageai_embedding_multilingual_2_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding_multilingual_2'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+-- Test VoyageAI reranking functions exist
+SELECT 'voyageai_rerank_distance_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_distance'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+SELECT 'voyageai_rerank_score_exists' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_score'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag');
+
+-- Test VoyageAI function signatures
+SELECT 'voyageai_embedding_signature' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_embedding'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag')
+  AND pronargs = 3;
+
+SELECT 'voyageai_rerank_distance_signature' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_distance'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag')
+  AND pronargs IN (3, 4);
+
+SELECT 'voyageai_rerank_score_signature' AS test_name,
+       count(*) > 0 AS result
+FROM pg_proc
+WHERE proname = 'voyageai_rerank_score'
+  AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'rag')
+  AND pronargs IN (3, 4);

docker-compose/ext-src/pgx_ulid-src/Makefile

@@ -0,0 +1,16 @@
+EXTENSION = pgx_ulid
+
+PGFILEDESC = "pgx_ulid - ULID type for PostgreSQL"
+
+PG_CONFIG ?= pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+PG_MAJOR_VERSION := $(word 2, $(subst ., , $(shell $(PG_CONFIG) --version)))
+ifeq ($(shell test $(PG_MAJOR_VERSION) -lt 17; echo $$?),0)
+  REGRESS_OPTS = --load-extension=ulid
+  REGRESS = 00_ulid_generation 01_ulid_conversions 03_ulid_errors
+else
+  REGRESS_OPTS = --load-extension=pgx_ulid
+  REGRESS = 00_ulid_generation 01_ulid_conversions 02_ulid_conversions 03_ulid_errors
+endif
+
+include $(PGXS)

docker-compose/ext-src/pgx_ulid-src/expected/00_ulid_generation.out

@@ -0,0 +1,60 @@
+-- Test basic ULID generation
+-- Test gen_ulid() function
+SELECT 'gen_ulid() returns a non-null value' as test_name,
+       gen_ulid() IS NOT NULL as result;
+              test_name              | result 
+-------------------------------------+--------
+ gen_ulid() returns a non-null value | t
+(1 row)
+
+-- Test that multiple calls to gen_ulid() return different values
+SELECT 'gen_ulid() returns unique values' as test_name,
+       gen_ulid() != gen_ulid() as result;
+            test_name             | result 
+----------------------------------+--------
+ gen_ulid() returns unique values | t
+(1 row)
+
+-- Test that gen_ulid() returns a value with the correct format
+SELECT 'gen_ulid() returns correctly formatted value' as test_name,
+       length(gen_ulid()::text) = 26 as result;
+                  test_name                   | result 
+----------------------------------------------+--------
+ gen_ulid() returns correctly formatted value | t
+(1 row)
+
+-- Test monotonic ULID generation
+SELECT 'gen_monotonic_ulid() returns a non-null value' as test_name,
+       gen_monotonic_ulid() IS NOT NULL as result;
+                   test_name                   | result 
+-----------------------------------------------+--------
+ gen_monotonic_ulid() returns a non-null value | t
+(1 row)
+
+-- Test that multiple calls to gen_monotonic_ulid() return different values
+SELECT 'gen_monotonic_ulid() returns unique values' as test_name,
+       gen_monotonic_ulid() != gen_monotonic_ulid() as result;
+                 test_name                  | result 
+--------------------------------------------+--------
+ gen_monotonic_ulid() returns unique values | t
+(1 row)
+
+-- Test that gen_monotonic_ulid() returns a value with the correct format
+SELECT 'gen_monotonic_ulid() returns correctly formatted value' as test_name,
+       length(gen_monotonic_ulid()::text) = 26 as result;
+                       test_name                        | result 
+--------------------------------------------------------+--------
+ gen_monotonic_ulid() returns correctly formatted value | t
+(1 row)
+
+-- Test that monotonic ULIDs are ordered correctly
+SELECT 'gen_monotonic_ulid() returns ordered values' as test_name,
+       u1 < u2 as result
+FROM (
+    SELECT gen_monotonic_ulid() as u1, gen_monotonic_ulid() as u2
+) subq;
+                  test_name                  | result 
+---------------------------------------------+--------
+ gen_monotonic_ulid() returns ordered values | t
+(1 row)
+

docker-compose/ext-src/pgx_ulid-src/expected/01_ulid_conversions.out

@@ -0,0 +1,55 @@
+-- Create a test ULID value
+CREATE TEMP TABLE test_ulids AS
+SELECT '01GV5PA9EQG7D82Q3Y4PKBZSYV'::ulid as test_ulid;
+-- Test conversion to text
+SELECT 'ulid to text conversion' as test_name,
+       test_ulid::text = '01GV5PA9EQG7D82Q3Y4PKBZSYV' as result
+FROM test_ulids;
+        test_name        | result 
+-------------------------+--------
+ ulid to text conversion | t
+(1 row)
+
+-- Test conversion to UUID
+SELECT 'ulid to UUID conversion' as test_name,
+       test_ulid::uuid::text = '0186cb65-25d7-81da-815c-7e25a6bfe7db' as result
+FROM test_ulids;
+        test_name        | result 
+-------------------------+--------
+ ulid to UUID conversion | t
+(1 row)
+
+-- Test conversion to bytea
+SELECT 'ulid to bytea conversion' as test_name,
+       length(test_ulid::bytea) = 16 as result
+FROM test_ulids;
+        test_name         | result 
+--------------------------+--------
+ ulid to bytea conversion | t
+(1 row)
+
+-- Test conversion to timestamp
+SELECT 'ulid to timestamp conversion' as test_name,
+       to_char(test_ulid::timestamp, 'YYYY-MM-DD HH24:MI:SS.MS') = '2023-03-10 04:00:49.111' as result
+FROM test_ulids;
+          test_name           | result 
+------------------------------+--------
+ ulid to timestamp conversion | t
+(1 row)
+
+-- Test conversion from UUID
+SELECT 'UUID to ulid conversion' as test_name,
+       '0186cb65-25d7-81da-815c-7e25a6bfe7db'::uuid::ulid::text = '01GV5PA9EQG7D82Q3Y4PKBZSYV' as result;
+        test_name        | result 
+-------------------------+--------
+ UUID to ulid conversion | t
+(1 row)
+
+-- Test conversion from timestamp
+SELECT 'timestamp to ulid conversion' as test_name,
+       '2023-03-10 12:00:49.111'::timestamp::ulid::text = '01GV5PA9EQ0000000000000000' as result;
+          test_name           | result 
+------------------------------+--------
+ timestamp to ulid conversion | t
+(1 row)
+

docker-compose/ext-src/pgx_ulid-src/expected/02_ulid_conversions.out

@@ -0,0 +1,8 @@
+-- Test conversion from timestamptz
+SELECT 'timestamptz to ulid conversion' as test_name,
+       '2023-03-10 04:00:49.111'::timestamptz::ulid::text = '01GV5PA9EQ0000000000000000' as result;
+           test_name            | result 
+--------------------------------+--------
+ timestamptz to ulid conversion | t
+(1 row)
+

docker-compose/ext-src/pgx_ulid-src/expected/03_ulid_errors.out

@@ -0,0 +1,19 @@
+-- Test ULID error handling
+-- Test invalid ULID string (too short)
+SELECT '01GV5PA9EQG7D82Q3Y4PKBZSY'::ulid;
+ERROR:  invalid input syntax for type ulid: "01GV5PA9EQG7D82Q3Y4PKBZSY": invalid length
+LINE 1: SELECT '01GV5PA9EQG7D82Q3Y4PKBZSY'::ulid;
+               ^
+-- Test invalid ULID string (invalid character)
+SELECT '01GV5PA9EQG7D82Q3Y4PKBZSYU'::ulid;
+ERROR:  invalid input syntax for type ulid: "01GV5PA9EQG7D82Q3Y4PKBZSYU": invalid character
+LINE 1: SELECT '01GV5PA9EQG7D82Q3Y4PKBZSYU'::ulid;
+               ^
+-- Test NULL handling
+SELECT 'NULL to ulid conversion returns NULL' as test_name,
+       NULL::ulid IS NULL as result;
+              test_name               | result 
+--------------------------------------+--------
+ NULL to ulid conversion returns NULL | t
+(1 row)
+

docker-compose/ext-src/pgx_ulid-src/sql/00_ulid_generation.sql

@@ -0,0 +1,32 @@
+-- Test basic ULID generation
+
+-- Test gen_ulid() function
+SELECT 'gen_ulid() returns a non-null value' as test_name,
+       gen_ulid() IS NOT NULL as result;
+
+-- Test that multiple calls to gen_ulid() return different values
+SELECT 'gen_ulid() returns unique values' as test_name,
+       gen_ulid() != gen_ulid() as result;
+
+-- Test that gen_ulid() returns a value with the correct format
+SELECT 'gen_ulid() returns correctly formatted value' as test_name,
+       length(gen_ulid()::text) = 26 as result;
+
+-- Test monotonic ULID generation
+SELECT 'gen_monotonic_ulid() returns a non-null value' as test_name,
+       gen_monotonic_ulid() IS NOT NULL as result;
+
+-- Test that multiple calls to gen_monotonic_ulid() return different values
+SELECT 'gen_monotonic_ulid() returns unique values' as test_name,
+       gen_monotonic_ulid() != gen_monotonic_ulid() as result;
+
+-- Test that gen_monotonic_ulid() returns a value with the correct format
+SELECT 'gen_monotonic_ulid() returns correctly formatted value' as test_name,
+       length(gen_monotonic_ulid()::text) = 26 as result;
+
+-- Test that monotonic ULIDs are ordered correctly
+SELECT 'gen_monotonic_ulid() returns ordered values' as test_name,
+       u1 < u2 as result
+FROM (
+    SELECT gen_monotonic_ulid() as u1, gen_monotonic_ulid() as u2
+) subq;

docker-compose/ext-src/pgx_ulid-src/sql/01_ulid_conversions.sql

@@ -0,0 +1,32 @@
+-- Create a test ULID value
+CREATE TEMP TABLE test_ulids AS
+SELECT '01GV5PA9EQG7D82Q3Y4PKBZSYV'::ulid as test_ulid;
+
+-- Test conversion to text
+SELECT 'ulid to text conversion' as test_name,
+       test_ulid::text = '01GV5PA9EQG7D82Q3Y4PKBZSYV' as result
+FROM test_ulids;
+
+-- Test conversion to UUID
+SELECT 'ulid to UUID conversion' as test_name,
+       test_ulid::uuid::text = '0186cb65-25d7-81da-815c-7e25a6bfe7db' as result
+FROM test_ulids;
+
+-- Test conversion to bytea
+SELECT 'ulid to bytea conversion' as test_name,
+       length(test_ulid::bytea) = 16 as result
+FROM test_ulids;
+
+-- Test conversion to timestamp
+SELECT 'ulid to timestamp conversion' as test_name,
+       to_char(test_ulid::timestamp, 'YYYY-MM-DD HH24:MI:SS.MS') = '2023-03-10 04:00:49.111' as result
+FROM test_ulids;
+
+-- Test conversion from UUID
+SELECT 'UUID to ulid conversion' as test_name,
+       '0186cb65-25d7-81da-815c-7e25a6bfe7db'::uuid::ulid::text = '01GV5PA9EQG7D82Q3Y4PKBZSYV' as result;
+
+-- Test conversion from timestamp
+SELECT 'timestamp to ulid conversion' as test_name,
+       '2023-03-10 12:00:49.111'::timestamp::ulid::text = '01GV5PA9EQ0000000000000000' as result;
+

docker-compose/ext-src/pgx_ulid-src/sql/02_ulid_conversions.sql

@@ -0,0 +1,3 @@
+-- Test conversion from timestamptz
+SELECT 'timestamptz to ulid conversion' as test_name,
+       '2023-03-10 04:00:49.111'::timestamptz::ulid::text = '01GV5PA9EQ0000000000000000' as result;

docker-compose/ext-src/pgx_ulid-src/sql/03_ulid_errors.sql

@@ -0,0 +1,12 @@
+-- Test ULID error handling
+
+-- Test invalid ULID string (too short)
+SELECT '01GV5PA9EQG7D82Q3Y4PKBZSY'::ulid;
+
+-- Test invalid ULID string (invalid character)
+SELECT '01GV5PA9EQG7D82Q3Y4PKBZSYU'::ulid;
+
+-- Test NULL handling
+SELECT 'NULL to ulid conversion returns NULL' as test_name,
+       NULL::ulid IS NULL as result;
+

docker-compose/ext-src/rag_bge_small_en_v15-src/Makefile

@@ -0,0 +1,10 @@
+EXTENSION = rag_bge_small_en_v15
+MODULE_big = rag_bge_small_en_v15
+OBJS = $(patsubst %.rs,%.o,$(wildcard src/*.rs))
+
+REGRESS = basic_functions embedding_functions basic_functions_enhanced embedding_functions_enhanced
+REGRESS_OPTS = --load-extension=vector --load-extension=rag_bge_small_en_v15
+
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)

docker-compose/ext-src/rag_bge_small_en_v15-src/expected/basic_functions.out

@@ -0,0 +1,7 @@
+-- Basic function tests
+SELECT rag_bge_small_en_v15.chunks_by_token_count('the cat sat on the mat', 3, 2);
+                 chunks_by_token_count                  
+--------------------------------------------------------
+ {"the cat sat","cat sat on","sat on the","on the mat"}
+(1 row)
+

docker-compose/ext-src/rag_bge_small_en_v15-src/expected/basic_functions_enhanced.out

@@ -0,0 +1,31 @@
+-- Basic function tests for chunks_by_token_count
+SELECT rag_bge_small_en_v15.chunks_by_token_count('the cat sat on the mat', 3, 2);
+                 chunks_by_token_count                  
+--------------------------------------------------------
+ {"the cat sat","cat sat on","sat on the","on the mat"}
+(1 row)
+
+SELECT rag_bge_small_en_v15.chunks_by_token_count('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', 5, 2);
+                                                                              chunks_by_token_count                                                                              
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ {"Lorem ipsum","ipsum dolor sit","sit amet,",consectetur,"adipiscing elit",elit.,"Sed do","do eiusmod",tempor,"incididunt ut","ut labore et","et dolore magna","magna aliqua."}
+(1 row)
+
+SELECT (rag_bge_small_en_v15.chunks_by_token_count('the cat', 5, 0))[1];
+ chunks_by_token_count 
+-----------------------
+ the cat
+(1 row)
+
+SELECT rag_bge_small_en_v15.chunks_by_token_count('', 5, 2);
+ chunks_by_token_count 
+-----------------------
+ {}
+(1 row)
+
+SELECT rag_bge_small_en_v15.chunks_by_token_count('a b c d e f g h i j k l m n o p', 3, 1);
+                      chunks_by_token_count                      
+-----------------------------------------------------------------
+ {"a b c","c d e","e f g","g h i","i j k","k l m","m n o","o p"}
+(1 row)
+

docker-compose/ext-src/rag_bge_small_en_v15-src/expected/embedding_functions.out

@@ -0,0 +1,15 @@
+-- Embedding function tests
+SELECT 'embedding_for_passage_test' AS test_name, 
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('the cat sat on the mat')) > 0 AS result;
+         test_name          | result 
+----------------------------+--------
+ embedding_for_passage_test | t
+(1 row)
+
+SELECT 'embedding_for_query_test' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('the cat sat on the mat')) > 0 AS result;
+        test_name         | result 
+--------------------------+--------
+ embedding_for_query_test | t
+(1 row)
+

docker-compose/ext-src/rag_bge_small_en_v15-src/expected/embedding_functions_enhanced.out

@@ -0,0 +1,52 @@
+-- Embedding function tests
+SELECT 'embedding_for_passage_test_1' AS test_name, 
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('the cat sat on the mat')) > 0 AS result;
+          test_name           | result 
+------------------------------+--------
+ embedding_for_passage_test_1 | t
+(1 row)
+
+SELECT 'embedding_for_passage_test_2' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('Lorem ipsum dolor sit amet')) > 0 AS result;
+          test_name           | result 
+------------------------------+--------
+ embedding_for_passage_test_2 | t
+(1 row)
+
+SELECT 'embedding_for_passage_test_3' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('')) > 0 AS result;
+          test_name           | result 
+------------------------------+--------
+ embedding_for_passage_test_3 | t
+(1 row)
+
+SELECT 'embedding_for_query_test_1' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('the cat sat on the mat')) > 0 AS result;
+         test_name          | result 
+----------------------------+--------
+ embedding_for_query_test_1 | t
+(1 row)
+
+SELECT 'embedding_for_query_test_2' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('Lorem ipsum dolor sit amet')) > 0 AS result;
+         test_name          | result 
+----------------------------+--------
+ embedding_for_query_test_2 | t
+(1 row)
+
+SELECT 'embedding_for_query_test_3' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('')) > 0 AS result;
+         test_name          | result 
+----------------------------+--------
+ embedding_for_query_test_3 | t
+(1 row)
+
+-- Test that passage and query embeddings have the same dimensions
+SELECT 'embedding_dimensions_match' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('test')) = 
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('test')) AS result;
+         test_name          | result 
+----------------------------+--------
+ embedding_dimensions_match | t
+(1 row)
+

docker-compose/ext-src/rag_bge_small_en_v15-src/sql/basic_functions.sql

@@ -0,0 +1,2 @@
+-- Basic function tests
+SELECT rag_bge_small_en_v15.chunks_by_token_count('the cat sat on the mat', 3, 2);

docker-compose/ext-src/rag_bge_small_en_v15-src/sql/basic_functions_enhanced.sql

@@ -0,0 +1,10 @@
+-- Basic function tests for chunks_by_token_count
+SELECT rag_bge_small_en_v15.chunks_by_token_count('the cat sat on the mat', 3, 2);
+
+SELECT rag_bge_small_en_v15.chunks_by_token_count('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', 5, 2);
+
+SELECT (rag_bge_small_en_v15.chunks_by_token_count('the cat', 5, 0))[1];
+
+SELECT rag_bge_small_en_v15.chunks_by_token_count('', 5, 2);
+
+SELECT rag_bge_small_en_v15.chunks_by_token_count('a b c d e f g h i j k l m n o p', 3, 1);

docker-compose/ext-src/rag_bge_small_en_v15-src/sql/embedding_functions.sql

@@ -0,0 +1,6 @@
+-- Embedding function tests
+SELECT 'embedding_for_passage_test' AS test_name, 
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('the cat sat on the mat')) > 0 AS result;
+
+SELECT 'embedding_for_query_test' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('the cat sat on the mat')) > 0 AS result;

docker-compose/ext-src/rag_bge_small_en_v15-src/sql/embedding_functions_enhanced.sql

@@ -0,0 +1,23 @@
+-- Embedding function tests
+SELECT 'embedding_for_passage_test_1' AS test_name, 
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('the cat sat on the mat')) > 0 AS result;
+
+SELECT 'embedding_for_passage_test_2' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('Lorem ipsum dolor sit amet')) > 0 AS result;
+
+SELECT 'embedding_for_passage_test_3' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('')) > 0 AS result;
+
+SELECT 'embedding_for_query_test_1' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('the cat sat on the mat')) > 0 AS result;
+
+SELECT 'embedding_for_query_test_2' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('Lorem ipsum dolor sit amet')) > 0 AS result;
+
+SELECT 'embedding_for_query_test_3' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('')) > 0 AS result;
+
+-- Test that passage and query embeddings have the same dimensions
+SELECT 'embedding_dimensions_match' AS test_name,
+       vector_dims(rag_bge_small_en_v15.embedding_for_passage('test')) = 
+       vector_dims(rag_bge_small_en_v15.embedding_for_query('test')) AS result;

docker-compose/ext-src/rag_jina_reranker_v1_tiny_en-src/Makefile

@@ -0,0 +1,10 @@
+EXTENSION = rag_jina_reranker_v1_tiny_en
+MODULE_big = rag_jina_reranker_v1_tiny_en
+OBJS = $(patsubst %.rs,%.o,$(wildcard src/*.rs))
+
+REGRESS = reranking_functions reranking_functions_enhanced
+REGRESS_OPTS = --load-extension=vector --load-extension=rag_jina_reranker_v1_tiny_en
+
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)

docker-compose/ext-src/rag_jina_reranker_v1_tiny_en-src/expected/reranking_functions.out

@@ -0,0 +1,25 @@
+-- Reranking function tests
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', 'the baboon played with the balloon');
+ rerank_distance 
+-----------------
+       0.8989152
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);
+    rerank_distance    
+-----------------------
+ {0.8989152,1.3018152}
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', 'the baboon played with the balloon');
+ rerank_score 
+--------------
+   -0.8989152
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);
+      rerank_score       
+-------------------------
+ {-0.8989152,-1.3018152}
+(1 row)
+

docker-compose/ext-src/rag_jina_reranker_v1_tiny_en-src/expected/reranking_functions_enhanced.out

@@ -0,0 +1,92 @@
+-- Reranking function tests - single passage
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', 'the baboon played with the balloon');
+ rerank_distance 
+-----------------
+       0.8989152
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', 'the tanks fired at the buildings');
+ rerank_distance 
+-----------------
+       1.3018152
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('query about cats', 'information about felines');
+ rerank_distance 
+-----------------
+       1.3133051
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('', 'empty query test');
+ rerank_distance 
+-----------------
+       0.7075559
+(1 row)
+
+-- Reranking function tests - array of passages
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat',
+    ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);
+    rerank_distance    
+-----------------------
+ {0.8989152,1.3018152}
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('query about programming',
+    ARRAY['Python is a programming language', 'Java is also a programming language', 'SQL is used for databases']);
+          rerank_distance           
+------------------------------------
+ {0.16591403,0.33475375,0.10132827}
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('empty array test', ARRAY[]::text[]);
+ rerank_distance 
+-----------------
+ {}
+(1 row)
+
+-- Reranking score function tests - single passage
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', 'the baboon played with the balloon');
+ rerank_score 
+--------------
+   -0.8989152
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', 'the tanks fired at the buildings');
+ rerank_score 
+--------------
+   -1.3018152
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('query about cats', 'information about felines');
+ rerank_score 
+--------------
+   -1.3133051
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('', 'empty query test');
+ rerank_score 
+--------------
+   -0.7075559
+(1 row)
+
+-- Reranking score function tests - array of passages
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat',
+    ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);
+      rerank_score       
+-------------------------
+ {-0.8989152,-1.3018152}
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('query about programming',
+    ARRAY['Python is a programming language', 'Java is also a programming language', 'SQL is used for databases']);
+             rerank_score              
+---------------------------------------
+ {-0.16591403,-0.33475375,-0.10132827}
+(1 row)
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('empty array test', ARRAY[]::text[]);
+ rerank_score 
+--------------
+ {}
+(1 row)
+

docker-compose/ext-src/rag_jina_reranker_v1_tiny_en-src/sql/reranking_functions.sql

@@ -0,0 +1,8 @@
+-- Reranking function tests
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', 'the baboon played with the balloon');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', 'the baboon played with the balloon');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);

docker-compose/ext-src/rag_jina_reranker_v1_tiny_en-src/sql/reranking_functions_enhanced.sql

@@ -0,0 +1,35 @@
+-- Reranking function tests - single passage
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', 'the baboon played with the balloon');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat', 'the tanks fired at the buildings');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('query about cats', 'information about felines');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('', 'empty query test');
+
+-- Reranking function tests - array of passages
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('the cat sat on the mat',
+    ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('query about programming',
+    ARRAY['Python is a programming language', 'Java is also a programming language', 'SQL is used for databases']);
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_distance('empty array test', ARRAY[]::text[]);
+
+-- Reranking score function tests - single passage
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', 'the baboon played with the balloon');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat', 'the tanks fired at the buildings');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('query about cats', 'information about felines');
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('', 'empty query test');
+
+-- Reranking score function tests - array of passages
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('the cat sat on the mat',
+    ARRAY['the baboon played with the balloon', 'the tanks fired at the buildings']);
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('query about programming',
+    ARRAY['Python is a programming language', 'Java is also a programming language', 'SQL is used for databases']);
+
+SELECT rag_jina_reranker_v1_tiny_en.rerank_score('empty array test', ARRAY[]::text[]);

docker-compose/run-tests.sh

@@ -20,4 +20,4 @@ for d in ${LIST}; do
 done
 [ -z "${FAILED}" ] && exit 0
 echo "${FAILED}"
-exit 1
+exit 1

libs/pageserver_api/src/config.rs

@@ -51,9 +51,54 @@ pub struct NodeMetadata {
 /// If there cannot be a static default value because we need to make runtime
 /// checks to determine the default, make it an `Option` (which defaults to None).
 /// The runtime check should be done in the consuming crate, i.e., `pageserver`.
+///
+/// Unknown fields are silently ignored during deserialization.
+/// The alternative, which we used in the past, was to set `deny_unknown_fields`,
+/// which fails deserialization, and hence pageserver startup, if there is an unknown field.
+/// The reason we don't do that anymore is that it complicates
+/// usage of config fields for feature flagging, which we commonly do for
+/// region-by-region rollouts.
+/// The complications mainly arise because the `pageserver.toml` contents on a
+/// prod server have a separate lifecycle from the pageserver binary.
+/// For instance, `pageserver.toml` contents today are defined in the internal
+/// infra repo, and thus introducing a new config field to pageserver and
+/// rolling it out to prod servers are separate commits in separate repos
+/// that can't be made or rolled back atomically.
+/// Rollbacks in particular pose a risk with deny_unknown_fields because
+/// the old pageserver binary may reject a new config field, resulting in
+/// an outage unless the person doing the pageserver rollback remembers
+/// to also revert the commit that added the config field in to the
+/// `pageserver.toml` templates in the internal infra repo.
+/// (A pre-deploy config check would eliminate this risk during rollbacks,
+///  cf [here](https://github.com/neondatabase/cloud/issues/24349).)
+/// In addition to this compatibility problem during emergency rollbacks,
+/// deny_unknown_fields adds further complications when decomissioning a feature
+/// flag: with deny_unknown_fields, we can't remove a flag from the [`ConfigToml`]
+/// until all prod servers' `pageserver.toml` files have been updated to a version
+/// that doesn't specify the flag. Otherwise new software would fail to start up.
+/// This adds the requirement for an intermediate step where the new config field
+/// is accepted but ignored, prolonging the decomissioning process by an entire
+/// release cycle.
+/// By contrast  with unknown fields silently ignored, decomissioning a feature
+/// flag is a one-step process: we can skip the intermediate step and straight
+/// remove the field from the [`ConfigToml`]. We leave the field in the
+/// `pageserver.toml` files on prod servers until we reach certainty that we
+/// will not roll back to old software whose behavior was dependent on config.
+/// Then we can remove the field from the templates in the internal infra repo.
+/// This process is [documented internally](
+/// https://docs.neon.build/storage/pageserver_configuration.html).
+///
+/// Note that above relaxed compatbility for the config format does NOT APPLY
+/// TO THE STORAGE FORMAT. As general guidance, when introducing storage format
+/// changes, ensure that the potential rollback target version will be compatible
+/// with the new format. This must hold regardless of what flags are set in in the `pageserver.toml`:
+/// any format version that exists in an environment must be compatible with the software that runs there.
+/// Use a pageserver.toml flag only to gate whether software _writes_ the new format.
+/// For more compatibility considerations, refer to [internal docs](
+/// https://docs.neon.build/storage/compat.html?highlight=compat#format-versions--compatibility)
 #[serde_as]
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
-#[serde(default, deny_unknown_fields)]
+#[serde(default)]
 pub struct ConfigToml {
     // types mapped 1:1 into the runtime PageServerConfig type
     pub listen_pg_addr: String,
@@ -134,7 +179,6 @@ pub struct ConfigToml {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(deny_unknown_fields)]
 pub struct DiskUsageEvictionTaskConfig {
     pub max_usage_pct: utils::serde_percent::Percent,
     pub min_avail_bytes: u64,
@@ -149,13 +193,11 @@ pub struct DiskUsageEvictionTaskConfig {
 
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "mode", rename_all = "kebab-case")]
-#[serde(deny_unknown_fields)]
 pub enum PageServicePipeliningConfig {
     Serial,
     Pipelined(PageServicePipeliningConfigPipelined),
 }
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(deny_unknown_fields)]
 pub struct PageServicePipeliningConfigPipelined {
     /// Causes runtime errors if larger than max get_vectored batch size.
     pub max_batch_size: NonZeroUsize,
@@ -171,7 +213,6 @@ pub enum PageServiceProtocolPipelinedExecutionStrategy {
 
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "mode", rename_all = "kebab-case")]
-#[serde(deny_unknown_fields)]
 pub enum GetVectoredConcurrentIo {
     /// The read path is fully sequential: layers are visited
     /// one after the other and IOs are issued and waited upon
@@ -240,13 +281,9 @@ impl Default for EvictionOrder {
 #[serde(transparent)]
 pub struct MaxVectoredReadBytes(pub NonZeroUsize);
 
-/// A tenant's calcuated configuration, which is the result of merging a
-/// tenant's TenantConfOpt with the global TenantConf from PageServerConf.
-///
-/// For storing and transmitting individual tenant's configuration, see
-/// TenantConfOpt.
+/// Tenant-level configuration values, used for various purposes.
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(deny_unknown_fields, default)]
+#[serde(default)]
 pub struct TenantConfigToml {
     // Flush out an inmemory layer, if it's holding WAL older than this
     // This puts a backstop on how much WAL needs to be re-digested if the

libs/pageserver_api/src/models.rs

@@ -572,65 +572,126 @@ pub struct TenantConfigPatch {
     pub gc_compaction_ratio_percent: FieldPatch<u64>,
 }
 
-/// An alternative representation of `pageserver::tenant::TenantConf` with
-/// simpler types.
-#[derive(Serialize, Deserialize, Debug, Default, Clone, Eq, PartialEq)]
+/// Like [`crate::config::TenantConfigToml`], but preserves the information
+/// about which parameters are set and which are not.
+///
+/// Used in many places, including durably stored ones.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[serde(default)] // this maps omitted fields in deserialization to None
 pub struct TenantConfig {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub checkpoint_distance: Option<u64>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub checkpoint_timeout: Option<Duration>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub compaction_target_size: Option<u64>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub compaction_period: Option<Duration>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub compaction_threshold: Option<usize>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub compaction_upper_limit: Option<usize>,
-    // defer parsing compaction_algorithm, like eviction_policy
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub compaction_l0_first: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub compaction_l0_semaphore: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub l0_flush_delay_threshold: Option<usize>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub l0_flush_stall_threshold: Option<usize>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub l0_flush_wait_upload: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub gc_horizon: Option<u64>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub gc_period: Option<Duration>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub image_creation_threshold: Option<usize>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub pitr_interval: Option<Duration>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub walreceiver_connect_timeout: Option<Duration>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub lagging_wal_timeout: Option<Duration>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub max_lsn_wal_lag: Option<NonZeroU64>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub eviction_policy: Option<EvictionPolicy>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub min_resident_size_override: Option<u64>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub evictions_low_residence_duration_metric_threshold: Option<Duration>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub heatmap_period: Option<Duration>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub lazy_slru_download: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub timeline_get_throttle: Option<ThrottleConfig>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub image_layer_creation_check_threshold: Option<u8>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub image_creation_preempt_threshold: Option<usize>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub lsn_lease_length: Option<Duration>,
-    #[serde(default)]
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub lsn_lease_length_for_ts: Option<Duration>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub timeline_offloading: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub wal_receiver_protocol_override: Option<PostgresClientProtocol>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub rel_size_v2_enabled: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub gc_compaction_enabled: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub gc_compaction_initial_threshold_kb: Option<u64>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub gc_compaction_ratio_percent: Option<u64>,
 }
 
@@ -809,6 +870,110 @@ impl TenantConfig {
             gc_compaction_ratio_percent,
         })
     }
+
+    pub fn merge(
+        &self,
+        global_conf: crate::config::TenantConfigToml,
+    ) -> crate::config::TenantConfigToml {
+        crate::config::TenantConfigToml {
+            checkpoint_distance: self
+                .checkpoint_distance
+                .unwrap_or(global_conf.checkpoint_distance),
+            checkpoint_timeout: self
+                .checkpoint_timeout
+                .unwrap_or(global_conf.checkpoint_timeout),
+            compaction_target_size: self
+                .compaction_target_size
+                .unwrap_or(global_conf.compaction_target_size),
+            compaction_period: self
+                .compaction_period
+                .unwrap_or(global_conf.compaction_period),
+            compaction_threshold: self
+                .compaction_threshold
+                .unwrap_or(global_conf.compaction_threshold),
+            compaction_upper_limit: self
+                .compaction_upper_limit
+                .unwrap_or(global_conf.compaction_upper_limit),
+            compaction_algorithm: self
+                .compaction_algorithm
+                .as_ref()
+                .unwrap_or(&global_conf.compaction_algorithm)
+                .clone(),
+            compaction_l0_first: self
+                .compaction_l0_first
+                .unwrap_or(global_conf.compaction_l0_first),
+            compaction_l0_semaphore: self
+                .compaction_l0_semaphore
+                .unwrap_or(global_conf.compaction_l0_semaphore),
+            l0_flush_delay_threshold: self
+                .l0_flush_delay_threshold
+                .or(global_conf.l0_flush_delay_threshold),
+            l0_flush_stall_threshold: self
+                .l0_flush_stall_threshold
+                .or(global_conf.l0_flush_stall_threshold),
+            l0_flush_wait_upload: self
+                .l0_flush_wait_upload
+                .unwrap_or(global_conf.l0_flush_wait_upload),
+            gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
+            gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
+            image_creation_threshold: self
+                .image_creation_threshold
+                .unwrap_or(global_conf.image_creation_threshold),
+            pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
+            walreceiver_connect_timeout: self
+                .walreceiver_connect_timeout
+                .unwrap_or(global_conf.walreceiver_connect_timeout),
+            lagging_wal_timeout: self
+                .lagging_wal_timeout
+                .unwrap_or(global_conf.lagging_wal_timeout),
+            max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag),
+            eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy),
+            min_resident_size_override: self
+                .min_resident_size_override
+                .or(global_conf.min_resident_size_override),
+            evictions_low_residence_duration_metric_threshold: self
+                .evictions_low_residence_duration_metric_threshold
+                .unwrap_or(global_conf.evictions_low_residence_duration_metric_threshold),
+            heatmap_period: self.heatmap_period.unwrap_or(global_conf.heatmap_period),
+            lazy_slru_download: self
+                .lazy_slru_download
+                .unwrap_or(global_conf.lazy_slru_download),
+            timeline_get_throttle: self
+                .timeline_get_throttle
+                .clone()
+                .unwrap_or(global_conf.timeline_get_throttle),
+            image_layer_creation_check_threshold: self
+                .image_layer_creation_check_threshold
+                .unwrap_or(global_conf.image_layer_creation_check_threshold),
+            image_creation_preempt_threshold: self
+                .image_creation_preempt_threshold
+                .unwrap_or(global_conf.image_creation_preempt_threshold),
+            lsn_lease_length: self
+                .lsn_lease_length
+                .unwrap_or(global_conf.lsn_lease_length),
+            lsn_lease_length_for_ts: self
+                .lsn_lease_length_for_ts
+                .unwrap_or(global_conf.lsn_lease_length_for_ts),
+            timeline_offloading: self
+                .timeline_offloading
+                .unwrap_or(global_conf.timeline_offloading),
+            wal_receiver_protocol_override: self
+                .wal_receiver_protocol_override
+                .or(global_conf.wal_receiver_protocol_override),
+            rel_size_v2_enabled: self
+                .rel_size_v2_enabled
+                .unwrap_or(global_conf.rel_size_v2_enabled),
+            gc_compaction_enabled: self
+                .gc_compaction_enabled
+                .unwrap_or(global_conf.gc_compaction_enabled),
+            gc_compaction_initial_threshold_kb: self
+                .gc_compaction_initial_threshold_kb
+                .unwrap_or(global_conf.gc_compaction_initial_threshold_kb),
+            gc_compaction_ratio_percent: self
+                .gc_compaction_ratio_percent
+                .unwrap_or(global_conf.gc_compaction_ratio_percent),
+        }
+    }
 }
 
 /// The policy for the aux file storage.
@@ -940,7 +1105,7 @@ pub struct CompactionAlgorithmSettings {
 }
 
 #[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]
-#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)]
+#[serde(tag = "mode", rename_all = "kebab-case")]
 pub enum L0FlushConfig {
     #[serde(rename_all = "snake_case")]
     Direct { max_concurrency: NonZeroUsize },

libs/pageserver_api/src/upcall_api.rs

@@ -1,4 +1,4 @@
-//! Types in this file are for pageserver's upward-facing API calls to the control plane,
+//! Types in this file are for pageserver's upward-facing API calls to the storage controller,
 //! required for acquiring and validating tenant generation numbers.
 //!
 //! See docs/rfcs/025-generation-numbers.md

libs/safekeeper_api/src/models.rs

@@ -23,6 +23,7 @@ pub struct TimelineCreateRequest {
     pub tenant_id: TenantId,
     pub timeline_id: TimelineId,
     pub mconf: Configuration,
+    /// In the PG_VERSION_NUM macro format, like 140017.
     pub pg_version: u32,
     pub system_id: Option<u64>,
     // By default WAL_SEGMENT_SIZE

libs/utils/src/sync/duplex/mpsc.rs

@@ -26,6 +26,10 @@ impl<S: Send, R: Send> Duplex<S, R> {
         self.tx.send(x).await
     }
 
+    pub fn try_send(&self, x: S) -> Result<(), mpsc::error::TrySendError<S>> {
+        self.tx.try_send(x)
+    }
+
     /// Receives the next value for this receiver.
     ///
     /// This method returns `None` if the channel has been closed and there are

pageserver/Cargo.toml

@@ -52,6 +52,7 @@ rustls.workspace = true
 scopeguard.workspace = true
 send-future.workspace = true
 serde.workspace = true
+serde_ignored.workspace = true
 serde_json = { workspace = true, features = ["raw_value"] }
 serde_path_to_error.workspace = true
 serde_with.workspace = true

pageserver/benches/bench_ingest.rs

@@ -13,6 +13,7 @@ use pageserver::{page_cache, virtual_file};
 use pageserver_api::key::Key;
 use pageserver_api::shard::TenantShardId;
 use pageserver_api::value::Value;
+use tokio_util::sync::CancellationToken;
 use utils::bin_ser::BeSer;
 use utils::id::{TenantId, TimelineId};
 use wal_decoder::serialized_batch::SerializedValueBatch;
@@ -57,11 +58,22 @@ async fn ingest(
 
     tokio::fs::create_dir_all(conf.timeline_path(&tenant_shard_id, &timeline_id)).await?;
 
-    let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
+    let ctx =
+        RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error).with_scope_debug_tools();
 
     let gate = utils::sync::gate::Gate::default();
+    let cancel = CancellationToken::new();
 
-    let layer = InMemoryLayer::create(conf, timeline_id, tenant_shard_id, lsn, &gate, &ctx).await?;
+    let layer = InMemoryLayer::create(
+        conf,
+        timeline_id,
+        tenant_shard_id,
+        lsn,
+        &gate,
+        &cancel,
+        &ctx,
+    )
+    .await?;
 
     let data = Value::Image(Bytes::from(vec![0u8; put_size]));
     let data_ser_size = data.serialized_size().unwrap() as usize;

pageserver/ctl/src/layer_map_analyzer.rs

@@ -131,7 +131,8 @@ async fn get_holes(path: &Utf8Path, max_holes: usize, ctx: &RequestContext) -> R
 pub(crate) async fn main(cmd: &AnalyzeLayerMapCmd) -> Result<()> {
     let storage_path = &cmd.path;
     let max_holes = cmd.max_holes.unwrap_or(DEFAULT_MAX_HOLES);
-    let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
+    let ctx =
+        RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error).with_scope_debug_tools();
 
     // Initialize virtual_file (file desriptor cache) and page cache which are needed to access layer persistent B-Tree.
     pageserver::virtual_file::init(

pageserver/ctl/src/layers.rs

@@ -76,7 +76,8 @@ async fn read_image_file(path: impl AsRef<Path>, ctx: &RequestContext) -> Result
 }
 
 pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> {
-    let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
+    let ctx =
+        RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error).with_scope_debug_tools();
     match cmd {
         LayerCmd::List { path } => {
             for tenant in fs::read_dir(path.join(TENANTS_SEGMENT_NAME))? {
@@ -176,7 +177,8 @@ pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> {
             );
             pageserver::page_cache::init(100);
 
-            let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error);
+            let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error)
+                .with_scope_debug_tools();
 
             macro_rules! rewrite_closure {
                 ($($summary_ty:tt)*) => {{