pageserver: route gRPC requests to child shards

OTel 0.28+ by default uses blocking operations in a dedicated thread.

.github/actionlint.yml

@@ -31,7 +31,7 @@ config-variables:
   - NEON_PROD_AWS_ACCOUNT_ID
   - PGREGRESS_PG16_PROJECT_ID
   - PGREGRESS_PG17_PROJECT_ID
-  - PREWARM_PROJECT_ID
+  - PREWARM_PGBENCH_SIZE
   - REMOTE_STORAGE_AZURE_CONTAINER
   - REMOTE_STORAGE_AZURE_REGION
   - SLACK_CICD_CHANNEL_ID

.github/workflows/benchbase_tpcc.yml

@@ -1,384 +0,0 @@
-name: TPC-C like benchmark using benchbase
-
-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    #          ┌───────────── minute (0 - 59)
-    #          │ ┌───────────── hour (0 - 23)
-    #          │ │ ┌───────────── day of the month (1 - 31)
-    #          │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
-    #          │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-    - cron:   '0 6 * * *' # run once a day at 6 AM UTC
-  workflow_dispatch: # adds ability to run this manually
-
-defaults:
-  run:
-    shell: bash -euxo pipefail {0}
-
-concurrency:
-  # Allow only one workflow globally because we do not want to be too noisy in production environment
-  group: benchbase-tpcc-workflow
-  cancel-in-progress: false
-
-permissions:
-  contents: read
-
-jobs:
-  benchbase-tpcc:
-    strategy:
-      fail-fast: false # allow other variants to continue even if one fails
-      matrix:
-        include:
-          - warehouses: 50 # defines number of warehouses and is used to compute number of terminals
-            max_rate: 800  # measured max TPS at scale factor based on experiments. Adjust if performance is better/worse
-            min_cu: 0.25   # simulate free tier plan (0.25 -2 CU)
-            max_cu: 2
-          - warehouses: 500 # serverless plan (2-8 CU)
-            max_rate: 2000
-            min_cu: 2
-            max_cu: 8
-          - warehouses: 1000 # business plan (2-16 CU)
-            max_rate: 2900
-            min_cu: 2
-            max_cu: 16
-      max-parallel: 1 # we want to run each workload size sequentially to avoid noisy neighbors
-    permissions:
-      contents: write
-      statuses: write
-      id-token: write # aws-actions/configure-aws-credentials
-    env:
-      PG_CONFIG: /tmp/neon/pg_install/v17/bin/pg_config
-      PSQL: /tmp/neon/pg_install/v17/bin/psql
-      PG_17_LIB_PATH: /tmp/neon/pg_install/v17/lib
-      POSTGRES_VERSION: 17
-    runs-on: [ self-hosted, us-east-2, x64 ]
-    timeout-minutes: 1440
-
-    steps:
-    - name: Harden the runner (Audit all outbound calls)
-      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
-      with:
-        egress-policy: audit
-
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-    - name: Configure AWS credentials # necessary to download artefacts
-      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
-      with:
-        aws-region: eu-central-1
-        role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-        role-duration-seconds: 18000 # 5 hours is currently max associated with IAM role
-
-    - name: Download Neon artifact
-      uses: ./.github/actions/download
-      with:
-        name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
-        path: /tmp/neon/
-        prefix: latest
-        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-
-    - name: Create Neon Project
-      id: create-neon-project-tpcc
-      uses: ./.github/actions/neon-project-create
-      with:
-        region_id: aws-us-east-2
-        postgres_version: ${{ env.POSTGRES_VERSION }}
-        compute_units: '[${{ matrix.min_cu }}, ${{ matrix.max_cu }}]'
-        api_key: ${{ secrets.NEON_PRODUCTION_API_KEY_4_BENCHMARKS }}
-        api_host: console.neon.tech  # production (!)
-
-    - name: Initialize Neon project
-      env:
-          BENCHMARK_TPCC_CONNSTR: ${{ steps.create-neon-project-tpcc.outputs.dsn }}
-          PROJECT_ID: ${{ steps.create-neon-project-tpcc.outputs.project_id }}
-      run: |
-        echo "Initializing Neon project with project_id: ${PROJECT_ID}"
-        export LD_LIBRARY_PATH=${PG_17_LIB_PATH}
-        
-        # Retry logic for psql connection with 1 minute sleep between attempts
-        for attempt in {1..3}; do
-          echo "Attempt ${attempt}/3: Creating extensions in Neon project"
-          if ${PSQL} "${BENCHMARK_TPCC_CONNSTR}" -c "CREATE EXTENSION IF NOT EXISTS neon; CREATE EXTENSION IF NOT EXISTS neon_utils;"; then
-            echo "Successfully created extensions"
-            break
-          else
-            echo "Failed to create extensions on attempt ${attempt}"
-            if [ ${attempt} -lt 3 ]; then
-              echo "Waiting 60 seconds before retry..."
-              sleep 60
-            else
-              echo "All attempts failed, exiting"
-              exit 1
-            fi
-          fi
-        done
-        
-        echo "BENCHMARK_TPCC_CONNSTR=${BENCHMARK_TPCC_CONNSTR}" >> $GITHUB_ENV
-
-    - name: Generate BenchBase workload configuration
-      env:
-        WAREHOUSES: ${{ matrix.warehouses }}
-        MAX_RATE: ${{ matrix.max_rate }}
-      run: |
-        echo "Generating BenchBase configs for warehouses: ${WAREHOUSES}, max_rate: ${MAX_RATE}"
-        
-        # Extract hostname and password from connection string
-        # Format: postgresql://username:password@hostname/database?params (no port for Neon)
-        HOSTNAME=$(echo "${BENCHMARK_TPCC_CONNSTR}" | sed -n 's|.*://[^:]*:[^@]*@\([^/]*\)/.*|\1|p')
-        PASSWORD=$(echo "${BENCHMARK_TPCC_CONNSTR}" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p')
-        
-        echo "Extracted hostname: ${HOSTNAME}"
-        
-        # Use runner temp (NVMe) as working directory
-        cd "${RUNNER_TEMP}"
-        
-        # Copy the generator script
-        cp "${GITHUB_WORKSPACE}/test_runner/performance/benchbase_tpc_c_helpers/generate_workload_size.py" .
-        
-        # Generate configs and scripts
-        python3 generate_workload_size.py \
-          --warehouses ${WAREHOUSES} \
-          --max-rate ${MAX_RATE} \
-          --hostname ${HOSTNAME} \
-          --password ${PASSWORD} \
-          --runner-arch ${{ runner.arch }}
-        
-        # Fix path mismatch: move generated configs and scripts to expected locations
-        mv ../configs ./configs
-        mv ../scripts ./scripts
-
-    - name: Prepare database (load data)
-      env:
-        WAREHOUSES: ${{ matrix.warehouses }}
-      run: |
-        cd "${RUNNER_TEMP}"
-        
-        echo "Loading ${WAREHOUSES} warehouses into database..."
-        
-        # Run the loader script and capture output to log file while preserving stdout/stderr
-        ./scripts/load_${WAREHOUSES}_warehouses.sh 2>&1 | tee "load_${WAREHOUSES}_warehouses.log"
-        
-        echo "Database loading completed"
-
-    - name: Run TPC-C benchmark (warmup phase, then benchmark at 70% of configuredmax TPS)
-      env:
-        WAREHOUSES: ${{ matrix.warehouses }}
-      run: |
-        cd "${RUNNER_TEMP}"
-        
-        echo "Running TPC-C benchmark with ${WAREHOUSES} warehouses..."
-        
-        # Run the optimal rate benchmark
-        ./scripts/execute_${WAREHOUSES}_warehouses_opt_rate.sh
-        
-        echo "Benchmark execution completed"
-
-    - name: Run TPC-C benchmark (warmup phase, then ramp down TPS and up again in 5 minute intervals)
-
-      env:
-          WAREHOUSES: ${{ matrix.warehouses }}
-      run: |
-        cd "${RUNNER_TEMP}"
-        
-        echo "Running TPC-C ramp-down-up with ${WAREHOUSES} warehouses..."
-        
-        # Run the optimal rate benchmark
-        ./scripts/execute_${WAREHOUSES}_warehouses_ramp_up.sh
-        
-        echo "Benchmark execution completed"
-
-    - name: Process results (upload to test results database and generate diagrams)
-      env:
-        WAREHOUSES: ${{ matrix.warehouses }}
-        MIN_CU: ${{ matrix.min_cu }}
-        MAX_CU: ${{ matrix.max_cu }}
-        PROJECT_ID: ${{ steps.create-neon-project-tpcc.outputs.project_id }}
-        REVISION: ${{ github.sha }}
-        PERF_DB_CONNSTR: ${{ secrets.PERF_TEST_RESULT_CONNSTR }}
-      run: |
-        cd "${RUNNER_TEMP}"
-        
-        echo "Creating temporary Python environment for results processing..."
-        
-        # Create temporary virtual environment
-        python3 -m venv temp_results_env
-        source temp_results_env/bin/activate
-        
-        # Install required packages in virtual environment
-        pip install matplotlib pandas psycopg2-binary
-        
-        echo "Copying results processing scripts..."
-        
-        # Copy both processing scripts
-        cp "${GITHUB_WORKSPACE}/test_runner/performance/benchbase_tpc_c_helpers/generate_diagrams.py" .
-        cp "${GITHUB_WORKSPACE}/test_runner/performance/benchbase_tpc_c_helpers/upload_results_to_perf_test_results.py" .
-        
-        echo "Processing load phase metrics..."
-        
-        # Find and process load log
-        LOAD_LOG=$(find . -name "load_${WAREHOUSES}_warehouses.log" -type f | head -1)
-        if [ -n "$LOAD_LOG" ]; then
-          echo "Processing load metrics from: $LOAD_LOG"
-          python upload_results_to_perf_test_results.py \
-            --load-log "$LOAD_LOG" \
-            --run-type "load" \
-            --warehouses "${WAREHOUSES}" \
-            --min-cu "${MIN_CU}" \
-            --max-cu "${MAX_CU}" \
-            --project-id "${PROJECT_ID}" \
-            --revision "${REVISION}" \
-            --connection-string "${PERF_DB_CONNSTR}"
-        else
-          echo "Warning: Load log file not found: load_${WAREHOUSES}_warehouses.log"
-        fi
-        
-        echo "Processing warmup results for optimal rate..."
-        
-        # Find and process warmup results
-        WARMUP_CSV=$(find results_warmup -name "*.results.csv" -type f | head -1)
-        WARMUP_JSON=$(find results_warmup -name "*.summary.json" -type f | head -1)
-        
-        if [ -n "$WARMUP_CSV" ] && [ -n "$WARMUP_JSON" ]; then
-          echo "Generating warmup diagram from: $WARMUP_CSV"
-          python generate_diagrams.py \
-            --input-csv "$WARMUP_CSV" \
-            --output-svg "warmup_${WAREHOUSES}_warehouses_performance.svg" \
-            --title-suffix "Warmup at max TPS"
-            
-          echo "Uploading warmup metrics from: $WARMUP_JSON"
-          python upload_results_to_perf_test_results.py \
-            --summary-json "$WARMUP_JSON" \
-            --results-csv "$WARMUP_CSV" \
-            --run-type "warmup" \
-            --min-cu "${MIN_CU}" \
-            --max-cu "${MAX_CU}" \
-            --project-id "${PROJECT_ID}" \
-            --revision "${REVISION}" \
-            --connection-string "${PERF_DB_CONNSTR}"
-        else
-          echo "Warning: Missing warmup results files (CSV: $WARMUP_CSV, JSON: $WARMUP_JSON)"
-        fi
-        
-        echo "Processing optimal rate results..."
-        
-        # Find and process optimal rate results  
-        OPTRATE_CSV=$(find results_opt_rate -name "*.results.csv" -type f | head -1)
-        OPTRATE_JSON=$(find results_opt_rate -name "*.summary.json" -type f | head -1)
-        
-        if [ -n "$OPTRATE_CSV" ] && [ -n "$OPTRATE_JSON" ]; then
-          echo "Generating optimal rate diagram from: $OPTRATE_CSV"
-          python generate_diagrams.py \
-            --input-csv "$OPTRATE_CSV" \
-            --output-svg "benchmark_${WAREHOUSES}_warehouses_performance.svg" \
-            --title-suffix "70% of max TPS"
-            
-          echo "Uploading optimal rate metrics from: $OPTRATE_JSON"
-          python upload_results_to_perf_test_results.py \
-            --summary-json "$OPTRATE_JSON" \
-            --results-csv "$OPTRATE_CSV" \
-            --run-type "opt-rate" \
-            --min-cu "${MIN_CU}" \
-            --max-cu "${MAX_CU}" \
-            --project-id "${PROJECT_ID}" \
-            --revision "${REVISION}" \
-            --connection-string "${PERF_DB_CONNSTR}"
-        else
-          echo "Warning: Missing optimal rate results files (CSV: $OPTRATE_CSV, JSON: $OPTRATE_JSON)"
-        fi
-
-        echo "Processing warmup 2 results for ramp down/up phase..."
-        
-        # Find and process warmup results
-        WARMUP_CSV=$(find results_warmup -name "*.results.csv" -type f | tail -1)
-        WARMUP_JSON=$(find results_warmup -name "*.summary.json" -type f | tail -1)
-        
-        if [ -n "$WARMUP_CSV" ] && [ -n "$WARMUP_JSON" ]; then
-          echo "Generating warmup diagram from: $WARMUP_CSV"
-          python generate_diagrams.py \
-            --input-csv "$WARMUP_CSV" \
-            --output-svg "warmup_2_${WAREHOUSES}_warehouses_performance.svg" \
-            --title-suffix "Warmup at max TPS"
-            
-          echo "Uploading warmup metrics from: $WARMUP_JSON"
-          python upload_results_to_perf_test_results.py \
-            --summary-json "$WARMUP_JSON" \
-            --results-csv "$WARMUP_CSV" \
-            --run-type "warmup" \
-            --min-cu "${MIN_CU}" \
-            --max-cu "${MAX_CU}" \
-            --project-id "${PROJECT_ID}" \
-            --revision "${REVISION}" \
-            --connection-string "${PERF_DB_CONNSTR}"
-        else
-          echo "Warning: Missing warmup results files (CSV: $WARMUP_CSV, JSON: $WARMUP_JSON)"
-        fi
-        
-        echo "Processing ramp results..."
-        
-        # Find and process ramp results  
-        RAMPUP_CSV=$(find results_ramp_up -name "*.results.csv" -type f | head -1)
-        RAMPUP_JSON=$(find results_ramp_up -name "*.summary.json" -type f | head -1)
-        
-        if [ -n "$RAMPUP_CSV" ] && [ -n "$RAMPUP_JSON" ]; then
-          echo "Generating ramp diagram from: $RAMPUP_CSV"
-          python generate_diagrams.py \
-            --input-csv "$RAMPUP_CSV" \
-            --output-svg "ramp_${WAREHOUSES}_warehouses_performance.svg" \
-            --title-suffix "ramp TPS down and up in 5 minute intervals"
-            
-          echo "Uploading ramp metrics from: $RAMPUP_JSON"
-          python upload_results_to_perf_test_results.py \
-            --summary-json "$RAMPUP_JSON" \
-            --results-csv "$RAMPUP_CSV" \
-            --run-type "ramp-up" \
-            --min-cu "${MIN_CU}" \
-            --max-cu "${MAX_CU}" \
-            --project-id "${PROJECT_ID}" \
-            --revision "${REVISION}" \
-            --connection-string "${PERF_DB_CONNSTR}"
-        else
-          echo "Warning: Missing ramp results files (CSV: $RAMPUP_CSV, JSON: $RAMPUP_JSON)"
-        fi
-        
-        # Deactivate and clean up virtual environment
-        deactivate
-        rm -rf temp_results_env
-        rm upload_results_to_perf_test_results.py
-        
-        echo "Results processing completed and environment cleaned up"
-
-    - name: Set date for upload
-      id: set-date
-      run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT
-
-    - name: Configure AWS credentials # necessary to upload results
-      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
-      with:
-        aws-region: us-east-2
-        role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-        role-duration-seconds: 900 # 900 is minimum value 
-        
-    - name: Upload benchmark results to S3
-      env:
-        S3_BUCKET: neon-public-benchmark-results
-        S3_PREFIX: benchbase-tpc-c/${{ steps.set-date.outputs.date }}/${{ github.run_id }}/${{ matrix.warehouses }}-warehouses
-      run: |
-        echo "Redacting passwords from configuration files before upload..."
-        
-        # Mask all passwords in XML config files
-        find "${RUNNER_TEMP}/configs" -name "*.xml" -type f -exec sed -i 's|<password>[^<]*</password>|<password>redacted</password>|g' {} \;
-        
-        echo "Uploading benchmark results to s3://${S3_BUCKET}/${S3_PREFIX}/"
-        
-        # Upload the entire benchmark directory recursively
-        aws s3 cp --only-show-errors --recursive "${RUNNER_TEMP}" s3://${S3_BUCKET}/${S3_PREFIX}/
-        
-        echo "Upload completed"
-        
-    - name: Delete Neon Project
-      if: ${{ always() }}
-      uses: ./.github/actions/neon-project-delete
-      with:
-        project_id: ${{ steps.create-neon-project-tpcc.outputs.project_id }}
-        api_key: ${{ secrets.NEON_PRODUCTION_API_KEY_4_BENCHMARKS }} 
-        api_host: console.neon.tech  # production (!)

.github/workflows/benchmarking.yml

@@ -418,7 +418,7 @@ jobs:
       statuses: write
       id-token: write # aws-actions/configure-aws-credentials
     env:
-      PROJECT_ID: ${{ vars.PREWARM_PROJECT_ID }}
+      PGBENCH_SIZE: ${{ vars.PREWARM_PGBENCH_SIZE }}
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
       DEFAULT_PG_VERSION: 17
       TEST_OUTPUT: /tmp/test_output

.github/workflows/build-build-tools-image.yml

@@ -146,9 +146,7 @@ jobs:
         with:
           file: build-tools/Dockerfile
           context: .
-          attests: |
-            type=provenance,mode=max
-            type=sbom,generator=docker.io/docker/buildkit-syft-scanner:1
+          provenance: false
           push: true
           pull: true
           build-args: |

.github/workflows/build_and_test.yml

@@ -634,9 +634,7 @@ jobs:
             DEBIAN_VERSION=bookworm
           secrets: |
             SUBZERO_ACCESS_TOKEN=${{ secrets.CI_ACCESS_TOKEN }}
-          attests: |
-            type=provenance,mode=max
-            type=sbom,generator=docker.io/docker/buildkit-syft-scanner:1
+          provenance: false
           push: true
           pull: true
           file: Dockerfile
@@ -749,9 +747,7 @@ jobs:
             PG_VERSION=${{ matrix.version.pg }}
             BUILD_TAG=${{ needs.meta.outputs.release-tag || needs.meta.outputs.build-tag }}
             DEBIAN_VERSION=${{ matrix.version.debian }}
-          attests: |
-            type=provenance,mode=max
-            type=sbom,generator=docker.io/docker/buildkit-syft-scanner:1
+          provenance: false
           push: true
           pull: true
           file: compute/compute-node.Dockerfile
@@ -770,9 +766,7 @@ jobs:
             PG_VERSION=${{ matrix.version.pg }}
             BUILD_TAG=${{ needs.meta.outputs.release-tag || needs.meta.outputs.build-tag }}
             DEBIAN_VERSION=${{ matrix.version.debian }}
-          attests: |
-            type=provenance,mode=max
-            type=sbom,generator=docker.io/docker/buildkit-syft-scanner:1
+          provenance: false
           push: true
           pull: true
           file: compute/compute-node.Dockerfile

.github/workflows/large_oltp_growth.yml

@@ -2,6 +2,9 @@ name: large oltp growth
 # workflow to grow the reuse branch of large oltp benchmark continuously (about 16 GB per run)
 
 on:
+  # uncomment to run on push for debugging your PR
+  # push:
+  #  branches: [ bodobolero/increase_large_oltp_workload ]
 
   schedule:
     # * is a special character in YAML so you have to quote this string

.github/workflows/pg-clients.yml

@@ -48,20 +48,8 @@ jobs:
     uses: ./.github/workflows/build-build-tools-image.yml
     secrets: inherit
 
-  generate-ch-tmppw:
-    runs-on: ubuntu-22.04
-    outputs:
-      tmp_val: ${{ steps.pwgen.outputs.tmp_val }}
-    steps:
-      - name: Generate a random password
-        id: pwgen
-        run: |
-          set +x
-          p=$(dd if=/dev/random bs=14 count=1 2>/dev/null | base64)
-          echo tmp_val="${p//\//}" >> "${GITHUB_OUTPUT}"
-
   test-logical-replication:
-    needs: [ build-build-tools-image, generate-ch-tmppw ]
+    needs: [ build-build-tools-image ]
     runs-on: ubuntu-22.04
 
     container:
@@ -72,21 +60,16 @@ jobs:
       options: --init --user root
     services:
       clickhouse:
-        image: clickhouse/clickhouse-server:25.6
-        env:
-          CLICKHOUSE_PASSWORD: ${{ needs.generate-ch-tmppw.outputs.tmp_val }}
-          PGSSLCERT: /tmp/postgresql.crt
+        image: clickhouse/clickhouse-server:24.6.3.64
         ports:
           - 9000:9000
           - 8123:8123
       zookeeper:
-        image: quay.io/debezium/zookeeper:3.1.3.Final
+        image: quay.io/debezium/zookeeper:2.7
         ports:
           - 2181:2181
-          - 2888:2888
-          - 3888:3888
       kafka:
-        image: quay.io/debezium/kafka:3.1.3.Final
+        image: quay.io/debezium/kafka:2.7
         env:
           ZOOKEEPER_CONNECT: "zookeeper:2181"
           KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
@@ -96,7 +79,7 @@ jobs:
         ports:
           - 9092:9092
       debezium:
-        image: quay.io/debezium/connect:3.1.3.Final
+        image: quay.io/debezium/connect:2.7
         env:
           BOOTSTRAP_SERVERS: kafka:9092
           GROUP_ID: 1
@@ -142,7 +125,6 @@ jobs:
           aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
         env:
           BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
-          CLICKHOUSE_PASSWORD: ${{ needs.generate-ch-tmppw.outputs.tmp_val }}
 
       - name: Delete Neon Project
         if: always()

.github/workflows/proxy-benchmark.yml

@@ -3,7 +3,7 @@ name: Periodic proxy performance test on unit-perf-aws-arm runners
 on:
   push: # TODO: remove after testing
     branches:
-      - test-proxy-bench # Runs on pushes to test-proxy-bench branch
+      - test-proxy-bench # Runs on pushes to branches starting with test-proxy-bench
   # schedule:
     # * is a special character in YAML so you have to quote this string
     #        ┌───────────── minute (0 - 59)
@@ -32,7 +32,7 @@ jobs:
       statuses: write
       contents: write
       pull-requests: write
-    runs-on: [ self-hosted, unit-perf-aws-arm ]
+    runs-on: [self-hosted, unit-perf-aws-arm]
     timeout-minutes: 60  # 1h timeout
     container:
       image: ghcr.io/neondatabase/build-tools:pinned-bookworm
@@ -55,58 +55,30 @@ jobs:
         {
           echo "PROXY_BENCH_PATH=$PROXY_BENCH_PATH"
           echo "NEON_DIR=${RUNNER_TEMP}/neon"
-          echo "NEON_PROXY_PATH=${RUNNER_TEMP}/neon/bin/proxy"
           echo "TEST_OUTPUT=${PROXY_BENCH_PATH}/test_output"
           echo ""
         } >> "$GITHUB_ENV"
 
-    - name: Cache poetry deps
-      uses: actions/cache@v4
-      with:
-        path: ~/.cache/pypoetry/virtualenvs
-        key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }}
-
-    - name: Install Python deps
-      shell: bash -euxo pipefail {0}
-      run: ./scripts/pysync
-
-    - name: show ulimits
-      shell: bash -euxo pipefail {0}
-      run: |
-        ulimit -a
-
     - name: Run proxy-bench
-      working-directory: ${{ env.PROXY_BENCH_PATH }}
-      run: ./run.sh --with-grafana --bare-metal
+      run: ${PROXY_BENCH_PATH}/run.sh
 
-    - name: Ingest Bench Results
+    - name: Ingest Bench Results # neon repo script
       if: always()
-      working-directory: ${{ env.NEON_DIR }}
       run: |
         mkdir -p $TEST_OUTPUT
         python $NEON_DIR/scripts/proxy_bench_results_ingest.py --out $TEST_OUTPUT
 
     - name: Push Metrics to Proxy perf database
-      shell: bash -euxo pipefail {0}
       if: always()
       env:
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PROXY_TEST_RESULT_CONNSTR }}"
         REPORT_FROM: $TEST_OUTPUT
-      working-directory: ${{ env.NEON_DIR }}
       run: $NEON_DIR/scripts/generate_and_push_perf_report.sh
 
+    - name: Docker cleanup
+      if: always()
+      run: docker compose down
+
     - name: Notify Failure
       if: failure()
-      run: echo "Proxy bench job failed" && exit 1
-
-    - name: Cleanup Test Resources
-      if: always()
-      shell: bash -euxo pipefail {0}
-      run: |
-        # Cleanup the test resources
-        if [[ -d "${TEST_OUTPUT}" ]]; then
-          rm -rf ${TEST_OUTPUT}
-        fi
-        if [[ -d "${PROXY_BENCH_PATH}/test_output" ]]; then
-          rm -rf ${PROXY_BENCH_PATH}/test_output
-        fi
+      run: echo "Proxy bench job failed" && exit 1

.gitignore

@@ -15,6 +15,7 @@ neon.iml
 /.neon
 /integration_tests/.neon
 compaction-suite-results.*
+pgxn/neon/communicator/communicator_bindings.h
 docker-compose/docker-compose-parallel.yml
 
 # Coverage

Cargo.lock

@@ -145,9 +145,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.98"
+version = "1.0.94"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
+checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7"
 dependencies = [
  "backtrace",
 ]
@@ -211,11 +211,11 @@ dependencies = [
 
 [[package]]
 name = "async-lock"
-version = "3.4.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18"
+checksum = "7125e42787d53db9dd54261812ef17e937c95a51e4d291373b670342fa44310c"
 dependencies = [
- "event-listener 5.4.0",
+ "event-listener 4.0.0",
  "event-listener-strategy",
  "pin-project-lite",
 ]
@@ -259,6 +259,17 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8ab6b55fe97976e46f91ddbed8d147d966475dc29b2032757ba47e02376fbc3"
 
+[[package]]
+name = "atomic_enum"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99e1aca718ea7b89985790c94aad72d77533063fe00bc497bb79a7c2dae6a661"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -1294,15 +1305,32 @@ dependencies = [
 
 [[package]]
 name = "communicator"
-version = "0.1.0"
+version = "0.0.0"
 dependencies = [
+ "atomic_enum",
  "axum",
+ "bytes",
  "cbindgen",
+ "clashmap",
  "http 1.3.1",
+ "libc",
  "measured",
+ "metrics",
+ "neon-shmem",
+ "nix 0.30.1",
+ "pageserver_api",
+ "pageserver_client_grpc",
+ "pageserver_page_api",
+ "prometheus",
+ "prost 0.13.5",
+ "strum_macros",
+ "thiserror 1.0.69",
  "tokio",
+ "tokio-pipe",
+ "tonic",
  "tracing",
  "tracing-subscriber",
+ "uring-common",
  "utils",
  "workspace_hack",
 ]
@@ -1388,7 +1416,6 @@ dependencies = [
  "tower-http",
  "tower-otel",
  "tracing",
- "tracing-appender",
  "tracing-opentelemetry",
  "tracing-subscriber",
  "tracing-utils",
@@ -1404,9 +1431,9 @@ dependencies = [
 
 [[package]]
 name = "concurrent-queue"
-version = "2.5.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
+checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1643,9 +1670,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.19"
+version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "crossterm"
@@ -2232,9 +2259,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
 
 [[package]]
 name = "event-listener"
-version = "5.4.0"
+version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae"
+checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae"
 dependencies = [
  "concurrent-queue",
  "parking",
@@ -2243,11 +2270,11 @@ dependencies = [
 
 [[package]]
 name = "event-listener-strategy"
-version = "0.5.4"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93"
+checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3"
 dependencies = [
- "event-listener 5.4.0",
+ "event-listener 4.0.0",
  "pin-project-lite",
 ]
 
@@ -2361,6 +2388,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -2402,9 +2435,9 @@ dependencies = [
 
 [[package]]
 name = "futures"
-version = "0.3.31"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2433,9 +2466,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.31"
+version = "0.3.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -2510,53 +2543,12 @@ dependencies = [
  "slab",
 ]
 
-[[package]]
-name = "gcp_auth"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf67f30198e045a039264c01fb44659ce82402d7771c50938beb41a5ac87733"
-dependencies = [
- "async-trait",
- "base64 0.22.1",
- "bytes",
- "chrono",
- "home",
- "http 1.3.1",
- "http-body-util",
- "hyper 1.4.1",
- "hyper-rustls 0.27.5",
- "hyper-util",
- "ring",
- "rustls-pemfile 2.1.1",
- "serde",
- "serde_json",
- "thiserror 1.0.69",
- "tokio",
- "tracing",
- "tracing-futures",
- "url",
-]
-
 [[package]]
 name = "gen_ops"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "304de19db7028420975a296ab0fcbbc8e69438c4ed254a1e41e2a7f37d5f0e0a"
 
-[[package]]
-name = "generator"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d18470a76cb7f8ff746cf1f7470914f900252ec36bbc40b569d74b1258446827"
-dependencies = [
- "cc",
- "cfg-if",
- "libc",
- "log",
- "rustversion",
- "windows 0.61.3",
-]
-
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -2769,6 +2761,16 @@ version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 
+[[package]]
+name = "hashbrown"
+version = "0.15.4"
+source = "git+https://github.com/quantumish/hashbrown.git?rev=6610e6d#6610e6d2b1f288ef7b0709a3efefbc846395dc5e"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash",
+]
+
 [[package]]
 name = "hashlink"
 version = "0.9.1"
@@ -2867,15 +2869,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "home"
-version = "0.5.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
-dependencies = [
- "windows-sys 0.59.0",
-]
-
 [[package]]
 name = "hostname"
 version = "0.4.0"
@@ -2884,7 +2877,7 @@ checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba"
 dependencies = [
  "cfg-if",
  "libc",
- "windows 0.52.0",
+ "windows",
 ]
 
 [[package]]
@@ -3111,24 +3104,6 @@ dependencies = [
  "tower-service",
 ]
 
-[[package]]
-name = "hyper-rustls"
-version = "0.27.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
-dependencies = [
- "futures-util",
- "http 1.3.1",
- "hyper 1.4.1",
- "hyper-util",
- "rustls 0.23.29",
- "rustls-native-certs 0.8.0",
- "rustls-pki-types",
- "tokio",
- "tokio-rustls 0.26.2",
- "tower-service",
-]
-
 [[package]]
 name = "hyper-timeout"
 version = "0.5.1"
@@ -3173,7 +3148,7 @@ dependencies = [
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "windows-core 0.52.0",
+ "windows-core",
 ]
 
 [[package]]
@@ -3724,19 +3699,6 @@ version = "0.4.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
 
-[[package]]
-name = "loom"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca"
-dependencies = [
- "cfg-if",
- "generator",
- "scoped-tls",
- "tracing",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "lru"
 version = "0.12.3"
@@ -3876,7 +3838,7 @@ dependencies = [
  "prometheus",
  "rand 0.9.1",
  "rand_distr",
- "twox-hash",
+ "twox-hash 1.6.3",
 ]
 
 [[package]]
@@ -3906,16 +3868,6 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
-[[package]]
-name = "mime_guess"
-version = "2.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
-dependencies = [
- "mime",
- "unicase",
-]
-
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -3963,25 +3915,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "moka"
-version = "0.12.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926"
-dependencies = [
- "crossbeam-channel",
- "crossbeam-epoch",
- "crossbeam-utils",
- "loom",
- "parking_lot 0.12.1",
- "portable-atomic",
- "rustc_version",
- "smallvec",
- "tagptr",
- "thiserror 1.0.69",
- "uuid",
-]
-
 [[package]]
 name = "multimap"
 version = "0.8.3"
@@ -3992,15 +3925,21 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 name = "neon-shmem"
 version = "0.1.0"
 dependencies = [
+ "ahash",
+ "criterion",
+ "hashbrown 0.15.4",
  "libc",
  "lock_api",
  "nix 0.30.1",
  "rand 0.9.1",
  "rand_distr",
  "rustc-hash 2.1.1",
+ "seahash",
  "tempfile",
  "thiserror 1.0.69",
+ "twox-hash 2.1.1",
  "workspace_hack",
+ "xxhash-rust",
 ]
 
 [[package]]
@@ -4455,13 +4394,16 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
+ "axum",
  "bytes",
  "camino",
  "clap",
  "futures",
  "hdrhistogram",
+ "http 1.3.1",
  "humantime",
  "humantime-serde",
+ "metrics",
  "pageserver_api",
  "pageserver_client",
  "pageserver_client_grpc",
@@ -4551,6 +4493,7 @@ dependencies = [
  "pageserver_client",
  "pageserver_compaction",
  "pageserver_page_api",
+ "peekable",
  "pem",
  "pin-project-lite",
  "postgres-protocol",
@@ -4564,6 +4507,7 @@ dependencies = [
  "pprof",
  "pq_proto",
  "procfs",
+ "prost 0.13.5",
  "rand 0.9.1",
  "range-set-blaze",
  "regex",
@@ -4600,7 +4544,7 @@ dependencies = [
  "tower 0.5.2",
  "tracing",
  "tracing-utils",
- "twox-hash",
+ "twox-hash 1.6.3",
  "url",
  "utils",
  "uuid",
@@ -4812,7 +4756,7 @@ dependencies = [
  "paste",
  "seq-macro",
  "thrift",
- "twox-hash",
+ "twox-hash 1.6.3",
  "zstd",
  "zstd-sys",
 ]
@@ -4858,6 +4802,15 @@ dependencies = [
  "sha2",
 ]
 
+[[package]]
+name = "peekable"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "225f9651e475709164f871dc2f5724956be59cb9edb055372ffeeab01ec2d20b"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "pem"
 version = "3.0.3"
@@ -5141,6 +5094,8 @@ dependencies = [
  "crc32c",
  "criterion",
  "env_logger",
+ "log",
+ "memoffset 0.9.0",
  "once_cell",
  "postgres",
  "postgres_ffi_types",
@@ -5493,6 +5448,7 @@ dependencies = [
  "futures",
  "gettid",
  "hashbrown 0.14.5",
+ "hashlink",
  "hex",
  "hmac",
  "hostname",
@@ -5514,7 +5470,6 @@ dependencies = [
  "lasso",
  "measured",
  "metrics",
- "moka",
  "once_cell",
  "opentelemetry",
  "ouroboros",
@@ -5581,7 +5536,6 @@ dependencies = [
  "workspace_hack",
  "x509-cert",
  "zerocopy 0.8.24",
- "zeroize",
 ]
 
 [[package]]
@@ -5936,11 +5890,8 @@ dependencies = [
  "bytes",
  "camino",
  "camino-tempfile",
- "chrono",
  "futures",
  "futures-util",
- "gcp_auth",
- "http 1.3.1",
  "http-body-util",
  "http-types",
  "humantime-serde",
@@ -5961,9 +5912,7 @@ dependencies = [
  "tokio-util",
  "toml_edit",
  "tracing",
- "url",
  "utils",
- "uuid",
 ]
 
 [[package]]
@@ -5993,7 +5942,6 @@ dependencies = [
  "js-sys",
  "log",
  "mime",
- "mime_guess",
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
@@ -6535,12 +6483,6 @@ dependencies = [
  "pin-project-lite",
 ]
 
-[[package]]
-name = "scoped-tls"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
-
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -6563,6 +6505,12 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "621e3680f3e07db4c9c2c3fb07c6223ab2fab2e54bd3c04c3ae037990f428c32"
 
+[[package]]
+name = "seahash"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
+
 [[package]]
 name = "sec1"
 version = "0.3.0"
@@ -7390,12 +7338,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "tagptr"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
-
 [[package]]
 name = "tar"
 version = "0.4.40"
@@ -7716,6 +7658,16 @@ dependencies = [
  "syn 2.0.100",
 ]
 
+[[package]]
+name = "tokio-pipe"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784"
+dependencies = [
+ "libc",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-postgres"
 version = "0.7.10"
@@ -8062,12 +8014,11 @@ dependencies = [
 
 [[package]]
 name = "tracing-appender"
-version = "0.2.3"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
+checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e"
 dependencies = [
  "crossbeam-channel",
- "thiserror 1.0.69",
  "time",
  "tracing-subscriber",
 ]
@@ -8103,16 +8054,6 @@ dependencies = [
  "tracing-subscriber",
 ]
 
-[[package]]
-name = "tracing-futures"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
-dependencies = [
- "pin-project",
- "tracing",
-]
-
 [[package]]
 name = "tracing-log"
 version = "0.2.0"
@@ -8263,6 +8204,15 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "twox-hash"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56"
+dependencies = [
+ "rand 0.9.1",
+]
+
 [[package]]
 name = "typed-json"
 version = "0.1.1"
@@ -8288,12 +8238,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "unicase"
-version = "2.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
-
 [[package]]
 name = "unicode-bidi"
 version = "0.3.17"
@@ -8781,32 +8725,10 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
 dependencies = [
- "windows-core 0.52.0",
+ "windows-core",
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "windows"
-version = "0.61.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
-dependencies = [
- "windows-collections",
- "windows-core 0.61.2",
- "windows-future",
- "windows-link",
- "windows-numerics",
-]
-
-[[package]]
-name = "windows-collections"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
-dependencies = [
- "windows-core 0.61.2",
-]
-
 [[package]]
 name = "windows-core"
 version = "0.52.0"
@@ -8816,86 +8738,6 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "windows-core"
-version = "0.61.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
-dependencies = [
- "windows-implement",
- "windows-interface",
- "windows-link",
- "windows-result",
- "windows-strings",
-]
-
-[[package]]
-name = "windows-future"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
-dependencies = [
- "windows-core 0.61.2",
- "windows-link",
- "windows-threading",
-]
-
-[[package]]
-name = "windows-implement"
-version = "0.60.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "windows-interface"
-version = "0.59.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "windows-link"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
-
-[[package]]
-name = "windows-numerics"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
-dependencies = [
- "windows-core 0.61.2",
- "windows-link",
-]
-
-[[package]]
-name = "windows-result"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
-dependencies = [
- "windows-link",
-]
-
-[[package]]
-name = "windows-strings"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
-dependencies = [
- "windows-link",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -8954,15 +8796,6 @@ dependencies = [
  "windows_x86_64_msvc 0.52.6",
 ]
 
-[[package]]
-name = "windows-threading"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
-dependencies = [
- "windows-link",
-]
-
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.48.0"
@@ -9099,8 +8932,7 @@ dependencies = [
  "clap",
  "clap_builder",
  "const-oid",
- "crossbeam-epoch",
- "crossbeam-utils",
+ "criterion",
  "crypto-bigint 0.5.5",
  "der 0.7.8",
  "deranged",
@@ -9143,10 +8975,8 @@ dependencies = [
  "num-iter",
  "num-rational",
  "num-traits",
- "once_cell",
  "p256 0.13.2",
  "parquet",
- "portable-atomic",
  "prettyplease",
  "proc-macro2",
  "prost 0.13.5",
@@ -9252,6 +9082,12 @@ version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd"
 
+[[package]]
+name = "xxhash-rust"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+
 [[package]]
 name = "yansi"
 version = "1.0.1"

Cargo.toml

@@ -46,10 +46,10 @@ members = [
     "libs/proxy/json",
     "libs/proxy/postgres-protocol2",
     "libs/proxy/postgres-types2",
-    "libs/proxy/subzero_core",
     "libs/proxy/tokio-postgres2",
     "endpoint_storage",
     "pgxn/neon/communicator",
+    "proxy/subzero_core",
 ]
 
 [workspace.package]
@@ -93,6 +93,7 @@ clap = { version = "4.0", features = ["derive", "env"] }
 clashmap = { version = "1.0", features = ["raw-api"] }
 comfy-table = "7.1"
 const_format = "0.2"
+crossbeam-utils = "0.8.21"
 crc32c = "0.6"
 diatomic-waker = { version = "0.2.3" }
 either = "1.8"
@@ -135,7 +136,7 @@ lock_api = "0.4.13"
 md5 = "0.7.0"
 measured = { version = "0.0.22", features=["lasso"] }
 measured-process = { version = "0.0.22" }
-moka = { version = "0.12", features = ["sync"] }
+memoffset = "0.9"
 nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket", "signal", "poll"] }
 # Do not update to >= 7.0.0, at least. The update will have a significant impact
 # on compute startup metrics (start_postgres_ms), >= 25% degradation.
@@ -152,6 +153,7 @@ parquet = { version = "53", default-features = false, features = ["zstd"] }
 parquet_derive = "53"
 pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
 pem = "3.0.3"
+peekable = "0.3.0"
 pin-project-lite = "0.2"
 pprof = { version = "0.14", features = ["criterion", "flamegraph", "frame-pointer", "prost-codec"] }
 procfs = "0.16"
@@ -190,6 +192,7 @@ smallvec = "1.11"
 smol_str = { version = "0.2.0", features = ["serde"] }
 socket2 = "0.5"
 spki = "0.7.3"
+spin = "0.9.8"
 strum = "0.26"
 strum_macros = "0.26"
 "subtle"  = "2.5.0"
@@ -201,7 +204,6 @@ thiserror = "1.0"
 tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms"] }
 tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
 tokio = { version = "1.43.1", features = ["macros"] }
-tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
 tokio-io-timeout = "1.2.0"
 tokio-postgres-rustls = "0.12.0"
 tokio-rustls = { version = "0.26.0", default-features = false, features = ["tls12", "ring"]}
@@ -222,7 +224,6 @@ tracing-log = "0.2"
 tracing-opentelemetry = "0.31"
 tracing-serde = "0.2.0"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
-tracing-appender = "0.2.3"
 try-lock = "0.2.5"
 test-log = { version = "0.2.17", default-features = false, features = ["log"] }
 twox-hash = { version = "1.6.3", default-features = false }
@@ -233,15 +234,17 @@ uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
 walkdir = "2.3.2"
 rustls-native-certs = "0.8"
 whoami = "1.5.1"
+zerocopy = { version = "0.8", features = ["derive", "simd"] }
 json-structural-diff = { version = "0.2.0" }
 x509-cert = { version = "0.2.5" }
-zerocopy = { version = "0.8", features = ["derive", "simd"] }
-zeroize = "1.8"
 
 ## TODO replace this with tracing
 env_logger = "0.11"
 log = "0.4"
 
+tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
+uring-common = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
+
 ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
 postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
 postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }

Dockerfile

@@ -78,7 +78,6 @@ WORKDIR /home/nonroot
 ARG GIT_VERSION=local
 ARG BUILD_TAG
 ARG ADDITIONAL_RUSTFLAGS=""
-ARG IO_ALIGNMENT=512
 ENV CARGO_FEATURES="default"
 
 # 3. Build cargo dependencies. Note that this step doesn't depend on anything else than
@@ -102,14 +101,9 @@ COPY --chown=nonroot --from=plan     /home/nonroot/Cargo.lock               Carg
 RUN  --mount=type=secret,uid=1000,id=SUBZERO_ACCESS_TOKEN \
     set -e \
     && if [ -s /run/secrets/SUBZERO_ACCESS_TOKEN ]; then \
-        export CARGO_FEATURES="${CARGO_FEATURES},rest_broker"; \
+        export CARGO_FEATURES="rest_broker"; \
     fi \
-    && if [ "$IO_ALIGNMENT" = "4k" ]; then \
-        export CARGO_FEATURES="${CARGO_FEATURES},io-align-4k"; \
-    elif [ "$IO_ALIGNMENT" = "512" ]; then \
-        export CARGO_FEATURES="${CARGO_FEATURES},io-align-512"; \
-    fi \
-    && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo auditable build \
+    && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo build \
       --features $CARGO_FEATURES \
       --bin pg_sni_router  \
       --bin pageserver  \

README.md

@@ -1,13 +1,13 @@
-[![Neon](https://github.com/user-attachments/assets/fd91da5f-44a9-41c7-9075-36a5b5608083)](https://neon.com)
+[![Neon](https://github.com/neondatabase/neon/assets/11527560/f15a17f0-836e-40c5-b35d-030606a6b660)](https://neon.tech)
 
 
 
 # Neon
 
-Neon is an open-source serverless Postgres database platform. It separates storage and compute and substitutes the PostgreSQL storage layer by redistributing data across a cluster of nodes.
+Neon is a serverless open-source alternative to AWS Aurora Postgres. It separates storage and compute and substitutes the PostgreSQL storage layer by redistributing data across a cluster of nodes.
 
 ## Quick start
-Try the [Neon Free Tier](https://neon.com/signup) to create a serverless Postgres instance. Then connect to it with your preferred Postgres client (psql, dbeaver, etc) or use the online [SQL Editor](https://neon.com/docs/get-started-with-neon/query-with-neon-sql-editor/). See [Connect from any application](https://neon.com/docs/connect/connect-from-any-app/) for connection instructions.
+Try the [Neon Free Tier](https://neon.tech/github) to create a serverless Postgres instance. Then connect to it with your preferred Postgres client (psql, dbeaver, etc) or use the online [SQL Editor](https://neon.tech/docs/get-started-with-neon/query-with-neon-sql-editor/). See [Connect from any application](https://neon.tech/docs/connect/connect-from-any-app/) for connection instructions.
 
 Alternatively, compile and run the project [locally](#running-local-installation).
 
@@ -301,8 +301,8 @@ See also README files in some source directories, and `rustdoc` style documentat
 
 Other resources:
 
-- [SELECT 'Hello, World'](https://neon.com/blog/hello-world/): Blog post by Nikita Shamgunov on the high level architecture
-- [Architecture decisions in Neon](https://neon.com/blog/architecture-decisions-in-neon/): Blog post by Heikki Linnakangas
+- [SELECT 'Hello, World'](https://neon.tech/blog/hello-world/): Blog post by Nikita Shamgunov on the high level architecture
+- [Architecture decisions in Neon](https://neon.tech/blog/architecture-decisions-in-neon/): Blog post by Heikki Linnakangas
 - [Neon: Serverless PostgreSQL!](https://www.youtube.com/watch?v=rES0yzeERns): Presentation on storage system by Heikki Linnakangas in the CMU Database Group seminar series
 
 ### Postgres-specific terms

build-tools/Dockerfile

@@ -39,13 +39,13 @@ COPY build-tools/patches/pgcopydbv017.patch /pgcopydbv017.patch
 
 RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
         set -e && \
-        apt-get update && \
-        apt-get install -y --no-install-recommends \
+        apt update && \
+        apt install -y --no-install-recommends \
         ca-certificates wget gpg && \
         wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg && \
         echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \
         apt-get update && \
-        apt-get install -y --no-install-recommends \
+        apt install -y --no-install-recommends \
         build-essential \
         autotools-dev \
         libedit-dev \
@@ -89,7 +89,8 @@ RUN useradd -ms /bin/bash nonroot -b /home
 # Use strict mode for bash to catch errors early
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
-RUN mkdir -p /pgcopydb/{bin,lib} && \    
+RUN mkdir -p /pgcopydb/bin && \
+    mkdir -p /pgcopydb/lib && \
     chmod -R 755 /pgcopydb && \
     chown -R nonroot:nonroot /pgcopydb
 
@@ -105,8 +106,8 @@ RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
 # 'gdb' is included so that we get backtraces of core dumps produced in
 # regression tests
 RUN set -e \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends \
+    && apt update \
+    && apt install -y \
         autoconf \
         automake \
         bison \
@@ -182,22 +183,22 @@ RUN curl -sL "https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/
 ENV LLVM_VERSION=20
 RUN curl -fsSL 'https://apt.llvm.org/llvm-snapshot.gpg.key' | apt-key add - \
     && echo "deb http://apt.llvm.org/${DEBIAN_VERSION}/ llvm-toolchain-${DEBIAN_VERSION}-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.stable.list \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends clang-${LLVM_VERSION} llvm-${LLVM_VERSION} \
+    && apt update \
+    && apt install -y clang-${LLVM_VERSION} llvm-${LLVM_VERSION} \
     && bash -c 'for f in /usr/bin/clang*-${LLVM_VERSION} /usr/bin/llvm*-${LLVM_VERSION}; do ln -s "${f}" "${f%-${LLVM_VERSION}}"; done' \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install node
 ENV NODE_VERSION=24
 RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \
-    && apt-get install -y --no-install-recommends nodejs \
+    && apt install -y nodejs \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install docker
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
     && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian ${DEBIAN_VERSION} stable" > /etc/apt/sources.list.d/docker.list \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends docker-ce docker-ce-cli \
+    && apt update \
+    && apt install -y docker-ce docker-ce-cli \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Configure sudo & docker
@@ -214,11 +215,12 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "aws
 # Mold: A Modern Linker
 ENV MOLD_VERSION=v2.37.1
 RUN set -e \
-    && git clone -b "${MOLD_VERSION}" --depth 1 https://github.com/rui314/mold.git \
+    && git clone https://github.com/rui314/mold.git \
     && mkdir mold/build \
-    && cd mold/build \    
+    && cd mold/build \
+    && git checkout ${MOLD_VERSION} \
     && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++ .. \
-    && cmake --build . -j "$(nproc)" \
+    && cmake --build . -j $(nproc) \
     && cmake --install . \
     && cd .. \
     && rm -rf mold
@@ -252,7 +254,7 @@ ENV ICU_VERSION=67.1
 ENV ICU_PREFIX=/usr/local/icu
 
 # Download and build static ICU
-RUN wget -O "/tmp/libicu-${ICU_VERSION}.tgz" https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION//./-}/icu4c-${ICU_VERSION//./_}-src.tgz && \
+RUN wget -O /tmp/libicu-${ICU_VERSION}.tgz https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION//./-}/icu4c-${ICU_VERSION//./_}-src.tgz && \
     echo "94a80cd6f251a53bd2a997f6f1b5ac6653fe791dfab66e1eb0227740fb86d5dc /tmp/libicu-${ICU_VERSION}.tgz" | sha256sum --check && \
     mkdir /tmp/icu && \
     pushd /tmp/icu && \
@@ -263,7 +265,8 @@ RUN wget -O "/tmp/libicu-${ICU_VERSION}.tgz" https://github.com/unicode-org/icu/
     make install && \
     popd && \
     rm -rf icu && \
-    rm -f /tmp/libicu-${ICU_VERSION}.tgz
+    rm -f /tmp/libicu-${ICU_VERSION}.tgz && \
+    popd
 
 # Switch to nonroot user
 USER nonroot:nonroot
@@ -276,19 +279,19 @@ ENV PYTHON_VERSION=3.11.12 \
     PYENV_ROOT=/home/nonroot/.pyenv \
     PATH=/home/nonroot/.pyenv/shims:/home/nonroot/.pyenv/bin:/home/nonroot/.poetry/bin:$PATH
 RUN set -e \
-    && cd "$HOME" \
+    && cd $HOME \
     && curl -sSO https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer \
     && chmod +x pyenv-installer \
     && ./pyenv-installer \
     && export PYENV_ROOT=/home/nonroot/.pyenv \
     && export PATH="$PYENV_ROOT/bin:$PATH" \
     && export PATH="$PYENV_ROOT/shims:$PATH" \
-    && pyenv install "${PYTHON_VERSION}" \
-    && pyenv global "${PYTHON_VERSION}" \
+    && pyenv install ${PYTHON_VERSION} \
+    && pyenv global ${PYTHON_VERSION} \
     && python --version \
-    && pip install --no-cache-dir --upgrade pip \
+    && pip install --upgrade pip \
     && pip --version \
-    && pip install --no-cache-dir pipenv wheel poetry
+    && pip install pipenv wheel poetry
 
 # Switch to nonroot user (again)
 USER nonroot:nonroot
@@ -299,7 +302,6 @@ WORKDIR /home/nonroot
 ENV RUSTC_VERSION=1.88.0
 ENV RUSTUP_HOME="/home/nonroot/.rustup"
 ENV PATH="/home/nonroot/.cargo/bin:${PATH}"
-ARG CARGO_AUDITABLE_VERSION=0.7.0
 ARG RUSTFILT_VERSION=0.2.1
 ARG CARGO_HAKARI_VERSION=0.9.36
 ARG CARGO_DENY_VERSION=0.18.2
@@ -315,16 +317,14 @@ RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux
     . "$HOME/.cargo/env" && \
     cargo --version && rustup --version && \
     rustup component add llvm-tools rustfmt clippy && \
-    cargo install cargo-auditable           --locked --version "${CARGO_AUDITABLE_VERSION}" && \
-    cargo auditable install cargo-auditable --locked --version "${CARGO_AUDITABLE_VERSION}" --force && \
-    cargo auditable install rustfilt                 --version "${RUSTFILT_VERSION}" && \
-    cargo auditable install cargo-hakari    --locked --version "${CARGO_HAKARI_VERSION}" && \
-    cargo auditable install cargo-deny      --locked --version "${CARGO_DENY_VERSION}" && \
-    cargo auditable install cargo-hack      --locked --version "${CARGO_HACK_VERSION}" && \
-    cargo auditable install cargo-nextest   --locked --version "${CARGO_NEXTEST_VERSION}" && \
-    cargo auditable install cargo-chef      --locked --version "${CARGO_CHEF_VERSION}" && \
-    cargo auditable install diesel_cli      --locked --version "${CARGO_DIESEL_CLI_VERSION}" \
-                                            --features postgres-bundled --no-default-features && \
+    cargo install rustfilt      --locked --version ${RUSTFILT_VERSION} && \
+    cargo install cargo-hakari  --locked --version ${CARGO_HAKARI_VERSION} && \
+    cargo install cargo-deny    --locked --version ${CARGO_DENY_VERSION} && \
+    cargo install cargo-hack    --locked --version ${CARGO_HACK_VERSION} && \
+    cargo install cargo-nextest --locked --version ${CARGO_NEXTEST_VERSION} && \
+    cargo install cargo-chef    --locked --version ${CARGO_CHEF_VERSION} && \
+    cargo install diesel_cli    --locked --version ${CARGO_DIESEL_CLI_VERSION} \
+                                --features postgres-bundled --no-default-features && \
     rm -rf /home/nonroot/.cargo/registry && \
     rm -rf /home/nonroot/.cargo/git
 

build-tools/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "name": "build-tools",
       "devDependencies": {
-        "@redocly/cli": "1.34.5",
+        "@redocly/cli": "1.34.4",
         "@sourcemeta/jsonschema": "10.0.0"
       }
     },
@@ -472,9 +472,9 @@
       }
     },
     "node_modules/@redocly/cli": {
-      "version": "1.34.5",
-      "resolved": "https://registry.npmjs.org/@redocly/cli/-/cli-1.34.5.tgz",
-      "integrity": "sha512-5IEwxs7SGP5KEXjBKLU8Ffdz9by/KqNSeBk6YUVQaGxMXK//uYlTJIPntgUXbo1KAGG2d2q2XF8y4iFz6qNeiw==",
+      "version": "1.34.4",
+      "resolved": "https://registry.npmjs.org/@redocly/cli/-/cli-1.34.4.tgz",
+      "integrity": "sha512-seH/GgrjSB1EeOsgJ/4Ct6Jk2N7sh12POn/7G8UQFARMyUMJpe1oHtBwT2ndfp4EFCpgBAbZ/82Iw6dwczNxEA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -484,14 +484,14 @@
         "@opentelemetry/sdk-trace-node": "1.26.0",
         "@opentelemetry/semantic-conventions": "1.27.0",
         "@redocly/config": "^0.22.0",
-        "@redocly/openapi-core": "1.34.5",
-        "@redocly/respect-core": "1.34.5",
+        "@redocly/openapi-core": "1.34.4",
+        "@redocly/respect-core": "1.34.4",
         "abort-controller": "^3.0.0",
         "chokidar": "^3.5.1",
         "colorette": "^1.2.0",
         "core-js": "^3.32.1",
         "dotenv": "16.4.7",
-        "form-data": "^4.0.4",
+        "form-data": "^4.0.0",
         "get-port-please": "^3.0.1",
         "glob": "^7.1.6",
         "handlebars": "^4.7.6",
@@ -522,9 +522,9 @@
       "license": "MIT"
     },
     "node_modules/@redocly/openapi-core": {
-      "version": "1.34.5",
-      "resolved": "https://registry.npmjs.org/@redocly/openapi-core/-/openapi-core-1.34.5.tgz",
-      "integrity": "sha512-0EbE8LRbkogtcCXU7liAyC00n9uNG9hJ+eMyHFdUsy9lB/WGqnEBgwjA9q2cyzAVcdTkQqTBBU1XePNnN3OijA==",
+      "version": "1.34.4",
+      "resolved": "https://registry.npmjs.org/@redocly/openapi-core/-/openapi-core-1.34.4.tgz",
+      "integrity": "sha512-hf53xEgpXIgWl3b275PgZU3OTpYh1RoD2LHdIfQ1JzBNTWsiNKczTEsI/4Tmh2N1oq9YcphhSMyk3lDh85oDjg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -544,21 +544,21 @@
       }
     },
     "node_modules/@redocly/respect-core": {
-      "version": "1.34.5",
-      "resolved": "https://registry.npmjs.org/@redocly/respect-core/-/respect-core-1.34.5.tgz",
-      "integrity": "sha512-GheC/g/QFztPe9UA9LamooSplQuy9pe0Yr8XGTqkz0ahivLDl7svoy/LSQNn1QH3XGtLKwFYMfTwFR2TAYyh5Q==",
+      "version": "1.34.4",
+      "resolved": "https://registry.npmjs.org/@redocly/respect-core/-/respect-core-1.34.4.tgz",
+      "integrity": "sha512-MitKyKyQpsizA4qCVv+MjXL4WltfhFQAoiKiAzrVR1Kusro3VhYb6yJuzoXjiJhR0ukLP5QOP19Vcs7qmj9dZg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@faker-js/faker": "^7.6.0",
         "@redocly/ajv": "8.11.2",
-        "@redocly/openapi-core": "1.34.5",
+        "@redocly/openapi-core": "1.34.4",
         "better-ajv-errors": "^1.2.0",
         "colorette": "^2.0.20",
         "concat-stream": "^2.0.0",
         "cookie": "^0.7.2",
         "dotenv": "16.4.7",
-        "form-data": "^4.0.4",
+        "form-data": "4.0.0",
         "jest-diff": "^29.3.1",
         "jest-matcher-utils": "^29.3.1",
         "js-yaml": "4.1.0",
@@ -582,6 +582,21 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@redocly/respect-core/node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/@sinclair/typebox": {
       "version": "0.27.8",
       "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz",
@@ -1330,9 +1345,9 @@
       "license": "MIT"
     },
     "node_modules/form-data": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
-      "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.3.tgz",
+      "integrity": "sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {

build-tools/package.json

@@ -2,7 +2,7 @@
   "name": "build-tools",
   "private": true,
   "devDependencies": {
-    "@redocly/cli": "1.34.5",
+    "@redocly/cli": "1.34.4",
     "@sourcemeta/jsonschema": "10.0.0"
   }
 }

compute/compute-node.Dockerfile

@@ -133,7 +133,7 @@ RUN case $DEBIAN_VERSION in \
       # Install newer version (3.25) from backports.
       # libstdc++-10-dev is required for plv8
       bullseye) \
-        echo "deb http://archive.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list; \
+        echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list; \
         VERSION_INSTALLS="cmake/bullseye-backports cmake-data/bullseye-backports libstdc++-10-dev"; \
       ;; \
       # Version-specific installs for Bookworm (PG17):

compute/etc/sql_exporter/checkpoints_req.17.sql

@@ -1 +1 @@
-SELECT num_requested AS checkpoints_req FROM pg_catalog.pg_stat_checkpointer;
+SELECT num_requested AS checkpoints_req FROM pg_stat_checkpointer;

compute/etc/sql_exporter/checkpoints_req.sql

@@ -1 +1 @@
-SELECT checkpoints_req FROM pg_catalog.pg_stat_bgwriter;
+SELECT checkpoints_req FROM pg_stat_bgwriter;

compute/etc/sql_exporter/checkpoints_timed.sql

@@ -1 +1 @@
-SELECT checkpoints_timed FROM pg_catalog.pg_stat_bgwriter;
+SELECT checkpoints_timed FROM pg_stat_bgwriter;

compute/etc/sql_exporter/compute_backpressure_throttling_seconds_total.sql

@@ -1 +1 @@
-SELECT (neon.backpressure_throttling_time()::pg_catalog.float8 / 1000000) AS throttled;
+SELECT (neon.backpressure_throttling_time()::float8 / 1000000) AS throttled;

compute/etc/sql_exporter/compute_current_lsn.sql

@@ -1,4 +1,4 @@
 SELECT CASE
-  WHEN pg_catalog.pg_is_in_recovery() THEN (pg_catalog.pg_last_wal_replay_lsn() - '0/0')::pg_catalog.FLOAT8
-  ELSE (pg_catalog.pg_current_wal_lsn() - '0/0')::pg_catalog.FLOAT8
+  WHEN pg_catalog.pg_is_in_recovery() THEN (pg_last_wal_replay_lsn() - '0/0')::FLOAT8
+  ELSE (pg_current_wal_lsn() - '0/0')::FLOAT8
 END AS lsn;

compute/etc/sql_exporter/compute_logical_snapshot_files.sql

@@ -1,7 +1,7 @@
 SELECT
-  (SELECT setting FROM pg_catalog.pg_settings WHERE name = 'neon.timeline_id') AS timeline_id,
+  (SELECT setting FROM pg_settings WHERE name = 'neon.timeline_id') AS timeline_id,
   -- Postgres creates temporary snapshot files of the form %X-%X.snap.%d.tmp.
   -- These temporary snapshot files are renamed to the actual snapshot files
   -- after they are completely built. We only WAL-log the completely built
   -- snapshot files
-  (SELECT COUNT(*) FROM pg_catalog.pg_ls_dir('pg_logical/snapshots') AS name WHERE name LIKE '%.snap') AS num_logical_snapshot_files;
+  (SELECT COUNT(*) FROM pg_ls_dir('pg_logical/snapshots') AS name WHERE name LIKE '%.snap') AS num_logical_snapshot_files;

compute/etc/sql_exporter/compute_logical_snapshots_bytes.15.sql

@@ -1,7 +1,7 @@
 SELECT
-  (SELECT pg_catalog.current_setting('neon.timeline_id')) AS timeline_id,
+  (SELECT current_setting('neon.timeline_id')) AS timeline_id,
   -- Postgres creates temporary snapshot files of the form %X-%X.snap.%d.tmp.
   -- These temporary snapshot files are renamed to the actual snapshot files
   -- after they are completely built. We only WAL-log the completely built
   -- snapshot files
-  (SELECT COALESCE(pg_catalog.sum(size), 0) FROM pg_catalog.pg_ls_logicalsnapdir() WHERE name LIKE '%.snap') AS logical_snapshots_bytes;
+  (SELECT COALESCE(sum(size), 0) FROM pg_ls_logicalsnapdir() WHERE name LIKE '%.snap') AS logical_snapshots_bytes;

compute/etc/sql_exporter/compute_logical_snapshots_bytes.sql

@@ -1,9 +1,9 @@
 SELECT
-  (SELECT setting FROM pg_catalog.pg_settings WHERE name = 'neon.timeline_id') AS timeline_id,
+  (SELECT setting FROM pg_settings WHERE name = 'neon.timeline_id') AS timeline_id,
   -- Postgres creates temporary snapshot files of the form %X-%X.snap.%d.tmp.
   -- These temporary snapshot files are renamed to the actual snapshot files
   -- after they are completely built. We only WAL-log the completely built
   -- snapshot files
-  (SELECT COALESCE(pg_catalog.sum((pg_catalog.pg_stat_file('pg_logical/snapshots/' || name, missing_ok => true)).size), 0)
-   FROM (SELECT * FROM pg_catalog.pg_ls_dir('pg_logical/snapshots') WHERE pg_ls_dir LIKE '%.snap') AS name
+  (SELECT COALESCE(sum((pg_stat_file('pg_logical/snapshots/' || name, missing_ok => true)).size), 0)
+    FROM (SELECT * FROM pg_ls_dir('pg_logical/snapshots') WHERE pg_ls_dir LIKE '%.snap') AS name
   ) AS logical_snapshots_bytes;

compute/etc/sql_exporter/compute_max_connections.sql

@@ -1 +1 @@
-SELECT pg_catalog.current_setting('max_connections') AS max_connections;
+SELECT current_setting('max_connections') as max_connections;

compute/etc/sql_exporter/compute_pg_oldest_frozen_xid_age.sql

@@ -1,4 +1,4 @@
 SELECT datname database_name,
-   pg_catalog.age(datfrozenxid) frozen_xid_age
-FROM pg_catalog.pg_database
+  age(datfrozenxid) frozen_xid_age
+FROM pg_database
 ORDER BY frozen_xid_age DESC LIMIT 10;

compute/etc/sql_exporter/compute_pg_oldest_mxid_age.sql

@@ -1,4 +1,4 @@
 SELECT datname database_name,
-  pg_catalog.mxid_age(datminmxid) min_mxid_age
-FROM pg_catalog.pg_database
+  mxid_age(datminmxid) min_mxid_age
+FROM pg_database
 ORDER BY min_mxid_age DESC LIMIT 10;

compute/etc/sql_exporter/compute_receive_lsn.sql

@@ -1,4 +1,4 @@
 SELECT CASE
-  WHEN pg_catalog.pg_is_in_recovery() THEN (pg_catalog.pg_last_wal_receive_lsn() - '0/0')::pg_catalog.FLOAT8
+  WHEN pg_catalog.pg_is_in_recovery() THEN (pg_last_wal_receive_lsn() - '0/0')::FLOAT8
   ELSE 0
 END AS lsn;

compute/etc/sql_exporter/compute_subscriptions_count.sql

@@ -1 +1 @@
-SELECT subenabled::pg_catalog.text AS enabled, pg_catalog.count(*) AS subscriptions_count FROM pg_catalog.pg_subscription GROUP BY subenabled;
+SELECT subenabled::text AS enabled, count(*) AS subscriptions_count FROM pg_subscription GROUP BY subenabled;

compute/etc/sql_exporter/connection_counts.sql

@@ -1 +1 @@
-SELECT datname, state, pg_catalog.count(*) AS count FROM pg_catalog.pg_stat_activity WHERE state <> '' GROUP BY datname, state;
+SELECT datname, state, count(*) AS count FROM pg_stat_activity WHERE state <> '' GROUP BY datname, state;

compute/etc/sql_exporter/db_total_size.sql

@@ -1,5 +1,5 @@
-SELECT pg_catalog.sum(pg_catalog.pg_database_size(datname)) AS total
-FROM pg_catalog.pg_database
+SELECT sum(pg_database_size(datname)) AS total
+FROM pg_database
 -- Ignore invalid databases, as we will likely have problems with
 -- getting their size from the Pageserver.
 WHERE datconnlimit != -2;

compute/etc/sql_exporter/lfc_approximate_working_set_size_windows.autoscaling.sql

@@ -3,6 +3,6 @@
 -- minutes.
 
 SELECT
-  x::pg_catalog.text AS duration_seconds,
+  x::text as duration_seconds,
   neon.approximate_working_set_size_seconds(x) AS size
 FROM (SELECT generate_series * 60 AS x FROM generate_series(1, 60)) AS t (x);

compute/etc/sql_exporter/lfc_approximate_working_set_size_windows.sql

@@ -3,6 +3,6 @@
 
 SELECT
   x AS duration,
-  neon.approximate_working_set_size_seconds(extract('epoch' FROM x::pg_catalog.interval)::pg_catalog.int4) AS size FROM (
+  neon.approximate_working_set_size_seconds(extract('epoch' FROM x::interval)::int) AS size FROM (
     VALUES ('5m'), ('15m'), ('1h')
   ) AS t (x);

compute/etc/sql_exporter/lfc_cache_size_limit.sql

@@ -1 +1 @@
-SELECT pg_catalog.pg_size_bytes(pg_catalog.current_setting('neon.file_cache_size_limit')) AS lfc_cache_size_limit;
+SELECT pg_size_bytes(current_setting('neon.file_cache_size_limit')) AS lfc_cache_size_limit;

compute/etc/sql_exporter/logical_slot_restart_lsn.sql

@@ -1,3 +1,3 @@
-SELECT slot_name, (restart_lsn - '0/0')::pg_catalog.FLOAT8 AS restart_lsn
-FROM pg_catalog.pg_replication_slots
+SELECT slot_name, (restart_lsn - '0/0')::FLOAT8 as restart_lsn
+FROM pg_replication_slots
 WHERE slot_type = 'logical';

compute/etc/sql_exporter/max_cluster_size.sql

@@ -1 +1 @@
-SELECT setting::pg_catalog.int4 AS max_cluster_size FROM pg_catalog.pg_settings WHERE name = 'neon.max_cluster_size';
+SELECT setting::int AS max_cluster_size FROM pg_settings WHERE name = 'neon.max_cluster_size';

compute/etc/sql_exporter/pg_stats_userdb.sql

@@ -1,13 +1,13 @@
 -- We export stats for 10 non-system databases. Without this limit it is too
 -- easy to abuse the system by creating lots of databases.
 
-SELECT pg_catalog.pg_database_size(datname) AS db_size,
+SELECT pg_database_size(datname) AS db_size,
   deadlocks,
   tup_inserted AS inserted,
   tup_updated AS updated,
   tup_deleted AS deleted,
   datname
-FROM pg_catalog.pg_stat_database
+FROM pg_stat_database
 WHERE datname IN (
   SELECT datname FROM pg_database
   -- Ignore invalid databases, as we will likely have problems with

compute/etc/sql_exporter/replication_delay_bytes.sql

@@ -3,4 +3,4 @@
 -- replay LSN may have advanced past the receive LSN we are using for the
 -- calculation.
 
-SELECT GREATEST(0, pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_last_wal_receive_lsn(), pg_catalog.pg_last_wal_replay_lsn())) AS replication_delay_bytes;
+SELECT GREATEST(0, pg_wal_lsn_diff(pg_last_wal_receive_lsn(), pg_last_wal_replay_lsn())) AS replication_delay_bytes;

compute/etc/sql_exporter/replication_delay_seconds.sql

@@ -1,5 +1,5 @@
 SELECT
   CASE
-    WHEN pg_catalog.pg_last_wal_receive_lsn() = pg_catalog.pg_last_wal_replay_lsn() THEN 0
-    ELSE GREATEST(0, EXTRACT (EPOCH FROM pg_catalog.now() - pg_catalog.pg_last_xact_replay_timestamp()))
+    WHEN pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn() THEN 0
+    ELSE GREATEST(0, EXTRACT (EPOCH FROM now() - pg_last_xact_replay_timestamp()))
   END AS replication_delay_seconds;

compute/etc/sql_exporter/retained_wal.sql

@@ -1,10 +1,10 @@
 SELECT
   slot_name,
-  pg_catalog.pg_wal_lsn_diff(
+  pg_wal_lsn_diff(
     CASE
-      WHEN pg_catalog.pg_is_in_recovery() THEN pg_catalog.pg_last_wal_replay_lsn()
-      ELSE pg_catalog.pg_current_wal_lsn()
+      WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn()
+      ELSE pg_current_wal_lsn()
     END,
-    restart_lsn)::pg_catalog.FLOAT8 AS retained_wal
-FROM pg_catalog.pg_replication_slots
+    restart_lsn)::FLOAT8 AS retained_wal
+FROM pg_replication_slots
 WHERE active = false;

compute/etc/sql_exporter/wal_is_lost.sql

@@ -4,4 +4,4 @@ SELECT
     WHEN wal_status = 'lost' THEN 1
     ELSE 0
   END AS wal_is_lost
-FROM pg_catalog.pg_replication_slots;
+FROM pg_replication_slots;

compute/patches/pg_repack.patch

@@ -1,11 +1,5 @@
-commit 5eb393810cf7c7bafa4e394dad2e349e2a8cb2cb
-Author: Alexey Masterov <alexey.masterov@databricks.com>
-Date:   Mon Jul 28 18:11:02 2025 +0200
-
-    Patch for pg_repack
-
 diff --git a/regress/Makefile b/regress/Makefile
-index bf6edcb..110e734 100644
+index bf6edcb..89b4c7f 100644
 --- a/regress/Makefile
 +++ b/regress/Makefile
 @@ -17,7 +17,7 @@ INTVERSION := $(shell echo $$(($$(echo $(VERSION).0 | sed 's/\([[:digit:]]\{1,\}
@@ -13,36 +7,18 @@ index bf6edcb..110e734 100644
  #
  
 -REGRESS := init-extension repack-setup repack-run error-on-invalid-idx no-error-on-invalid-idx after-schema repack-check nosuper tablespace get_order_by trigger
-+REGRESS := init-extension noautovacuum repack-setup repack-run error-on-invalid-idx no-error-on-invalid-idx after-schema repack-check nosuper get_order_by trigger autovacuum
++REGRESS := init-extension repack-setup repack-run error-on-invalid-idx no-error-on-invalid-idx after-schema repack-check nosuper get_order_by trigger
  
  USE_PGXS = 1	# use pgxs if not in contrib directory
  PGXS := $(shell $(PG_CONFIG) --pgxs)
-diff --git a/regress/expected/autovacuum.out b/regress/expected/autovacuum.out
-new file mode 100644
-index 0000000..e7f2363
---- /dev/null
-+++ b/regress/expected/autovacuum.out
-@@ -0,0 +1,7 @@
-+ALTER SYSTEM SET autovacuum='on';
-+SELECT pg_reload_conf();
-+ pg_reload_conf 
-+----------------
-+ t
-+(1 row)
-+
-diff --git a/regress/expected/noautovacuum.out b/regress/expected/noautovacuum.out
-new file mode 100644
-index 0000000..fc7978e
---- /dev/null
-+++ b/regress/expected/noautovacuum.out
-@@ -0,0 +1,7 @@
-+ALTER SYSTEM SET autovacuum='off';
-+SELECT pg_reload_conf();
-+ pg_reload_conf 
-+----------------
-+ t
-+(1 row)
-+
+diff --git a/regress/expected/init-extension.out b/regress/expected/init-extension.out
+index 9f2e171..f6e4f8d 100644
+--- a/regress/expected/init-extension.out
++++ b/regress/expected/init-extension.out
+@@ -1,3 +1,2 @@
+ SET client_min_messages = warning;
+ CREATE EXTENSION pg_repack;
+-RESET client_min_messages;
 diff --git a/regress/expected/nosuper.out b/regress/expected/nosuper.out
 index 8d0a94e..63b68bf 100644
 --- a/regress/expected/nosuper.out
@@ -74,22 +50,14 @@ index 8d0a94e..63b68bf 100644
  INFO: repacking table "public.tbl_cluster"
  ERROR: query failed: ERROR:  current transaction is aborted, commands ignored until end of transaction block
  DETAIL: query was: RESET lock_timeout
-diff --git a/regress/sql/autovacuum.sql b/regress/sql/autovacuum.sql
-new file mode 100644
-index 0000000..a8eda63
---- /dev/null
-+++ b/regress/sql/autovacuum.sql
-@@ -0,0 +1,2 @@
-+ALTER SYSTEM SET autovacuum='on';
-+SELECT pg_reload_conf();
-diff --git a/regress/sql/noautovacuum.sql b/regress/sql/noautovacuum.sql
-new file mode 100644
-index 0000000..13d4836
---- /dev/null
-+++ b/regress/sql/noautovacuum.sql
-@@ -0,0 +1,2 @@
-+ALTER SYSTEM SET autovacuum='off';
-+SELECT pg_reload_conf();
+diff --git a/regress/sql/init-extension.sql b/regress/sql/init-extension.sql
+index 9f2e171..f6e4f8d 100644
+--- a/regress/sql/init-extension.sql
++++ b/regress/sql/init-extension.sql
+@@ -1,3 +1,2 @@
+ SET client_min_messages = warning;
+ CREATE EXTENSION pg_repack;
+-RESET client_min_messages;
 diff --git a/regress/sql/nosuper.sql b/regress/sql/nosuper.sql
 index 072f0fa..dbe60f8 100644
 --- a/regress/sql/nosuper.sql

compute/vm-image-spec-bookworm.yaml

@@ -26,13 +26,7 @@ commands:
   - name: postgres-exporter
     user: nobody
     sysvInitAction: respawn
-    # Turn off database collector (`--no-collector.database`), we don't use `pg_database_size_bytes` metric anyway, see
-    # https://github.com/neondatabase/flux-fleet/blob/5e19b3fd897667b70d9a7ad4aa06df0ca22b49ff/apps/base/compute-metrics/scrape-compute-pg-exporter-neon.yaml#L29
-    # but it's enabled by default and it doesn't filter out invalid databases, see
-    # https://github.com/prometheus-community/postgres_exporter/blob/06a553c8166512c9d9c5ccf257b0f9bba8751dbc/collector/pg_database.go#L67
-    # so if it hits one, it starts spamming logs
-    #   ERROR:  [NEON_SMGR] [reqid d9700000018] could not read db size of db 705302 from page server at lsn 5/A2457EB0
-    shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --no-collector.database --config.file=/etc/postgres_exporter.yml'
+    shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --config.file=/etc/postgres_exporter.yml'
   - name: pgbouncer-exporter
     user: postgres
     sysvInitAction: respawn

compute/vm-image-spec-bullseye.yaml

@@ -26,13 +26,7 @@ commands:
   - name: postgres-exporter
     user: nobody
     sysvInitAction: respawn
-    # Turn off database collector (`--no-collector.database`), we don't use `pg_database_size_bytes` metric anyway, see
-    # https://github.com/neondatabase/flux-fleet/blob/5e19b3fd897667b70d9a7ad4aa06df0ca22b49ff/apps/base/compute-metrics/scrape-compute-pg-exporter-neon.yaml#L29
-    # but it's enabled by default and it doesn't filter out invalid databases, see
-    # https://github.com/prometheus-community/postgres_exporter/blob/06a553c8166512c9d9c5ccf257b0f9bba8751dbc/collector/pg_database.go#L67
-    # so if it hits one, it starts spamming logs
-    #   ERROR:  [NEON_SMGR] [reqid d9700000018] could not read db size of db 705302 from page server at lsn 5/A2457EB0
-    shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --no-collector.database --config.file=/etc/postgres_exporter.yml'
+    shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter pgaudit.log=none" /bin/postgres_exporter --config.file=/etc/postgres_exporter.yml'
   - name: pgbouncer-exporter
     user: postgres
     sysvInitAction: respawn

compute_tools/Cargo.toml

@@ -62,7 +62,6 @@ tokio-stream.workspace = true
 tonic.workspace = true
 tower-otel.workspace = true
 tracing.workspace = true
-tracing-appender.workspace = true
 tracing-opentelemetry.workspace = true
 tracing-subscriber.workspace = true
 tracing-utils.workspace = true

compute_tools/README.md

@@ -52,14 +52,8 @@ stateDiagram-v2
   Init --> Running : Started Postgres
   Running --> TerminationPendingFast : Requested termination
   Running --> TerminationPendingImmediate : Requested termination
-  Running --> ConfigurationPending : Received a /configure request with spec
-  Running --> RefreshConfigurationPending : Received a /refresh_configuration request, compute node will pull a new spec and reconfigure
-  RefreshConfigurationPending --> RefreshConfiguration: Received compute spec and started configuration
-  RefreshConfiguration --> Running : Compute has been re-configured
-  RefreshConfiguration --> RefreshConfigurationPending : Configuration failed and to be retried
   TerminationPendingFast --> Terminated compute with 30s delay for cplane to inspect status
   TerminationPendingImmediate --> Terminated : Terminated compute immediately
-  Failed --> RefreshConfigurationPending : Received a /refresh_configuration request
   Failed --> [*] : Compute exited
   Terminated --> [*] : Compute exited
 ```

compute_tools/src/bin/compute_ctl.rs

@@ -49,10 +49,9 @@ use compute_tools::compute::{
     BUILD_TAG, ComputeNode, ComputeNodeParams, forward_termination_signal,
 };
 use compute_tools::extension_server::get_pg_version_string;
+use compute_tools::logger::*;
 use compute_tools::params::*;
-use compute_tools::pg_isready::get_pg_isready_bin;
 use compute_tools::spec::*;
-use compute_tools::{hadron_metrics, installed_extensions, logger::*};
 use rlimit::{Resource, setrlimit};
 use signal_hook::consts::{SIGINT, SIGQUIT, SIGTERM};
 use signal_hook::iterator::Signals;
@@ -82,15 +81,6 @@ struct Cli {
     #[arg(long, default_value_t = 3081)]
     pub internal_http_port: u16,
 
-    /// Backwards-compatible --http-port for Hadron deployments. Functionally the
-    /// same as --external-http-port.
-    #[arg(
-        long,
-        conflicts_with = "external_http_port",
-        conflicts_with = "internal_http_port"
-    )]
-    pub http_port: Option<u16>,
-
     #[arg(short = 'D', long, value_name = "DATADIR")]
     pub pgdata: String,
 
@@ -190,26 +180,6 @@ impl Cli {
     }
 }
 
-// Hadron helpers to get compatible compute_ctl http ports from Cli. The old `--http-port`
-// arg is used and acts the same as `--external-http-port`. The internal http port is defined
-// to be http_port + 1. Hadron runs in the dblet environment which uses the host network, so
-// we need to be careful with the ports to choose.
-fn get_external_http_port(cli: &Cli) -> u16 {
-    if cli.lakebase_mode {
-        return cli.http_port.unwrap_or(cli.external_http_port);
-    }
-    cli.external_http_port
-}
-fn get_internal_http_port(cli: &Cli) -> u16 {
-    if cli.lakebase_mode {
-        return cli
-            .http_port
-            .map(|p| p + 1)
-            .unwrap_or(cli.internal_http_port);
-    }
-    cli.internal_http_port
-}
-
 fn main() -> Result<()> {
     let cli = Cli::parse();
 
@@ -224,28 +194,15 @@ fn main() -> Result<()> {
         .build()?;
     let _rt_guard = runtime.enter();
 
-    let mut log_dir = None;
-    if cli.lakebase_mode {
-        log_dir = std::env::var("COMPUTE_CTL_LOG_DIRECTORY").ok();
-    }
-
-    let (tracing_provider, _file_logs_guard) = init(cli.dev, log_dir)?;
+    let tracing_provider = init(cli.dev)?;
 
     // enable core dumping for all child processes
     setrlimit(Resource::CORE, rlimit::INFINITY, rlimit::INFINITY)?;
 
-    if cli.lakebase_mode {
-        installed_extensions::initialize_metrics();
-        hadron_metrics::initialize_metrics();
-    }
-
     let connstr = Url::parse(&cli.connstr).context("cannot parse connstr as a URL")?;
 
     let config = get_config(&cli)?;
 
-    let external_http_port = get_external_http_port(&cli);
-    let internal_http_port = get_internal_http_port(&cli);
-
     let compute_node = ComputeNode::new(
         ComputeNodeParams {
             compute_id: cli.compute_id,
@@ -254,8 +211,8 @@ fn main() -> Result<()> {
             pgdata: cli.pgdata.clone(),
             pgbin: cli.pgbin.clone(),
             pgversion: get_pg_version_string(&cli.pgbin),
-            external_http_port,
-            internal_http_port,
+            external_http_port: cli.external_http_port,
+            internal_http_port: cli.internal_http_port,
             remote_ext_base_url: cli.remote_ext_base_url.clone(),
             resize_swap_on_bind: cli.resize_swap_on_bind,
             set_disk_quota_for_fs: cli.set_disk_quota_for_fs,
@@ -269,31 +226,20 @@ fn main() -> Result<()> {
                 cli.installed_extensions_collection_interval,
             )),
             pg_init_timeout: cli.pg_init_timeout.map(Duration::from_secs),
-            pg_isready_bin: get_pg_isready_bin(&cli.pgbin),
-            instance_id: std::env::var("INSTANCE_ID").ok(),
             lakebase_mode: cli.lakebase_mode,
-            build_tag: BUILD_TAG.to_string(),
-            control_plane_uri: cli.control_plane_uri,
-            config_path_test_only: cli.config,
         },
         config,
     )?;
 
-    let exit_code = compute_node.run().context("running compute node")?;
+    let exit_code = compute_node.run()?;
 
     scenario.teardown();
 
     deinit_and_exit(tracing_provider, exit_code);
 }
 
-fn init(
-    dev_mode: bool,
-    log_dir: Option<String>,
-) -> Result<(
-    Option<tracing_utils::Provider>,
-    Option<tracing_appender::non_blocking::WorkerGuard>,
-)> {
-    let (provider, file_logs_guard) = init_tracing_and_logging(DEFAULT_LOG_LEVEL, &log_dir)?;
+fn init(dev_mode: bool) -> Result<Option<tracing_utils::Provider>> {
+    let provider = init_tracing_and_logging(DEFAULT_LOG_LEVEL)?;
 
     let mut signals = Signals::new([SIGINT, SIGTERM, SIGQUIT])?;
     thread::spawn(move || {
@@ -304,7 +250,7 @@ fn init(
 
     info!("compute build_tag: {}", &BUILD_TAG.to_string());
 
-    Ok((provider, file_logs_guard))
+    Ok(provider)
 }
 
 fn get_config(cli: &Cli) -> Result<ComputeConfig> {

compute_tools/src/checker.rs

@@ -24,9 +24,9 @@ pub async fn check_writability(compute: &ComputeNode) -> Result<()> {
     });
 
     let query = "
-    INSERT INTO public.health_check VALUES (1, pg_catalog.now())
+    INSERT INTO health_check VALUES (1, now())
         ON CONFLICT (id) DO UPDATE
-         SET updated_at = pg_catalog.now();";
+         SET updated_at = now();";
 
     match client.simple_query(query).await {
         Result::Ok(result) => {

compute_tools/src/compute.rs

@@ -6,8 +6,8 @@ use compute_api::responses::{
     LfcPrewarmState, PromoteState, TlsConfig,
 };
 use compute_api::spec::{
-    ComputeAudit, ComputeFeature, ComputeMode, ComputeSpec, ExtVersion, GenericOption,
-    PageserverConnectionInfo, PageserverProtocol, PgIdent, Role,
+    ComputeAudit, ComputeFeature, ComputeMode, ComputeSpec, ExtVersion, PageserverConnectionInfo,
+    PageserverProtocol, PageserverShardConnectionInfo, PageserverShardInfo, PgIdent,
 };
 use futures::StreamExt;
 use futures::future::join_all;
@@ -22,7 +22,6 @@ use postgres::NoTls;
 use postgres::error::SqlState;
 use remote_storage::{DownloadError, RemotePath};
 use std::collections::{HashMap, HashSet};
-use std::ffi::OsString;
 use std::os::unix::fs::{PermissionsExt, symlink};
 use std::path::Path;
 use std::process::{Command, Stdio};
@@ -32,23 +31,18 @@ use std::sync::{Arc, Condvar, Mutex, RwLock};
 use std::time::{Duration, Instant};
 use std::{env, fs};
 use tokio::{spawn, sync::watch, task::JoinHandle, time};
-use tokio_util::sync::CancellationToken;
 use tracing::{Instrument, debug, error, info, instrument, warn};
 use url::Url;
-use utils::backoff::{
-    DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS, exponential_backoff_duration,
-};
 use utils::id::{TenantId, TimelineId};
 use utils::lsn::Lsn;
 use utils::measured_stream::MeasuredReader;
 use utils::pid_file;
-use utils::shard::{ShardIndex, ShardNumber, ShardStripeSize};
+use utils::shard::{ShardCount, ShardIndex, ShardNumber};
 
 use crate::configurator::launch_configurator;
 use crate::disk_quota::set_disk_quota;
-use crate::hadron_metrics::COMPUTE_ATTACHED;
 use crate::installed_extensions::get_installed_extensions;
-use crate::logger::{self, startup_context_from_env};
+use crate::logger::startup_context_from_env;
 use crate::lsn_lease::launch_lsn_lease_bg_task_for_static;
 use crate::metrics::COMPUTE_CTL_UP;
 use crate::monitor::launch_monitor;
@@ -120,17 +114,11 @@ pub struct ComputeNodeParams {
 
     /// Interval for installed extensions collection
     pub installed_extensions_collection_interval: Arc<AtomicU64>,
-    /// Hadron instance ID of the compute node.
-    pub instance_id: Option<String>,
+
     /// Timeout of PG compute startup in the Init state.
     pub pg_init_timeout: Option<Duration>,
-    // Path to the `pg_isready` binary.
-    pub pg_isready_bin: String,
-    pub lakebase_mode: bool,
 
-    pub build_tag: String,
-    pub control_plane_uri: Option<String>,
-    pub config_path_test_only: Option<OsString>,
+    pub lakebase_mode: bool,
 }
 
 type TaskHandle = Mutex<Option<JoinHandle<()>>>;
@@ -196,7 +184,6 @@ pub struct ComputeState {
     pub startup_span: Option<tracing::span::Span>,
 
     pub lfc_prewarm_state: LfcPrewarmState,
-    pub lfc_prewarm_token: CancellationToken,
     pub lfc_offload_state: LfcOffloadState,
 
     /// WAL flush LSN that is set after terminating Postgres and syncing safekeepers if
@@ -222,7 +209,6 @@ impl ComputeState {
             lfc_offload_state: LfcOffloadState::default(),
             terminate_flush_lsn: None,
             promote_state: None,
-            lfc_prewarm_token: CancellationToken::new(),
         }
     }
 
@@ -302,6 +288,72 @@ impl ParsedSpec {
     }
 }
 
+/// Extract PageserverConnectionInfo from a comma-separated list of libpq connection strings.
+///
+/// This is used for backwards-compatilibity, to parse the legacye `pageserver_connstr`
+/// field in the compute spec, or the 'neon.pageserver_connstring' GUC. Nowadays, the
+/// 'pageserver_connection_info' field should be used instead.
+fn extract_pageserver_conninfo_from_connstr(
+    connstr: &str,
+    stripe_size: Option<u32>,
+) -> Result<PageserverConnectionInfo, anyhow::Error> {
+    let shard_infos: Vec<_> = connstr
+        .split(',')
+        .map(|connstr| PageserverShardInfo {
+            pageservers: vec![PageserverShardConnectionInfo {
+                id: None,
+                libpq_url: Some(connstr.to_string()),
+                grpc_url: None,
+            }],
+        })
+        .collect();
+
+    match shard_infos.len() {
+        0 => anyhow::bail!("empty connection string"),
+        1 => {
+            // We assume that if there's only connection string, it means "unsharded",
+            // rather than a sharded system with just a single shard. The latter is
+            // possible in principle, but we never do it.
+            let shard_count = ShardCount::unsharded();
+            let only_shard = shard_infos.first().unwrap().clone();
+            let shards = vec![(ShardIndex::unsharded(), only_shard)];
+            Ok(PageserverConnectionInfo {
+                shard_count,
+                stripe_size: None,
+                shards: shards.into_iter().collect(),
+                prefer_protocol: PageserverProtocol::Libpq,
+            })
+        }
+        n => {
+            if stripe_size.is_none() {
+                anyhow::bail!("{n} shards but no stripe_size");
+            }
+            let shard_count = ShardCount(n.try_into()?);
+            let shards = shard_infos
+                .into_iter()
+                .enumerate()
+                .map(|(idx, shard_info)| {
+                    (
+                        ShardIndex {
+                            shard_count,
+                            shard_number: ShardNumber(
+                                idx.try_into().expect("shard number fits in u8"),
+                            ),
+                        },
+                        shard_info,
+                    )
+                })
+                .collect();
+            Ok(PageserverConnectionInfo {
+                shard_count,
+                stripe_size,
+                shards,
+                prefer_protocol: PageserverProtocol::Libpq,
+            })
+        }
+    }
+}
+
 impl TryFrom<ComputeSpec> for ParsedSpec {
     type Error = anyhow::Error;
     fn try_from(spec: ComputeSpec) -> Result<Self, anyhow::Error> {
@@ -315,7 +367,7 @@ impl TryFrom<ComputeSpec> for ParsedSpec {
         let mut pageserver_conninfo = spec.pageserver_connection_info.clone();
         if pageserver_conninfo.is_none() {
             if let Some(pageserver_connstr_field) = &spec.pageserver_connstring {
-                pageserver_conninfo = Some(PageserverConnectionInfo::from_connstr(
+                pageserver_conninfo = Some(extract_pageserver_conninfo_from_connstr(
                     pageserver_connstr_field,
                     spec.shard_stripe_size,
                 )?);
@@ -325,12 +377,12 @@ impl TryFrom<ComputeSpec> for ParsedSpec {
             if let Some(guc) = spec.cluster.settings.find("neon.pageserver_connstring") {
                 let stripe_size = if let Some(guc) = spec.cluster.settings.find("neon.stripe_size")
                 {
-                    Some(ShardStripeSize(u32::from_str(&guc)?))
+                    Some(u32::from_str(&guc)?)
                 } else {
                     None
                 };
                 pageserver_conninfo =
-                    Some(PageserverConnectionInfo::from_connstr(&guc, stripe_size)?);
+                    Some(extract_pageserver_conninfo_from_connstr(&guc, stripe_size)?);
             }
         }
         let pageserver_conninfo = pageserver_conninfo.ok_or(anyhow::anyhow!(
@@ -442,130 +494,6 @@ struct StartVmMonitorResult {
     vm_monitor: Option<JoinHandle<Result<()>>>,
 }
 
-// BEGIN_HADRON
-/// This function creates roles that are used by Databricks.
-/// These roles are not needs to be botostrapped at PG Compute provisioning time.
-/// The auth method for these roles are configured in databricks_pg_hba.conf in universe repository.
-pub(crate) fn create_databricks_roles() -> Vec<String> {
-    let roles = vec![
-        // Role for prometheus_stats_exporter
-        Role {
-            name: "databricks_monitor".to_string(),
-            // This uses "local" connection and auth method for that is "trust", so no password is needed.
-            encrypted_password: None,
-            options: Some(vec![GenericOption {
-                name: "IN ROLE pg_monitor".to_string(),
-                value: None,
-                vartype: "string".to_string(),
-            }]),
-        },
-        // Role for brickstore control plane
-        Role {
-            name: "databricks_control_plane".to_string(),
-            // Certificate user does not need password.
-            encrypted_password: None,
-            options: Some(vec![GenericOption {
-                name: "SUPERUSER".to_string(),
-                value: None,
-                vartype: "string".to_string(),
-            }]),
-        },
-        // Role for brickstore httpgateway.
-        Role {
-            name: "databricks_gateway".to_string(),
-            // Certificate user does not need password.
-            encrypted_password: None,
-            options: None,
-        },
-    ];
-
-    roles
-        .into_iter()
-        .map(|role| {
-            let query = format!(
-                r#"
-                DO $$
-                    BEGIN
-                        IF NOT EXISTS (
-                            SELECT FROM pg_catalog.pg_roles WHERE rolname = '{}')
-                        THEN
-                            CREATE ROLE {} {};
-                        END IF;
-                    END
-                $$;"#,
-                role.name,
-                role.name.pg_quote(),
-                role.to_pg_options(),
-            );
-            query
-        })
-        .collect()
-}
-
-/// Databricks-specific environment variables to be passed to the `postgres` sub-process.
-pub struct DatabricksEnvVars {
-    /// The Databricks "endpoint ID" of the compute instance. Used by `postgres` to check
-    /// the token scopes of internal auth tokens.
-    pub endpoint_id: String,
-    /// Hostname of the Databricks workspace URL this compute instance belongs to.
-    /// Used by postgres to verify Databricks PAT tokens.
-    pub workspace_host: String,
-
-    pub lakebase_mode: bool,
-}
-
-impl DatabricksEnvVars {
-    pub fn new(
-        compute_spec: &ComputeSpec,
-        compute_id: Option<&String>,
-        instance_id: Option<String>,
-        lakebase_mode: bool,
-    ) -> Self {
-        let endpoint_id = if let Some(instance_id) = instance_id {
-            // Use instance_id as endpoint_id if it is set. This code path is for PuPr model.
-            instance_id
-        } else {
-            // Use compute_id as endpoint_id if instance_id is not set. The code path is for PrPr model.
-            // compute_id is a string format of "{endpoint_id}/{compute_idx}"
-            // endpoint_id is a uuid. We only need to pass down endpoint_id to postgres.
-            // Panics if compute_id is not set or not in the expected format.
-            compute_id.unwrap().split('/').next().unwrap().to_string()
-        };
-        let workspace_host = compute_spec
-            .databricks_settings
-            .as_ref()
-            .map(|s| s.databricks_workspace_host.clone())
-            .unwrap_or("".to_string());
-        Self {
-            endpoint_id,
-            workspace_host,
-            lakebase_mode,
-        }
-    }
-
-    /// Constants for the names of Databricks-specific postgres environment variables.
-    const DATABRICKS_ENDPOINT_ID_ENVVAR: &'static str = "DATABRICKS_ENDPOINT_ID";
-    const DATABRICKS_WORKSPACE_HOST_ENVVAR: &'static str = "DATABRICKS_WORKSPACE_HOST";
-
-    /// Convert DatabricksEnvVars to a list of string pairs that can be passed as env vars. Consumes `self`.
-    pub fn to_env_var_list(self) -> Vec<(String, String)> {
-        if !self.lakebase_mode {
-            // In neon env, we don't need to pass down the env vars to postgres.
-            return vec![];
-        }
-        vec![
-            (
-                Self::DATABRICKS_ENDPOINT_ID_ENVVAR.to_string(),
-                self.endpoint_id.clone(),
-            ),
-            (
-                Self::DATABRICKS_WORKSPACE_HOST_ENVVAR.to_string(),
-                self.workspace_host.clone(),
-            ),
-        ]
-    }
-}
-
 impl ComputeNode {
     pub fn new(params: ComputeNodeParams, config: ComputeConfig) -> Result<Self> {
         let connstr = params.connstr.as_str();
@@ -589,7 +517,7 @@ impl ComputeNode {
         // that can affect `compute_ctl` and prevent it from properly configuring the database schema.
         // Unset them via connection string options before connecting to the database.
         // N.B. keep it in sync with `ZENITH_OPTIONS` in `get_maintenance_client()`.
-        const EXTRA_OPTIONS: &str = "-c role=cloud_admin -c default_transaction_read_only=off -c search_path='' -c statement_timeout=0 -c pgaudit.log=none";
+        const EXTRA_OPTIONS: &str = "-c role=cloud_admin -c default_transaction_read_only=off -c search_path=public -c statement_timeout=0 -c pgaudit.log=none";
         let options = match conn_conf.get_options() {
             // Allow the control plane to override any options set by the
             // compute
@@ -602,11 +530,7 @@ impl ComputeNode {
         let mut new_state = ComputeState::new();
         if let Some(spec) = config.spec {
             let pspec = ParsedSpec::try_from(spec).map_err(|msg| anyhow::anyhow!(msg))?;
-            if params.lakebase_mode {
-                ComputeNode::set_spec(&params, &mut new_state, pspec);
-            } else {
-                new_state.pspec = Some(pspec);
-            }
+            new_state.pspec = Some(pspec);
         }
 
         Ok(ComputeNode {
@@ -651,7 +575,6 @@ impl ComputeNode {
             port: this.params.external_http_port,
             config: this.compute_ctl_config.clone(),
             compute_id: this.params.compute_id.clone(),
-            instance_id: this.params.instance_id.clone(),
         }
         .launch(&this);
 
@@ -1204,14 +1127,7 @@ impl ComputeNode {
         // If it is something different then create_dir() will error out anyway.
         let pgdata = &self.params.pgdata;
         let _ok = fs::remove_dir_all(pgdata);
-        if self.params.lakebase_mode {
-            // Ignore creation errors if the directory already exists (e.g. mounting it ahead of time).
-            // If it is something different then PG startup will error out anyway.
-            let _ok = fs::create_dir(pgdata);
-        } else {
-            fs::create_dir(pgdata)?;
-        }
-
+        fs::create_dir(pgdata)?;
         fs::set_permissions(pgdata, fs::Permissions::from_mode(0o700))?;
 
         Ok(())
@@ -1268,10 +1184,22 @@ impl ComputeNode {
             shard_number: ShardNumber(0),
             shard_count: spec.pageserver_conninfo.shard_count,
         };
-        let shard0_url = spec
+        let shard0 = spec
             .pageserver_conninfo
-            .shard_url(ShardNumber(0), PageserverProtocol::Grpc)?
-            .to_owned();
+            .shards
+            .get(&shard0_index)
+            .ok_or_else(|| {
+                anyhow::anyhow!("shard connection info missing for shard {}", shard0_index)
+            })?;
+        let pageserver = shard0
+            .pageservers
+            .first()
+            .expect("must have at least one pageserver");
+        let shard0_url = pageserver
+            .grpc_url
+            .clone()
+            .expect("no grpc_url for shard 0");
+
         let (reader, connected) = tokio::runtime::Handle::current().block_on(async move {
             let mut client = page_api::Client::connect(
                 shard0_url,
@@ -1309,10 +1237,26 @@ impl ComputeNode {
     /// Fetches a basebackup via libpq. The connstring must use postgresql://. Returns the timestamp
     /// when the connection was established, and the (compressed) size of the basebackup.
     fn try_get_basebackup_libpq(&self, spec: &ParsedSpec, lsn: Lsn) -> Result<(Instant, usize)> {
-        let shard0_connstr = spec
+        let shard0_index = ShardIndex {
+            shard_number: ShardNumber(0),
+            shard_count: spec.pageserver_conninfo.shard_count,
+        };
+        let shard0 = spec
             .pageserver_conninfo
-            .shard_url(ShardNumber(0), PageserverProtocol::Libpq)?;
-        let mut config = postgres::Config::from_str(shard0_connstr)?;
+            .shards
+            .get(&shard0_index)
+            .ok_or_else(|| {
+                anyhow::anyhow!("shard connection info missing for shard {}", shard0_index)
+            })?;
+        let pageserver = shard0
+            .pageservers
+            .first()
+            .expect("must have at least one pageserver");
+        let shard0_connstr = pageserver
+            .libpq_url
+            .clone()
+            .expect("no libpq_url for shard 0");
+        let mut config = postgres::Config::from_str(&shard0_connstr)?;
 
         // Use the storage auth token from the config file, if given.
         // Note: this overrides any password set in the connection string.
@@ -1560,41 +1504,6 @@ impl ComputeNode {
         Ok(lsn)
     }
 
-    fn sync_safekeepers_with_retries(&self, storage_auth_token: Option<String>) -> Result<Lsn> {
-        let max_retries = 5;
-        let mut attempts = 0;
-        loop {
-            let result = self.sync_safekeepers(storage_auth_token.clone());
-            match &result {
-                Ok(_) => {
-                    if attempts > 0 {
-                        tracing::info!("sync_safekeepers succeeded after {attempts} retries");
-                    }
-                    return result;
-                }
-                Err(e) if attempts < max_retries => {
-                    tracing::info!(
-                        "sync_safekeepers failed, will retry (attempt {attempts}): {e:#}"
-                    );
-                }
-                Err(err) => {
-                    tracing::warn!(
-                        "sync_safekeepers still failed after {attempts} retries, giving up: {err:?}"
-                    );
-                    return result;
-                }
-            }
-            // sleep and retry
-            let backoff = exponential_backoff_duration(
-                attempts,
-                DEFAULT_BASE_BACKOFF_SECONDS,
-                DEFAULT_MAX_BACKOFF_SECONDS,
-            );
-            std::thread::sleep(backoff);
-            attempts += 1;
-        }
-    }
-
     /// Do all the preparations like PGDATA directory creation, configuration,
     /// safekeepers sync, basebackup, etc.
     #[instrument(skip_all)]
@@ -1604,8 +1513,6 @@ impl ComputeNode {
         let pgdata_path = Path::new(&self.params.pgdata);
 
         let tls_config = self.tls_config(&pspec.spec);
-        let databricks_settings = spec.databricks_settings.as_ref();
-        let postgres_port = self.params.connstr.port();
 
         // Remove/create an empty pgdata directory and put configuration there.
         self.create_pgdata()?;
@@ -1613,11 +1520,8 @@ impl ComputeNode {
             pgdata_path,
             &self.params,
             &pspec.spec,
-            postgres_port,
             self.params.internal_http_port,
             tls_config,
-            databricks_settings,
-            self.params.lakebase_mode,
         )?;
 
         // Syncing safekeepers is only safe with primary nodes: if a primary
@@ -1630,7 +1534,7 @@ impl ComputeNode {
                     lsn
                 } else {
                     info!("starting safekeepers syncing");
-                    self.sync_safekeepers_with_retries(pspec.storage_auth_token.clone())
+                    self.sync_safekeepers(pspec.storage_auth_token.clone())
                         .with_context(|| "failed to sync safekeepers")?
                 };
                 info!("safekeepers synced at LSN {}", lsn);
@@ -1649,28 +1553,8 @@ impl ComputeNode {
         self.get_basebackup(compute_state, lsn)
             .with_context(|| format!("failed to get basebackup@{lsn}"))?;
 
-        if let Some(settings) = databricks_settings {
-            copy_tls_certificates(
-                &settings.pg_compute_tls_settings.key_file,
-                &settings.pg_compute_tls_settings.cert_file,
-                pgdata_path,
-            )?;
-
-            // Update pg_hba.conf received with basebackup including additional databricks settings.
-            update_pg_hba(pgdata_path, Some(&settings.databricks_pg_hba))?;
-            update_pg_ident(pgdata_path, Some(&settings.databricks_pg_ident))?;
-        } else {
-            // Update pg_hba.conf received with basebackup.
-            update_pg_hba(pgdata_path, None)?;
-        }
-
-        if let Some(databricks_settings) = spec.databricks_settings.as_ref() {
-            copy_tls_certificates(
-                &databricks_settings.pg_compute_tls_settings.key_file,
-                &databricks_settings.pg_compute_tls_settings.cert_file,
-                pgdata_path,
-            )?;
-        }
+        // Update pg_hba.conf received with basebackup.
+        update_pg_hba(pgdata_path, None)?;
 
         // Place pg_dynshmem under /dev/shm. This allows us to use
         // 'dynamic_shared_memory_type = mmap' so that the files are placed in
@@ -1711,7 +1595,7 @@ impl ComputeNode {
         // symlink doesn't affect anything.
         //
         // See https://github.com/neondatabase/autoscaling/issues/800
-        std::fs::remove_dir_all(pgdata_path.join("pg_dynshmem"))?;
+        std::fs::remove_dir(pgdata_path.join("pg_dynshmem"))?;
         symlink("/dev/shm/", pgdata_path.join("pg_dynshmem"))?;
 
         match spec.mode {
@@ -1726,12 +1610,6 @@ impl ComputeNode {
 
     /// Start and stop a postgres process to warm up the VM for startup.
     pub fn prewarm_postgres_vm_memory(&self) -> Result<()> {
-        if self.params.lakebase_mode {
-            // We are running in Hadron mode. Disabling this prewarming step for now as it could run
-            // into dblet port conflicts and also doesn't add much value with our current infra.
-            info!("Skipping postgres prewarming in Hadron mode");
-            return Ok(());
-        }
         info!("prewarming VM memory");
 
         // Create pgdata
@@ -1789,36 +1667,14 @@ impl ComputeNode {
     pub fn start_postgres(&self, storage_auth_token: Option<String>) -> Result<PostgresHandle> {
         let pgdata_path = Path::new(&self.params.pgdata);
 
-        let env_vars: Vec<(String, String)> = if self.params.lakebase_mode {
-            let databricks_env_vars = {
-                let state = self.state.lock().unwrap();
-                let spec = &state.pspec.as_ref().unwrap().spec;
-                DatabricksEnvVars::new(
-                    spec,
-                    Some(&self.params.compute_id),
-                    self.params.instance_id.clone(),
-                    self.params.lakebase_mode,
-                )
-            };
-
-            info!(
-                "Starting Postgres for databricks endpoint id: {}",
-                &databricks_env_vars.endpoint_id
-            );
-
-            let mut env_vars = databricks_env_vars.to_env_var_list();
-            env_vars.extend(storage_auth_token.map(|t| ("NEON_AUTH_TOKEN".to_string(), t)));
-            env_vars
-        } else if let Some(storage_auth_token) = &storage_auth_token {
-            vec![("NEON_AUTH_TOKEN".to_owned(), storage_auth_token.to_owned())]
-        } else {
-            vec![]
-        };
-
         // Run postgres as a child process.
         let mut pg = maybe_cgexec(&self.params.pgbin)
             .args(["-D", &self.params.pgdata])
-            .envs(env_vars)
+            .envs(if let Some(storage_auth_token) = &storage_auth_token {
+                vec![("NEON_AUTH_TOKEN", storage_auth_token)]
+            } else {
+                vec![]
+            })
             .stderr(Stdio::piped())
             .spawn()
             .expect("cannot start postgres process");
@@ -1925,7 +1781,7 @@ impl ComputeNode {
 
                     // It doesn't matter what were the options before, here we just want
                     // to connect and create a new superuser role.
-                    const ZENITH_OPTIONS: &str = "-c role=zenith_admin -c default_transaction_read_only=off -c search_path='' -c statement_timeout=0";
+                    const ZENITH_OPTIONS: &str = "-c role=zenith_admin -c default_transaction_read_only=off -c search_path=public -c statement_timeout=0";
                     zenith_admin_conf.options(ZENITH_OPTIONS);
 
                     let mut client =
@@ -1970,15 +1826,7 @@ impl ComputeNode {
     /// Do initial configuration of the already started Postgres.
     #[instrument(skip_all)]
     pub fn apply_config(&self, compute_state: &ComputeState) -> Result<()> {
-        let mut conf = self.get_tokio_conn_conf(Some("compute_ctl:apply_config"));
-
-        if self.params.lakebase_mode {
-            // Set a 2-minute statement_timeout for the session applying config. The individual SQL statements
-            // used in apply_spec_sql() should not take long (they are just creating users and installing
-            // extensions). If any of them are stuck for an extended period of time it usually indicates a
-            // pageserver connectivity problem and we should bail out.
-            conf.options("-c statement_timeout=2min");
-        }
+        let conf = self.get_tokio_conn_conf(Some("compute_ctl:apply_config"));
 
         let conf = Arc::new(conf);
         let spec = Arc::new(
@@ -2040,34 +1888,6 @@ impl ComputeNode {
         Ok::<(), anyhow::Error>(())
     }
 
-    // Signal to the configurator to refresh the configuration by pulling a new spec from the HCC.
-    // Note that this merely triggers a notification on a condition variable the configurator thread
-    // waits on. The configurator thread (in configurator.rs) pulls the new spec from the HCC and
-    // applies it.
-    pub async fn signal_refresh_configuration(&self) -> Result<()> {
-        let states_allowing_configuration_refresh = [
-            ComputeStatus::Running,
-            ComputeStatus::Failed,
-            ComputeStatus::RefreshConfigurationPending,
-        ];
-
-        let mut state = self.state.lock().expect("state lock poisoned");
-        if states_allowing_configuration_refresh.contains(&state.status) {
-            state.status = ComputeStatus::RefreshConfigurationPending;
-            self.state_changed.notify_all();
-            Ok(())
-        } else if state.status == ComputeStatus::Init {
-            // If the compute is in Init state, we can't refresh the configuration immediately,
-            // but we should be able to do that soon.
-            Ok(())
-        } else {
-            Err(anyhow::anyhow!(
-                "Cannot refresh compute configuration in state {:?}",
-                state.status
-            ))
-        }
-    }
-
     // Wrapped this around `pg_ctl reload`, but right now we don't use
     // `pg_ctl` for start / stop.
     #[instrument(skip_all)]
@@ -2129,16 +1949,12 @@ impl ComputeNode {
 
         // Write new config
         let pgdata_path = Path::new(&self.params.pgdata);
-        let postgres_port = self.params.connstr.port();
         config::write_postgres_conf(
             pgdata_path,
             &self.params,
             &spec,
-            postgres_port,
             self.params.internal_http_port,
             tls_config,
-            spec.databricks_settings.as_ref(),
-            self.params.lakebase_mode,
         )?;
 
         self.pg_reload_conf()?;
@@ -2244,8 +2060,6 @@ impl ComputeNode {
                             // wait
                             ComputeStatus::Init
                             | ComputeStatus::Configuration
-                            | ComputeStatus::RefreshConfiguration
-                            | ComputeStatus::RefreshConfigurationPending
                             | ComputeStatus::Empty => {
                                 state = self.state_changed.wait(state).unwrap();
                             }
@@ -2296,17 +2110,7 @@ impl ComputeNode {
     pub fn check_for_core_dumps(&self) -> Result<()> {
         let core_dump_dir = match std::env::consts::OS {
             "macos" => Path::new("/cores/"),
-            // BEGIN HADRON
-            // NB: Read core dump files from a fixed location outside of
-            // the data directory since `compute_ctl` wipes the data directory
-            // across container restarts.
-            _ => {
-                if self.params.lakebase_mode {
-                    Path::new("/databricks/logs/brickstore")
-                } else {
-                    Path::new(&self.params.pgdata)
-                }
-            } // END HADRON
+            _ => Path::new(&self.params.pgdata),
         };
 
         // Collect core dump paths if any
@@ -2380,13 +2184,13 @@ impl ComputeNode {
         let result = client
             .simple_query(
                 "SELECT
-    pg_catalog.row_to_json(pss)
+    row_to_json(pg_stat_statements)
 FROM
-    public.pg_stat_statements pss
+    pg_stat_statements
 WHERE
-    pss.userid != 'cloud_admin'::pg_catalog.regrole::pg_catalog.oid
+    userid != 'cloud_admin'::regrole::oid
 ORDER BY
-    (pss.mean_exec_time + pss.mean_plan_time) DESC
+    (mean_exec_time + mean_plan_time) DESC
 LIMIT 100",
             )
             .await;
@@ -2514,11 +2318,11 @@ LIMIT 100",
 
         // check the role grants first - to gracefully handle read-replicas.
         let select = "SELECT privilege_type
-            FROM pg_catalog.pg_namespace
-                JOIN LATERAL (SELECT * FROM aclexplode(nspacl) AS x) AS acl ON true
-                JOIN pg_catalog.pg_user users ON acl.grantee = users.usesysid
-            WHERE users.usename OPERATOR(pg_catalog.=) $1::pg_catalog.name
-                AND nspname OPERATOR(pg_catalog.=) $2::pg_catalog.name";
+            FROM pg_namespace
+                JOIN LATERAL (SELECT * FROM aclexplode(nspacl) AS x) acl ON true
+                JOIN pg_user users ON acl.grantee = users.usesysid
+            WHERE users.usename = $1
+                AND nspname = $2";
         let rows = db_client
             .query(select, &[role_name, schema_name])
             .await
@@ -2587,9 +2391,8 @@ LIMIT 100",
                 .await
                 .with_context(|| format!("Failed to execute query: {query}"))?;
         } else {
-            let query = format!(
-                "CREATE EXTENSION IF NOT EXISTS {ext_name} WITH SCHEMA public VERSION {quoted_version}"
-            );
+            let query =
+                format!("CREATE EXTENSION IF NOT EXISTS {ext_name} WITH VERSION {quoted_version}");
             db_client
                 .simple_query(&query)
                 .await
@@ -2620,7 +2423,7 @@ LIMIT 100",
         if let Some(libs) = spec.cluster.settings.find("shared_preload_libraries") {
             libs_vec = libs
                 .split(&[',', '\'', ' '])
-                .filter(|s| *s != "neon" && *s != "databricks_auth" && !s.is_empty())
+                .filter(|s| *s != "neon" && !s.is_empty())
                 .map(str::to_string)
                 .collect();
         }
@@ -2639,7 +2442,7 @@ LIMIT 100",
             if let Some(libs) = shared_preload_libraries_line.split("='").nth(1) {
                 preload_libs_vec = libs
                     .split(&[',', '\'', ' '])
-                    .filter(|s| *s != "neon" && *s != "databricks_auth" && !s.is_empty())
+                    .filter(|s| *s != "neon" && !s.is_empty())
                     .map(str::to_string)
                     .collect();
             }
@@ -2780,7 +2583,7 @@ LIMIT 100",
                 // 4. We start again and try to prewarm with the state from 2. instead of the previous complete state
                 if matches!(
                     prewarm_state,
-                    LfcPrewarmState::Completed { .. }
+                    LfcPrewarmState::Completed
                         | LfcPrewarmState::NotPrewarmed
                         | LfcPrewarmState::Skipped
                 ) {
@@ -2813,34 +2616,6 @@ LIMIT 100",
             );
         }
     }
-
-    /// Set the compute spec and update related metrics.
-    /// This is the central place where pspec is updated.
-    pub fn set_spec(params: &ComputeNodeParams, state: &mut ComputeState, pspec: ParsedSpec) {
-        state.pspec = Some(pspec);
-        ComputeNode::update_attached_metric(params, state);
-        let _ = logger::update_ids(&params.instance_id, &Some(params.compute_id.clone()));
-    }
-
-    pub fn update_attached_metric(params: &ComputeNodeParams, state: &mut ComputeState) {
-        // Update the pg_cctl_attached gauge when all identifiers are available.
-        if let Some(instance_id) = &params.instance_id {
-            if let Some(pspec) = &state.pspec {
-                // Clear all values in the metric
-                COMPUTE_ATTACHED.reset();
-
-                // Set new metric value
-                COMPUTE_ATTACHED
-                    .with_label_values(&[
-                        &params.compute_id,
-                        instance_id,
-                        &pspec.tenant_id.to_string(),
-                        &pspec.timeline_id.to_string(),
-                    ])
-                    .set(1);
-            }
-        }
-    }
 }
 
 pub async fn installed_extensions(conf: tokio_postgres::Config) -> Result<()> {

compute_tools/src/compute_prewarm.rs

@@ -7,11 +7,18 @@ use http::StatusCode;
 use reqwest::Client;
 use std::mem::replace;
 use std::sync::Arc;
-use std::time::Instant;
-use tokio::{io::AsyncReadExt, select, spawn};
-use tokio_util::sync::CancellationToken;
+use tokio::{io::AsyncReadExt, spawn};
 use tracing::{error, info};
 
+#[derive(serde::Serialize, Default)]
+pub struct LfcPrewarmStateWithProgress {
+    #[serde(flatten)]
+    base: LfcPrewarmState,
+    total: i32,
+    prewarmed: i32,
+    skipped: i32,
+}
+
 /// A pair of url and a token to query endpoint storage for LFC prewarm-related tasks
 struct EndpointStoragePair {
     url: String,
@@ -20,7 +27,7 @@ struct EndpointStoragePair {
 
 const KEY: &str = "lfc_state";
 impl EndpointStoragePair {
-    /// endpoint_id is set to None while prewarming from other endpoint, see compute_promote.rs
+    /// endpoint_id is set to None while prewarming from other endpoint, see replica promotion
     /// If not None, takes precedence over pspec.spec.endpoint_id
     fn from_spec_and_endpoint(
         pspec: &crate::compute::ParsedSpec,
@@ -46,8 +53,36 @@ impl EndpointStoragePair {
 }
 
 impl ComputeNode {
-    pub async fn lfc_prewarm_state(&self) -> LfcPrewarmState {
-        self.state.lock().unwrap().lfc_prewarm_state.clone()
+    // If prewarm failed, we want to get overall number of segments as well as done ones.
+    // However, this function should be reliable even if querying postgres failed.
+    pub async fn lfc_prewarm_state(&self) -> LfcPrewarmStateWithProgress {
+        info!("requesting LFC prewarm state from postgres");
+        let mut state = LfcPrewarmStateWithProgress::default();
+        {
+            state.base = self.state.lock().unwrap().lfc_prewarm_state.clone();
+        }
+
+        let client = match ComputeNode::get_maintenance_client(&self.tokio_conn_conf).await {
+            Ok(client) => client,
+            Err(err) => {
+                error!(%err, "connecting to postgres");
+                return state;
+            }
+        };
+        let row = match client
+            .query_one("select * from neon.get_prewarm_info()", &[])
+            .await
+        {
+            Ok(row) => row,
+            Err(err) => {
+                error!(%err, "querying LFC prewarm status");
+                return state;
+            }
+        };
+        state.total = row.try_get(0).unwrap_or_default();
+        state.prewarmed = row.try_get(1).unwrap_or_default();
+        state.skipped = row.try_get(2).unwrap_or_default();
+        state
     }
 
     pub fn lfc_offload_state(&self) -> LfcOffloadState {
@@ -55,37 +90,36 @@ impl ComputeNode {
     }
 
     /// If there is a prewarm request ongoing, return `false`, `true` otherwise.
-    /// Has a failpoint "compute-prewarm"
     pub fn prewarm_lfc(self: &Arc<Self>, from_endpoint: Option<String>) -> bool {
-        let token: CancellationToken;
         {
-            let state = &mut self.state.lock().unwrap();
-            token = state.lfc_prewarm_token.clone();
-            if let LfcPrewarmState::Prewarming =
-                replace(&mut state.lfc_prewarm_state, LfcPrewarmState::Prewarming)
-            {
+            let state = &mut self.state.lock().unwrap().lfc_prewarm_state;
+            if let LfcPrewarmState::Prewarming = replace(state, LfcPrewarmState::Prewarming) {
                 return false;
             }
         }
         crate::metrics::LFC_PREWARMS.inc();
 
-        let this = self.clone();
+        let cloned = self.clone();
         spawn(async move {
-            let prewarm_state = match this.prewarm_impl(from_endpoint, token).await {
-                Ok(state) => state,
+            let state = match cloned.prewarm_impl(from_endpoint).await {
+                Ok(true) => LfcPrewarmState::Completed,
+                Ok(false) => {
+                    info!(
+                        "skipping LFC prewarm because LFC state is not found in endpoint storage"
+                    );
+                    LfcPrewarmState::Skipped
+                }
                 Err(err) => {
                     crate::metrics::LFC_PREWARM_ERRORS.inc();
                     error!(%err, "could not prewarm LFC");
-                    let error = format!("{err:#}");
-                    LfcPrewarmState::Failed { error }
+
+                    LfcPrewarmState::Failed {
+                        error: err.to_string(),
+                    }
                 }
             };
 
-            let state = &mut this.state.lock().unwrap();
-            if let LfcPrewarmState::Cancelled = prewarm_state {
-                state.lfc_prewarm_token = CancellationToken::new();
-            }
-            state.lfc_prewarm_state = prewarm_state;
+            cloned.state.lock().unwrap().lfc_prewarm_state = state;
         });
         true
     }
@@ -97,101 +131,51 @@ impl ComputeNode {
     }
 
     /// Request LFC state from endpoint storage and load corresponding pages into Postgres.
-    async fn prewarm_impl(
-        &self,
-        from_endpoint: Option<String>,
-        token: CancellationToken,
-    ) -> Result<LfcPrewarmState> {
-        let EndpointStoragePair {
-            url,
-            token: storage_token,
-        } = self.endpoint_storage_pair(from_endpoint)?;
-
-        #[cfg(feature = "testing")]
-        fail::fail_point!("compute-prewarm", |_| bail!("compute-prewarm failpoint"));
+    /// Returns a result with `false` if the LFC state is not found in endpoint storage.
+    async fn prewarm_impl(&self, from_endpoint: Option<String>) -> Result<bool> {
+        let EndpointStoragePair { url, token } = self.endpoint_storage_pair(from_endpoint)?;
 
         info!(%url, "requesting LFC state from endpoint storage");
-        let mut now = Instant::now();
-        let request = Client::new().get(&url).bearer_auth(storage_token);
-        let response = select! {
-            _ = token.cancelled() => return Ok(LfcPrewarmState::Cancelled),
-            response = request.send() => response
-        }
-        .context("querying endpoint storage")?;
-
-        match response.status() {
+        let request = Client::new().get(&url).bearer_auth(token);
+        let res = request.send().await.context("querying endpoint storage")?;
+        let status = res.status();
+        match status {
             StatusCode::OK => (),
-            StatusCode::NOT_FOUND => return Ok(LfcPrewarmState::Skipped),
-            status => bail!("{status} querying endpoint storage"),
+            StatusCode::NOT_FOUND => {
+                return Ok(false);
+            }
+            _ => bail!("{status} querying endpoint storage"),
         }
-        let state_download_time_ms = now.elapsed().as_millis() as u32;
-        now = Instant::now();
 
         let mut uncompressed = Vec::new();
-        let lfc_state = select! {
-            _ = token.cancelled() => return Ok(LfcPrewarmState::Cancelled),
-            lfc_state = response.bytes() => lfc_state
-        }
-        .context("getting request body from endpoint storage")?;
-
-        let mut decoder = ZstdDecoder::new(lfc_state.iter().as_slice());
-        select! {
-            _ = token.cancelled() => return Ok(LfcPrewarmState::Cancelled),
-            read = decoder.read_to_end(&mut uncompressed) => read
-        }
-        .context("decoding LFC state")?;
-        let uncompress_time_ms = now.elapsed().as_millis() as u32;
-        now = Instant::now();
-
+        let lfc_state = res
+            .bytes()
+            .await
+            .context("getting request body from endpoint storage")?;
+        ZstdDecoder::new(lfc_state.iter().as_slice())
+            .read_to_end(&mut uncompressed)
+            .await
+            .context("decoding LFC state")?;
         let uncompressed_len = uncompressed.len();
-        info!(%url, "downloaded LFC state, uncompressed size {uncompressed_len}");
 
-        // Client connection and prewarm info querying are fast and therefore don't need
-        // cancellation
-        let client = ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
+        info!(%url, "downloaded LFC state, uncompressed size {uncompressed_len}, loading into Postgres");
+
+        ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
             .await
-            .context("connecting to postgres")?;
-        let pg_token = client.cancel_token();
-
-        let params: Vec<&(dyn postgres_types::ToSql + Sync)> = vec![&uncompressed];
-        select! {
-            res = client.query_one("select neon.prewarm_local_cache($1)", &params) => res,
-            _ = token.cancelled() => {
-                pg_token.cancel_query(postgres::NoTls).await
-                    .context("cancelling neon.prewarm_local_cache()")?;
-                return Ok(LfcPrewarmState::Cancelled)
-            }
-        }
-        .context("loading LFC state into postgres")
-        .map(|_| ())?;
-        let prewarm_time_ms = now.elapsed().as_millis() as u32;
-
-        let row = client
-            .query_one("select * from neon.get_prewarm_info()", &[])
+            .context("connecting to postgres")?
+            .query_one("select neon.prewarm_local_cache($1)", &[&uncompressed])
             .await
-            .context("querying prewarm info")?;
-        let total = row.try_get(0).unwrap_or_default();
-        let prewarmed = row.try_get(1).unwrap_or_default();
-        let skipped = row.try_get(2).unwrap_or_default();
+            .context("loading LFC state into postgres")
+            .map(|_| ())?;
 
-        Ok(LfcPrewarmState::Completed {
-            total,
-            prewarmed,
-            skipped,
-            state_download_time_ms,
-            uncompress_time_ms,
-            prewarm_time_ms,
-        })
+        Ok(true)
     }
 
     /// If offload request is ongoing, return false, true otherwise
     pub fn offload_lfc(self: &Arc<Self>) -> bool {
         {
             let state = &mut self.state.lock().unwrap().lfc_offload_state;
-            if matches!(
-                replace(state, LfcOffloadState::Offloading),
-                LfcOffloadState::Offloading
-            ) {
+            if replace(state, LfcOffloadState::Offloading) == LfcOffloadState::Offloading {
                 return false;
             }
         }
@@ -203,10 +187,7 @@ impl ComputeNode {
     pub async fn offload_lfc_async(self: &Arc<Self>) {
         {
             let state = &mut self.state.lock().unwrap().lfc_offload_state;
-            if matches!(
-                replace(state, LfcOffloadState::Offloading),
-                LfcOffloadState::Offloading
-            ) {
+            if replace(state, LfcOffloadState::Offloading) == LfcOffloadState::Offloading {
                 return;
             }
         }
@@ -215,69 +196,48 @@ impl ComputeNode {
 
     async fn offload_lfc_with_state_update(&self) {
         crate::metrics::LFC_OFFLOADS.inc();
-        let state = match self.offload_lfc_impl().await {
-            Ok(state) => state,
-            Err(err) => {
-                crate::metrics::LFC_OFFLOAD_ERRORS.inc();
-                error!(%err, "could not offload LFC");
-                let error = format!("{err:#}");
-                LfcOffloadState::Failed { error }
-            }
+
+        let Err(err) = self.offload_lfc_impl().await else {
+            self.state.lock().unwrap().lfc_offload_state = LfcOffloadState::Completed;
+            return;
+        };
+
+        crate::metrics::LFC_OFFLOAD_ERRORS.inc();
+        error!(%err, "could not offload LFC state to endpoint storage");
+        self.state.lock().unwrap().lfc_offload_state = LfcOffloadState::Failed {
+            error: err.to_string(),
         };
-        self.state.lock().unwrap().lfc_offload_state = state;
     }
 
-    async fn offload_lfc_impl(&self) -> Result<LfcOffloadState> {
+    async fn offload_lfc_impl(&self) -> Result<()> {
         let EndpointStoragePair { url, token } = self.endpoint_storage_pair(None)?;
         info!(%url, "requesting LFC state from Postgres");
 
-        let mut now = Instant::now();
-        let row = ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
+        let mut compressed = Vec::new();
+        ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
             .await
             .context("connecting to postgres")?
             .query_one("select neon.get_local_cache_state()", &[])
             .await
-            .context("querying LFC state")?;
-        let state = row
-            .try_get::<usize, Option<&[u8]>>(0)
-            .context("deserializing LFC state")?;
-        let Some(state) = state else {
-            info!(%url, "empty LFC state, not exporting");
-            return Ok(LfcOffloadState::Skipped);
-        };
-        let state_query_time_ms = now.elapsed().as_millis() as u32;
-        now = Instant::now();
-
-        let mut compressed = Vec::new();
-        ZstdEncoder::new(state)
+            .context("querying LFC state")?
+            .try_get::<usize, &[u8]>(0)
+            .context("deserializing LFC state")
+            .map(ZstdEncoder::new)?
             .read_to_end(&mut compressed)
             .await
             .context("compressing LFC state")?;
-        let compress_time_ms = now.elapsed().as_millis() as u32;
-        now = Instant::now();
 
         let compressed_len = compressed.len();
-        info!(%url, "downloaded LFC state, compressed size {compressed_len}");
+        info!(%url, "downloaded LFC state, compressed size {compressed_len}, writing to endpoint storage");
 
         let request = Client::new().put(url).bearer_auth(token).body(compressed);
-        let response = request
-            .send()
-            .await
-            .context("writing to endpoint storage")?;
-        let state_upload_time_ms = now.elapsed().as_millis() as u32;
-        let status = response.status();
-        if status != StatusCode::OK {
-            bail!("request to endpoint storage failed: {status}");
+        match request.send().await {
+            Ok(res) if res.status() == StatusCode::OK => Ok(()),
+            Ok(res) => bail!(
+                "Request to endpoint storage failed with status: {}",
+                res.status()
+            ),
+            Err(err) => Err(err).context("writing to endpoint storage"),
         }
-
-        Ok(LfcOffloadState::Completed {
-            compress_time_ms,
-            state_query_time_ms,
-            state_upload_time_ms,
-        })
-    }
-
-    pub fn cancel_prewarm(self: &Arc<Self>) {
-        self.state.lock().unwrap().lfc_prewarm_token.cancel();
     }
 }

compute_tools/src/compute_promote.rs

@@ -1,56 +1,69 @@
 use crate::compute::ComputeNode;
-use anyhow::{Context, bail};
-use compute_api::responses::{LfcPrewarmState, PromoteConfig, PromoteState};
-use std::time::Instant;
-use tracing::info;
+use anyhow::{Context, Result, bail};
+use compute_api::{
+    responses::{LfcPrewarmState, PromoteState, SafekeepersLsn},
+    spec::ComputeMode,
+};
+use std::{sync::Arc, time::Duration};
+use tokio::time::sleep;
+use utils::lsn::Lsn;
 
 impl ComputeNode {
-    /// Returns only when promote fails or succeeds. If http client calling this function
-    /// disconnects, this does not stop promotion, and subsequent calls block until promote finishes.
+    /// Returns only when promote fails or succeeds. If a network error occurs
+    /// and http client disconnects, this does not stop promotion, and subsequent
+    /// calls block until promote finishes.
     /// Called by control plane on secondary after primary endpoint is terminated
-    /// Has a failpoint "compute-promotion"
-    pub async fn promote(self: &std::sync::Arc<Self>, cfg: PromoteConfig) -> PromoteState {
-        let this = self.clone();
-        let promote_fn = async move || match this.promote_impl(cfg).await {
-            Ok(state) => state,
-            Err(err) => {
-                tracing::error!(%err, "promoting replica");
-                let error = format!("{err:#}");
-                PromoteState::Failed { error }
-            }
-        };
+    pub async fn promote(self: &Arc<Self>, safekeepers_lsn: SafekeepersLsn) -> PromoteState {
+        let cloned = self.clone();
         let start_promotion = || {
             let (tx, rx) = tokio::sync::watch::channel(PromoteState::NotPromoted);
-            tokio::spawn(async move { tx.send(promote_fn().await) });
+            tokio::spawn(async move {
+                tx.send(match cloned.promote_impl(safekeepers_lsn).await {
+                    Ok(_) => PromoteState::Completed,
+                    Err(err) => {
+                        tracing::error!(%err, "promoting");
+                        PromoteState::Failed {
+                            error: err.to_string(),
+                        }
+                    }
+                })
+            });
             rx
         };
 
         let mut task;
-        // promote_impl locks self.state so we need to unlock it before calling task.changed()
+        // self.state is unlocked after block ends so we lock it in promote_impl
+        // and task.changed() is reached
         {
-            let promote_state = &mut self.state.lock().unwrap().promote_state;
-            task = promote_state.get_or_insert_with(start_promotion).clone()
-        }
-        if task.changed().await.is_err() {
-            let error = "promote sender dropped".to_string();
-            return PromoteState::Failed { error };
+            task = self
+                .state
+                .lock()
+                .unwrap()
+                .promote_state
+                .get_or_insert_with(start_promotion)
+                .clone()
         }
+        task.changed().await.expect("promote sender dropped");
         task.borrow().clone()
     }
 
-    async fn promote_impl(&self, cfg: PromoteConfig) -> anyhow::Result<PromoteState> {
+    // Why do we have to supply safekeepers?
+    // For secondary we use primary_connection_conninfo so safekeepers field is empty
+    async fn promote_impl(&self, safekeepers_lsn: SafekeepersLsn) -> Result<()> {
         {
             let state = self.state.lock().unwrap();
             let mode = &state.pspec.as_ref().unwrap().spec.mode;
-            if *mode != compute_api::spec::ComputeMode::Replica {
-                bail!("compute mode \"{}\" is not replica", mode.to_type_str());
+            if *mode != ComputeMode::Replica {
+                bail!("{} is not replica", mode.to_type_str());
             }
+
+            // we don't need to query Postgres so not self.lfc_prewarm_state()
             match &state.lfc_prewarm_state {
-                status @ (LfcPrewarmState::NotPrewarmed | LfcPrewarmState::Prewarming) => {
-                    bail!("compute {status}")
+                LfcPrewarmState::NotPrewarmed | LfcPrewarmState::Prewarming => {
+                    bail!("prewarm not requested or pending")
                 }
                 LfcPrewarmState::Failed { error } => {
-                    tracing::warn!(%error, "compute prewarm failed")
+                    tracing::warn!(%error, "replica prewarm failed")
                 }
                 _ => {}
             }
@@ -59,66 +72,51 @@ impl ComputeNode {
         let client = ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
             .await
             .context("connecting to postgres")?;
-        let mut now = Instant::now();
 
-        let primary_lsn = cfg.wal_flush_lsn;
-        let mut standby_lsn = utils::lsn::Lsn::INVALID;
+        let primary_lsn = safekeepers_lsn.wal_flush_lsn;
+        let mut last_wal_replay_lsn: Lsn = Lsn::INVALID;
         const RETRIES: i32 = 20;
         for i in 0..=RETRIES {
             let row = client
-                .query_one("SELECT pg_catalog.pg_last_wal_replay_lsn()", &[])
+                .query_one("SELECT pg_last_wal_replay_lsn()", &[])
                 .await
                 .context("getting last replay lsn")?;
             let lsn: u64 = row.get::<usize, postgres_types::PgLsn>(0).into();
-            standby_lsn = lsn.into();
-            if standby_lsn >= primary_lsn {
+            last_wal_replay_lsn = lsn.into();
+            if last_wal_replay_lsn >= primary_lsn {
                 break;
             }
-            info!(%standby_lsn, %primary_lsn, "catching up, try {i}");
-            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+            tracing::info!("Try {i}, replica lsn {last_wal_replay_lsn}, primary lsn {primary_lsn}");
+            sleep(Duration::from_secs(1)).await;
         }
-        if standby_lsn < primary_lsn {
+        if last_wal_replay_lsn < primary_lsn {
             bail!("didn't catch up with primary in {RETRIES} retries");
         }
-        let lsn_wait_time_ms = now.elapsed().as_millis() as u32;
-        now = Instant::now();
 
         // using $1 doesn't work with ALTER SYSTEM SET
         let safekeepers_sql = format!(
             "ALTER SYSTEM SET neon.safekeepers='{}'",
-            cfg.spec.safekeeper_connstrings.join(",")
+            safekeepers_lsn.safekeepers
         );
         client
             .query(&safekeepers_sql, &[])
             .await
             .context("setting safekeepers")?;
         client
-            .query(
-                "ALTER SYSTEM SET synchronous_standby_names=walproposer",
-                &[],
-            )
-            .await
-            .context("setting synchronous_standby_names")?;
-        client
-            .query("SELECT pg_catalog.pg_reload_conf()", &[])
+            .query("SELECT pg_reload_conf()", &[])
             .await
             .context("reloading postgres config")?;
-
-        #[cfg(feature = "testing")]
-        fail::fail_point!("compute-promotion", |_| bail!(
-            "compute-promotion failpoint"
-        ));
-
         let row = client
-            .query_one("SELECT * FROM pg_catalog.pg_promote()", &[])
+            .query_one("SELECT * FROM pg_promote()", &[])
             .await
             .context("pg_promote")?;
         if !row.get::<usize, bool>(0) {
-            bail!("pg_promote() failed");
+            bail!("pg_promote() returned false");
         }
-        let pg_promote_time_ms = now.elapsed().as_millis() as u32;
-        let now = Instant::now();
 
+        let client = ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
+            .await
+            .context("connecting to postgres")?;
         let row = client
             .query_one("SHOW transaction_read_only", &[])
             .await
@@ -127,47 +125,8 @@ impl ComputeNode {
             bail!("replica in read only mode after promotion");
         }
 
-        // Already checked validity in http handler
-        #[allow(unused_mut)]
-        let mut new_pspec = crate::compute::ParsedSpec::try_from(cfg.spec).expect("invalid spec");
-        {
-            let mut state = self.state.lock().unwrap();
-
-            // Local setup has different ports for pg process (port=) for primary and secondary.
-            // Primary is stopped so we need secondary's "port" value
-            #[cfg(feature = "testing")]
-            {
-                let old_spec = &state.pspec.as_ref().unwrap().spec;
-                let Some(old_conf) = old_spec.cluster.postgresql_conf.as_ref() else {
-                    bail!("pspec.spec.cluster.postgresql_conf missing for endpoint");
-                };
-                let set: std::collections::HashMap<&str, &str> = old_conf
-                    .split_terminator('\n')
-                    .map(|e| e.split_once("=").expect("invalid item"))
-                    .collect();
-
-                let Some(new_conf) = new_pspec.spec.cluster.postgresql_conf.as_mut() else {
-                    bail!("pspec.spec.cluster.postgresql_conf missing for supplied config");
-                };
-                new_conf.push_str(&format!("port={}\n", set["port"]));
-            }
-
-            tracing::debug!("applied spec: {:#?}", new_pspec.spec);
-            if self.params.lakebase_mode {
-                ComputeNode::set_spec(&self.params, &mut state, new_pspec);
-            } else {
-                state.pspec = Some(new_pspec);
-            }
-        }
-
-        info!("applied new spec, reconfiguring as primary");
-        self.reconfigure()?;
-        let reconfigure_time_ms = now.elapsed().as_millis() as u32;
-
-        Ok(PromoteState::Completed {
-            lsn_wait_time_ms,
-            pg_promote_time_ms,
-            reconfigure_time_ms,
-        })
+        let mut state = self.state.lock().unwrap();
+        state.pspec.as_mut().unwrap().spec.mode = ComputeMode::Primary;
+        Ok(())
     }
 }

compute_tools/src/config.rs

@@ -7,14 +7,11 @@ use std::io::prelude::*;
 use std::path::Path;
 
 use compute_api::responses::TlsConfig;
-use compute_api::spec::{
-    ComputeAudit, ComputeMode, ComputeSpec, DatabricksSettings, GenericOption,
-};
+use compute_api::spec::{ComputeAudit, ComputeMode, ComputeSpec, GenericOption};
 
 use crate::compute::ComputeNodeParams;
 use crate::pg_helpers::{
-    DatabricksSettingsExt as _, GenericOptionExt, GenericOptionsSearch, PgOptionsSerialize,
-    escape_conf_value,
+    GenericOptionExt, GenericOptionsSearch, PgOptionsSerialize, escape_conf_value,
 };
 use crate::tls::{self, SERVER_CRT, SERVER_KEY};
 
@@ -45,16 +42,12 @@ pub fn line_in_file(path: &Path, line: &str) -> Result<bool> {
 }
 
 /// Create or completely rewrite configuration file specified by `path`
-#[allow(clippy::too_many_arguments)]
 pub fn write_postgres_conf(
     pgdata_path: &Path,
     params: &ComputeNodeParams,
     spec: &ComputeSpec,
-    postgres_port: Option<u16>,
     extension_server_port: u16,
     tls_config: &Option<TlsConfig>,
-    databricks_settings: Option<&DatabricksSettings>,
-    lakebase_mode: bool,
 ) -> Result<()> {
     let path = pgdata_path.join("postgresql.conf");
     // File::create() destroys the file content if it exists.
@@ -73,12 +66,13 @@ pub fn write_postgres_conf(
         if let Some(stripe_size) = conninfo.stripe_size {
             writeln!(
                 file,
-                "# from compute spec's pageserver_connection_info.stripe_size field"
+                "# from compute spec's pageserver_conninfo.stripe_size field"
             )?;
             writeln!(file, "neon.stripe_size={stripe_size}")?;
         }
 
         let mut libpq_urls: Option<Vec<String>> = Some(Vec::new());
+        let mut grpc_urls: Option<Vec<String>> = Some(Vec::new());
         let num_shards = if conninfo.shard_count.0 == 0 {
             1 // unsharded, treat it as a single shard
         } else {
@@ -111,11 +105,19 @@ pub fn write_postgres_conf(
             } else {
                 libpq_urls = None
             }
+            // Similarly for gRPC URLs
+            if let Some(url) = &first_pageserver.grpc_url {
+                if let Some(ref mut urls) = grpc_urls {
+                    urls.push(url.clone());
+                }
+            } else {
+                grpc_urls = None
+            }
         }
         if let Some(libpq_urls) = libpq_urls {
             writeln!(
                 file,
-                "# derived from compute spec's pageserver_connection_info field"
+                "# derived from compute spec's pageserver_conninfo field"
             )?;
             writeln!(
                 file,
@@ -125,12 +127,26 @@ pub fn write_postgres_conf(
         } else {
             writeln!(file, "# no neon.pageserver_connstring")?;
         }
+        if let Some(grpc_urls) = grpc_urls {
+            writeln!(
+                file,
+                "# derived from compute spec's pageserver_conninfo field"
+            )?;
+            writeln!(
+                file,
+                "neon.pageserver_grpc_urls={}",
+                escape_conf_value(&grpc_urls.join(","))
+            )?;
+        } else {
+            writeln!(file, "# no neon.pageserver_grpc_urls")?;
+        }
     } else {
         // Stripe size GUC should be defined prior to connection string
         if let Some(stripe_size) = spec.shard_stripe_size {
             writeln!(file, "# from compute spec's shard_stripe_size field")?;
             writeln!(file, "neon.stripe_size={stripe_size}")?;
         }
+
         if let Some(s) = &spec.pageserver_connstring {
             writeln!(file, "# from compute spec's pageserver_connstring field")?;
             writeln!(file, "neon.pageserver_connstring={}", escape_conf_value(s))?;
@@ -357,24 +373,6 @@ pub fn write_postgres_conf(
         writeln!(file, "log_destination='stderr,syslog'")?;
     }
 
-    if lakebase_mode {
-        // Explicitly set the port based on the connstr, overriding any previous port setting.
-        // Note: It is important that we don't specify a different port again after this.
-        let port = postgres_port.expect("port must be present in connstr");
-        writeln!(file, "port = {port}")?;
-
-        // This is databricks specific settings.
-        // This should be at the end of the file but before `compute_ctl_temp_override.conf` below
-        // so that it can override any settings above.
-        // `compute_ctl_temp_override.conf` is intended to override any settings above during specific operations.
-        // To prevent potential breakage in the future, we keep it above `compute_ctl_temp_override.conf`.
-        writeln!(file, "# Databricks settings start")?;
-        if let Some(settings) = databricks_settings {
-            writeln!(file, "{}", settings.as_pg_settings())?;
-        }
-        writeln!(file, "# Databricks settings end")?;
-    }
-
     // This is essential to keep this line at the end of the file,
     // because it is intended to override any settings above.
     writeln!(file, "include_if_exists = 'compute_ctl_temp_override.conf'")?;

compute_tools/src/configurator.rs

@@ -1,40 +1,23 @@
-use std::fs::File;
+use std::sync::Arc;
 use std::thread;
-use std::{path::Path, sync::Arc};
 
-use anyhow::Result;
-use compute_api::responses::{ComputeConfig, ComputeStatus};
+use compute_api::responses::ComputeStatus;
 use tracing::{error, info, instrument};
 
-use crate::compute::{ComputeNode, ParsedSpec};
-use crate::spec::get_config_from_control_plane;
+use crate::compute::ComputeNode;
 
 #[instrument(skip_all)]
 fn configurator_main_loop(compute: &Arc<ComputeNode>) {
     info!("waiting for reconfiguration requests");
     loop {
         let mut state = compute.state.lock().unwrap();
-        /* BEGIN_HADRON */
-        // RefreshConfiguration should only be used inside the loop
-        assert_ne!(state.status, ComputeStatus::RefreshConfiguration);
-        /* END_HADRON */
 
-        if compute.params.lakebase_mode {
-            while state.status != ComputeStatus::ConfigurationPending
-                && state.status != ComputeStatus::RefreshConfigurationPending
-                && state.status != ComputeStatus::Failed
-            {
-                info!("configurator: compute status: {:?}, sleeping", state.status);
-                state = compute.state_changed.wait(state).unwrap();
-            }
-        } else {
-            // We have to re-check the status after re-acquiring the lock because it could be that
-            // the status has changed while we were waiting for the lock, and we might not need to
-            // wait on the condition variable. Otherwise, we might end up in some soft-/deadlock, i.e.
-            // we are waiting for a condition variable that will never be signaled.
-            if state.status != ComputeStatus::ConfigurationPending {
-                state = compute.state_changed.wait(state).unwrap();
-            }
+        // We have to re-check the status after re-acquiring the lock because it could be that
+        // the status has changed while we were waiting for the lock, and we might not need to
+        // wait on the condition variable. Otherwise, we might end up in some soft-/deadlock, i.e.
+        // we are waiting for a condition variable that will never be signaled.
+        if state.status != ComputeStatus::ConfigurationPending {
+            state = compute.state_changed.wait(state).unwrap();
         }
 
         // Re-check the status after waking up
@@ -54,136 +37,6 @@ fn configurator_main_loop(compute: &Arc<ComputeNode>) {
             // XXX: used to test that API is blocking
             // std::thread::sleep(std::time::Duration::from_millis(10000));
 
-            compute.set_status(new_status);
-        } else if state.status == ComputeStatus::RefreshConfigurationPending {
-            info!(
-                "compute node suspects its configuration is out of date, now refreshing configuration"
-            );
-            state.set_status(ComputeStatus::RefreshConfiguration, &compute.state_changed);
-            // Drop the lock guard here to avoid holding the lock while downloading config from the control plane / HCC.
-            // This is the only thread that can move compute_ctl out of the `RefreshConfiguration` state, so it
-            // is safe to drop the lock like this.
-            drop(state);
-
-            let get_config_result: anyhow::Result<ComputeConfig> =
-                if let Some(config_path) = &compute.params.config_path_test_only {
-                    // This path is only to make testing easier. In production we always get the config from the HCC.
-                    info!(
-                        "reloading config.json from path: {}",
-                        config_path.to_string_lossy()
-                    );
-                    let path = Path::new(config_path);
-                    if let Ok(file) = File::open(path) {
-                        match serde_json::from_reader::<File, ComputeConfig>(file) {
-                            Ok(config) => Ok(config),
-                            Err(e) => {
-                                error!("could not parse config file: {}", e);
-                                Err(anyhow::anyhow!("could not parse config file: {}", e))
-                            }
-                        }
-                    } else {
-                        error!(
-                            "could not open config file at path: {:?}",
-                            config_path.to_string_lossy()
-                        );
-                        Err(anyhow::anyhow!(
-                            "could not open config file at path: {}",
-                            config_path.to_string_lossy()
-                        ))
-                    }
-                } else if let Some(control_plane_uri) = &compute.params.control_plane_uri {
-                    get_config_from_control_plane(control_plane_uri, &compute.params.compute_id)
-                } else {
-                    Err(anyhow::anyhow!("config_path_test_only is not set"))
-                };
-
-            // Parse any received ComputeSpec and transpose the result into a Result<Option<ParsedSpec>>.
-            let parsed_spec_result: Result<Option<ParsedSpec>> =
-                get_config_result.and_then(|config| {
-                    if let Some(spec) = config.spec {
-                        if let Ok(pspec) = ParsedSpec::try_from(spec) {
-                            Ok(Some(pspec))
-                        } else {
-                            Err(anyhow::anyhow!("could not parse spec"))
-                        }
-                    } else {
-                        Ok(None)
-                    }
-                });
-
-            let new_status: ComputeStatus;
-            match parsed_spec_result {
-                // Control plane (HCM) returned a spec and we were able to parse it.
-                Ok(Some(pspec)) => {
-                    {
-                        let mut state = compute.state.lock().unwrap();
-                        // Defensive programming to make sure this thread is indeed the only one that can move the compute
-                        // node out of the `RefreshConfiguration` state. Would be nice if we can encode this invariant
-                        // into the type system.
-                        assert_eq!(state.status, ComputeStatus::RefreshConfiguration);
-
-                        if state
-                            .pspec
-                            .as_ref()
-                            .map(|ps| ps.pageserver_conninfo.clone())
-                            == Some(pspec.pageserver_conninfo.clone())
-                        {
-                            info!(
-                                "Refresh configuration: Retrieved spec is the same as the current spec. Waiting for control plane to update the spec before attempting reconfiguration."
-                            );
-                            state.status = ComputeStatus::Running;
-                            compute.state_changed.notify_all();
-                            drop(state);
-                            std::thread::sleep(std::time::Duration::from_secs(5));
-                            continue;
-                        }
-                        // state.pspec is consumed by compute.reconfigure() below. Note that compute.reconfigure() will acquire
-                        // the compute.state lock again so we need to have the lock guard go out of scope here. We could add a
-                        // "locked" variant of compute.reconfigure() that takes the lock guard as an argument to make this cleaner,
-                        // but it's not worth forking the codebase too much for this minor point alone right now.
-                        state.pspec = Some(pspec);
-                    }
-                    match compute.reconfigure() {
-                        Ok(_) => {
-                            info!("Refresh configuration: compute node configured");
-                            new_status = ComputeStatus::Running;
-                        }
-                        Err(e) => {
-                            error!(
-                                "Refresh configuration: could not configure compute node: {}",
-                                e
-                            );
-                            // Set the compute node back to the `RefreshConfigurationPending` state if the configuration
-                            // was not successful. It should be okay to treat this situation the same as if the loop
-                            // hasn't executed yet as long as the detection side keeps notifying.
-                            new_status = ComputeStatus::RefreshConfigurationPending;
-                        }
-                    }
-                }
-                // Control plane (HCM)'s response does not contain a spec. This is the "Empty" attachment case.
-                Ok(None) => {
-                    info!(
-                        "Compute Manager signaled that this compute is no longer attached to any storage. Exiting."
-                    );
-                    // We just immediately terminate the whole compute_ctl in this case. It's not necessary to attempt a
-                    // clean shutdown as Postgres is probably not responding anyway (which is why we are in this refresh
-                    // configuration state).
-                    std::process::exit(1);
-                }
-                // Various error cases:
-                // - The request to the control plane (HCM) either failed or returned a malformed spec.
-                // - compute_ctl itself is configured incorrectly (e.g., compute_id is not set).
-                Err(e) => {
-                    error!(
-                        "Refresh configuration: error getting a parsed spec: {:?}",
-                        e
-                    );
-                    new_status = ComputeStatus::RefreshConfigurationPending;
-                    // We may be dealing with an overloaded HCM if we end up in this path. Backoff 5 seconds before
-                    // retrying to avoid hammering the HCM.
-                    std::thread::sleep(std::time::Duration::from_secs(5));
-                }
-            }
             compute.set_status(new_status);
         } else if state.status == ComputeStatus::Failed {
             info!("compute node is now in Failed state, exiting");

compute_tools/src/hadron_metrics.rs

@@ -1,60 +0,0 @@
-use metrics::{
-    IntCounter, IntGaugeVec, core::Collector, proto::MetricFamily, register_int_counter,
-    register_int_gauge_vec,
-};
-use once_cell::sync::Lazy;
-
-// Counter keeping track of the number of PageStream request errors reported by Postgres.
-// An error is registered every time Postgres calls compute_ctl's /refresh_configuration API.
-// Postgres will invoke this API if it detected trouble with PageStream requests (get_page@lsn,
-// get_base_backup, etc.) it sends to any pageserver. An increase in this counter value typically
-// indicates Postgres downtime, as PageStream requests are critical for Postgres to function.
-pub static POSTGRES_PAGESTREAM_REQUEST_ERRORS: Lazy<IntCounter> = Lazy::new(|| {
-    register_int_counter!(
-        "pg_cctl_pagestream_request_errors_total",
-        "Number of PageStream request errors reported by the postgres process"
-    )
-    .expect("failed to define a metric")
-});
-
-// Counter keeping track of the number of compute configuration errors due to Postgres statement
-// timeouts. An error is registered every time `ComputeNode::reconfigure()` fails due to Postgres
-// error code 57014 (query cancelled). This statement timeout typically occurs when postgres is
-// stuck in a problematic retry loop when the PS is reject its connection requests (usually due
-// to PG pointing at the wrong PS). We should investigate the root cause when this counter value
-// increases by checking PG and PS logs.
-pub static COMPUTE_CONFIGURE_STATEMENT_TIMEOUT_ERRORS: Lazy<IntCounter> = Lazy::new(|| {
-    register_int_counter!(
-        "pg_cctl_configure_statement_timeout_errors_total",
-        "Number of compute configuration errors due to Postgres statement timeouts."
-    )
-    .expect("failed to define a metric")
-});
-
-pub static COMPUTE_ATTACHED: Lazy<IntGaugeVec> = Lazy::new(|| {
-    register_int_gauge_vec!(
-        "pg_cctl_attached",
-        "Compute node attached status (1 if attached)",
-        &[
-            "pg_compute_id",
-            "pg_instance_id",
-            "tenant_id",
-            "timeline_id"
-        ]
-    )
-    .expect("failed to define a metric")
-});
-
-pub fn collect() -> Vec<MetricFamily> {
-    let mut metrics = Vec::new();
-    metrics.extend(POSTGRES_PAGESTREAM_REQUEST_ERRORS.collect());
-    metrics.extend(COMPUTE_CONFIGURE_STATEMENT_TIMEOUT_ERRORS.collect());
-    metrics.extend(COMPUTE_ATTACHED.collect());
-    metrics
-}
-
-pub fn initialize_metrics() {
-    Lazy::force(&POSTGRES_PAGESTREAM_REQUEST_ERRORS);
-    Lazy::force(&COMPUTE_CONFIGURE_STATEMENT_TIMEOUT_ERRORS);
-    Lazy::force(&COMPUTE_ATTACHED);
-}

compute_tools/src/http/middleware/authorize.rs

@@ -16,29 +16,13 @@ use crate::http::JsonResponse;
 #[derive(Clone, Debug)]
 pub(in crate::http) struct Authorize {
     compute_id: String,
-    // BEGIN HADRON
-    // Hadron instance ID. Only set if it's a Lakebase V1 a.k.a. Hadron instance.
-    instance_id: Option<String>,
-    // END HADRON
     jwks: JwkSet,
     validation: Validation,
 }
 
 impl Authorize {
-    pub fn new(compute_id: String, instance_id: Option<String>, jwks: JwkSet) -> Self {
+    pub fn new(compute_id: String, jwks: JwkSet) -> Self {
         let mut validation = Validation::new(Algorithm::EdDSA);
-
-        // BEGIN HADRON
-        let use_rsa = jwks.keys.iter().any(|jwk| {
-            jwk.common
-                .key_algorithm
-                .is_some_and(|alg| alg == jsonwebtoken::jwk::KeyAlgorithm::RS256)
-        });
-        if use_rsa {
-            validation = Validation::new(Algorithm::RS256);
-        }
-        // END HADRON
-
         validation.validate_exp = true;
         // Unused by the control plane
         validation.validate_nbf = false;
@@ -50,7 +34,6 @@ impl Authorize {
 
         Self {
             compute_id,
-            instance_id,
             jwks,
             validation,
         }
@@ -64,20 +47,10 @@ impl AsyncAuthorizeRequest<Body> for Authorize {
 
     fn authorize(&mut self, mut request: Request<Body>) -> Self::Future {
         let compute_id = self.compute_id.clone();
-        let is_hadron_instance = self.instance_id.is_some();
         let jwks = self.jwks.clone();
         let validation = self.validation.clone();
 
         Box::pin(async move {
-            // BEGIN HADRON
-            // In Hadron deployments the "external" HTTP endpoint on compute_ctl can only be
-            // accessed by trusted components (enforced by dblet network policy), so we can bypass
-            // all auth here.
-            if is_hadron_instance {
-                return Ok(request);
-            }
-            // END HADRON
-
             let TypedHeader(Authorization(bearer)) = request
                 .extract_parts::<TypedHeader<Authorization<Bearer>>>()
                 .await

compute_tools/src/http/openapi_spec.yaml

@@ -96,7 +96,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/ComputeSchemaWithLsn"
+                $ref: "#/components/schemas/SafekeepersLsn"
       responses:
         200:
           description: Promote succeeded or wasn't started
@@ -139,15 +139,6 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/LfcPrewarmState"
-    delete:
-      tags:
-        - Prewarm
-      summary: Cancel ongoing LFC prewarm
-      description: ""
-      operationId: cancelLfcPrewarm
-      responses:
-        202:
-          description: Prewarm cancelled
 
   /lfc/offload:
     post:
@@ -306,7 +297,14 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/ComputeSchema"
+              type: object
+              required:
+                - spec
+              properties:
+                spec:
+                  # XXX: I don't want to explain current spec in the OpenAPI format,
+                  # as it could be changed really soon. Consider doing it later.
+                  type: object
       responses:
         200:
           description: Compute configuration finished.
@@ -593,30 +591,26 @@ components:
           type: string
           example: "1.0.0"
 
-    ComputeSchema:
+    SafekeepersLsn:
       type: object
       required:
-        - spec
-      properties:
-        spec:
-          type: object
-    ComputeSchemaWithLsn:
-      type: object
-      required:
-        - spec
+        - safekeepers
         - wal_flush_lsn
       properties:
-        spec:
-          $ref: "#/components/schemas/ComputeState"
-        wal_flush_lsn:
+        safekeepers:
+          description: Primary replica safekeepers
+          type: string
+        wal_flush_lsn:
+          description: Primary last WAL flush LSN
           type: string
-          description: "last WAL flush LSN"
-          example: "0/028F10D8"
 
     LfcPrewarmState:
       type: object
       required:
         - status
+        - total
+        - prewarmed
+        - skipped
       properties:
         status:
           description: LFC prewarm status
@@ -634,15 +628,6 @@ components:
         skipped:
           description: Pages processed but not prewarmed
           type: integer
-        state_download_time_ms:
-          description: Time it takes to download LFC state to compute
-          type: integer
-        uncompress_time_ms:
-          description: Time it takes to uncompress LFC state
-          type: integer
-        prewarm_time_ms:
-          description: Time it takes to prewarm LFC state in Postgres
-          type: integer
 
     LfcOffloadState:
       type: object
@@ -651,21 +636,11 @@ components:
       properties:
         status:
           description: LFC offload status
-          enum: [not_offloaded, offloading, completed, skipped, failed]
+          enum: [not_offloaded, offloading, completed, failed]
           type: string
         error:
           description: LFC offload error, if any
           type: string
-        state_query_time_ms:
-          description: Time it takes to get LFC state from Postgres
-          type: integer
-        compress_time_ms:
-          description: Time it takes to compress LFC state
-          type: integer
-        state_upload_time_ms:
-          description: Time it takes to upload LFC state to endpoint storage
-          type: integer
-
 
     PromoteState:
       type: object
@@ -679,15 +654,6 @@ components:
         error:
           description: Promote error, if any
           type: string
-        lsn_wait_time_ms:
-          description: Time it takes for secondary to catch up with primary WAL flush LSN
-          type: integer
-        pg_promote_time_ms:
-          description: Time it takes to call pg_promote on secondary
-          type: integer
-        reconfigure_time_ms:
-          description: Time it takes to reconfigure promoted secondary
-          type: integer
 
     SetRoleGrantsRequest:
       type: object

compute_tools/src/http/routes/configure.rs

@@ -43,12 +43,7 @@ pub(in crate::http) async fn configure(
         // configure request for tracing purposes.
         state.startup_span = Some(tracing::Span::current());
 
-        if compute.params.lakebase_mode {
-            ComputeNode::set_spec(&compute.params, &mut state, pspec);
-        } else {
-            state.pspec = Some(pspec);
-        }
-
+        state.pspec = Some(pspec);
         state.set_status(ComputeStatus::ConfigurationPending, &compute.state_changed);
         drop(state);
     }

compute_tools/src/http/routes/hadron_liveness_probe.rs

@@ -1,34 +0,0 @@
-use crate::pg_isready::pg_isready;
-use crate::{compute::ComputeNode, http::JsonResponse};
-use axum::{extract::State, http::StatusCode, response::Response};
-use std::sync::Arc;
-
-/// NOTE: NOT ENABLED YET
-/// Detect if the compute is alive.
-/// Called by the liveness probe of the compute container.
-pub(in crate::http) async fn hadron_liveness_probe(
-    State(compute): State<Arc<ComputeNode>>,
-) -> Response {
-    let port = match compute.params.connstr.port() {
-        Some(port) => port,
-        None => {
-            return JsonResponse::error(
-                StatusCode::INTERNAL_SERVER_ERROR,
-                "Failed to get the port from the connection string",
-            );
-        }
-    };
-    match pg_isready(&compute.params.pg_isready_bin, port) {
-        Ok(_) => {
-            // The connection is successful, so the compute is alive.
-            // Return a 200 OK response.
-            JsonResponse::success(StatusCode::OK, "ok")
-        }
-        Err(e) => {
-            tracing::error!("Hadron liveness probe failed: {}", e);
-            // The connection failed, so the compute is not alive.
-            // Return a 500 Internal Server Error response.
-            JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e)
-        }
-    }
-}

compute_tools/src/http/routes/lfc.rs

@@ -1,11 +1,12 @@
+use crate::compute_prewarm::LfcPrewarmStateWithProgress;
 use crate::http::JsonResponse;
 use axum::response::{IntoResponse, Response};
 use axum::{Json, http::StatusCode};
 use axum_extra::extract::OptionalQuery;
-use compute_api::responses::{LfcOffloadState, LfcPrewarmState};
+use compute_api::responses::LfcOffloadState;
 type Compute = axum::extract::State<std::sync::Arc<crate::compute::ComputeNode>>;
 
-pub(in crate::http) async fn prewarm_state(compute: Compute) -> Json<LfcPrewarmState> {
+pub(in crate::http) async fn prewarm_state(compute: Compute) -> Json<LfcPrewarmStateWithProgress> {
     Json(compute.lfc_prewarm_state().await)
 }
 
@@ -45,8 +46,3 @@ pub(in crate::http) async fn offload(compute: Compute) -> Response {
         )
     }
 }
-
-pub(in crate::http) async fn cancel_prewarm(compute: Compute) -> StatusCode {
-    compute.cancel_prewarm();
-    StatusCode::ACCEPTED
-}

compute_tools/src/http/routes/metrics.rs

@@ -13,7 +13,6 @@ use metrics::{Encoder, TextEncoder};
 
 use crate::communicator_socket_client::connect_communicator_socket;
 use crate::compute::ComputeNode;
-use crate::hadron_metrics;
 use crate::http::JsonResponse;
 use crate::metrics::collect;
 
@@ -22,18 +21,11 @@ pub(in crate::http) async fn get_metrics() -> Response {
     // When we call TextEncoder::encode() below, it will immediately return an
     // error if a metric family has no metrics, so we need to preemptively
     // filter out metric families with no metrics.
-    let mut metrics = collect()
+    let metrics = collect()
         .into_iter()
         .filter(|m| !m.get_metric().is_empty())
         .collect::<Vec<MetricFamily>>();
 
-    // Add Hadron metrics.
-    let hadron_metrics: Vec<MetricFamily> = hadron_metrics::collect()
-        .into_iter()
-        .filter(|m| !m.get_metric().is_empty())
-        .collect();
-    metrics.extend(hadron_metrics);
-
     let encoder = TextEncoder::new();
     let mut buffer = vec![];
 

compute_tools/src/http/routes/mod.rs

@@ -10,13 +10,11 @@ pub(in crate::http) mod extension_server;
 pub(in crate::http) mod extensions;
 pub(in crate::http) mod failpoints;
 pub(in crate::http) mod grants;
-pub(in crate::http) mod hadron_liveness_probe;
 pub(in crate::http) mod insights;
 pub(in crate::http) mod lfc;
 pub(in crate::http) mod metrics;
 pub(in crate::http) mod metrics_json;
 pub(in crate::http) mod promote;
-pub(in crate::http) mod refresh_configuration;
 pub(in crate::http) mod status;
 pub(in crate::http) mod terminate;
 

compute_tools/src/http/routes/promote.rs

@@ -1,25 +1,14 @@
 use crate::http::JsonResponse;
-use axum::extract::Json;
-use compute_api::responses::PromoteConfig;
+use axum::Form;
 use http::StatusCode;
 
 pub(in crate::http) async fn promote(
     compute: axum::extract::State<std::sync::Arc<crate::compute::ComputeNode>>,
-    Json(cfg): Json<PromoteConfig>,
+    Form(safekeepers_lsn): Form<compute_api::responses::SafekeepersLsn>,
 ) -> axum::response::Response {
-    // Return early at the cost of extra parsing spec
-    let pspec = match crate::compute::ParsedSpec::try_from(cfg.spec) {
-        Ok(p) => p,
-        Err(e) => return JsonResponse::error(StatusCode::BAD_REQUEST, e),
-    };
-
-    let cfg = PromoteConfig {
-        spec: pspec.spec,
-        wal_flush_lsn: cfg.wal_flush_lsn,
-    };
-    let state = compute.promote(cfg).await;
-    if let compute_api::responses::PromoteState::Failed { error: _ } = state {
-        return JsonResponse::create_response(StatusCode::INTERNAL_SERVER_ERROR, state);
+    let state = compute.promote(safekeepers_lsn).await;
+    if let compute_api::responses::PromoteState::Failed { error } = state {
+        return JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, error);
     }
     JsonResponse::success(StatusCode::OK, state)
 }

compute_tools/src/http/routes/refresh_configuration.rs

@@ -1,29 +0,0 @@
-// This file is added by Hadron
-
-use std::sync::Arc;
-
-use axum::{
-    extract::State,
-    response::{IntoResponse, Response},
-};
-use http::StatusCode;
-
-use crate::compute::ComputeNode;
-use crate::hadron_metrics::POSTGRES_PAGESTREAM_REQUEST_ERRORS;
-use crate::http::JsonResponse;
-
-/// The /refresh_configuration POST method is used to nudge compute_ctl to pull a new spec
-/// from the HCC and attempt to reconfigure Postgres with the new spec. The method does not wait
-/// for the reconfiguration to complete. Rather, it simply delivers a signal that will cause
-/// configuration to be reloaded in a best effort manner. Invocation of this method does not
-/// guarantee that a reconfiguration will occur. The caller should consider keep sending this
-/// request while it believes that the compute configuration is out of date.
-pub(in crate::http) async fn refresh_configuration(
-    State(compute): State<Arc<ComputeNode>>,
-) -> Response {
-    POSTGRES_PAGESTREAM_REQUEST_ERRORS.inc();
-    match compute.signal_refresh_configuration().await {
-        Ok(_) => StatusCode::OK.into_response(),
-        Err(e) => JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e),
-    }
-}

compute_tools/src/http/routes/terminate.rs

@@ -1,7 +1,7 @@
 use crate::compute::{ComputeNode, forward_termination_signal};
 use crate::http::JsonResponse;
 use axum::extract::State;
-use axum::response::{IntoResponse, Response};
+use axum::response::Response;
 use axum_extra::extract::OptionalQuery;
 use compute_api::responses::{ComputeStatus, TerminateMode, TerminateResponse};
 use http::StatusCode;
@@ -33,29 +33,7 @@ pub(in crate::http) async fn terminate(
         if !matches!(state.status, ComputeStatus::Empty | ComputeStatus::Running) {
             return JsonResponse::invalid_status(state.status);
         }
-
-        // If compute is Empty, there's no Postgres to terminate. The regular compute_ctl termination path
-        // assumes Postgres to be configured and running, so we just special-handle this case by exiting
-        // the process directly.
-        if compute.params.lakebase_mode && state.status == ComputeStatus::Empty {
-            drop(state);
-            info!("terminating empty compute - will exit process");
-
-            // Queue a task to exit the process after 5 seconds. The 5-second delay aims to
-            // give enough time for the HTTP response to be sent so that HCM doesn't get an abrupt
-            // connection termination.
-            tokio::spawn(async {
-                tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
-                info!("exiting process after terminating empty compute");
-                std::process::exit(0);
-            });
-
-            return StatusCode::OK.into_response();
-        }
-
-        // For Running status, proceed with normal termination
         state.set_status(mode.into(), &compute.state_changed);
-        drop(state);
     }
 
     forward_termination_signal(false);

compute_tools/src/http/server.rs

@@ -23,8 +23,7 @@ use super::{
     middleware::authorize::Authorize,
     routes::{
         check_writability, configure, database_schema, dbs_and_roles, extension_server, extensions,
-        grants, hadron_liveness_probe, insights, lfc, metrics, metrics_json, promote,
-        refresh_configuration, status, terminate,
+        grants, insights, lfc, metrics, metrics_json, promote, status, terminate,
     },
 };
 use crate::compute::ComputeNode;
@@ -44,7 +43,6 @@ pub enum Server {
         port: u16,
         config: ComputeCtlConfig,
         compute_id: String,
-        instance_id: Option<String>,
     },
 }
 
@@ -69,12 +67,7 @@ impl From<&Server> for Router<Arc<ComputeNode>> {
                         post(extension_server::download_extension),
                     )
                     .route("/extensions", post(extensions::install_extension))
-                    .route("/grants", post(grants::add_grant))
-                    // Hadron: Compute-initiated configuration refresh
-                    .route(
-                        "/refresh_configuration",
-                        post(refresh_configuration::refresh_configuration),
-                    );
+                    .route("/grants", post(grants::add_grant));
 
                 // Add in any testing support
                 if cfg!(feature = "testing") {
@@ -86,10 +79,7 @@ impl From<&Server> for Router<Arc<ComputeNode>> {
                 router
             }
             Server::External {
-                config,
-                compute_id,
-                instance_id,
-                ..
+                config, compute_id, ..
             } => {
                 let unauthenticated_router = Router::<Arc<ComputeNode>>::new()
                     .route("/metrics", get(metrics::get_metrics))
@@ -99,12 +89,7 @@ impl From<&Server> for Router<Arc<ComputeNode>> {
                     );
 
                 let authenticated_router = Router::<Arc<ComputeNode>>::new()
-                    .route(
-                        "/lfc/prewarm",
-                        get(lfc::prewarm_state)
-                            .post(lfc::prewarm)
-                            .delete(lfc::cancel_prewarm),
-                    )
+                    .route("/lfc/prewarm", get(lfc::prewarm_state).post(lfc::prewarm))
                     .route("/lfc/offload", get(lfc::offload_state).post(lfc::offload))
                     .route("/promote", post(promote::promote))
                     .route("/check_writability", post(check_writability::is_writable))
@@ -115,13 +100,8 @@ impl From<&Server> for Router<Arc<ComputeNode>> {
                     .route("/metrics.json", get(metrics_json::get_metrics))
                     .route("/status", get(status::get_status))
                     .route("/terminate", post(terminate::terminate))
-                    .route(
-                        "/hadron_liveness_probe",
-                        get(hadron_liveness_probe::hadron_liveness_probe),
-                    )
                     .layer(AsyncRequireAuthorizationLayer::new(Authorize::new(
                         compute_id.clone(),
-                        instance_id.clone(),
                         config.jwks.clone(),
                     )));
 

compute_tools/src/installed_extensions.rs

@@ -2,7 +2,6 @@ use std::collections::HashMap;
 
 use anyhow::Result;
 use compute_api::responses::{InstalledExtension, InstalledExtensions};
-use once_cell::sync::Lazy;
 use tokio_postgres::error::Error as PostgresError;
 use tokio_postgres::{Client, Config, NoTls};
 
@@ -19,7 +18,7 @@ async fn list_dbs(client: &mut Client) -> Result<Vec<String>, PostgresError> {
         .query(
             "SELECT datname FROM pg_catalog.pg_database
                 WHERE datallowconn
-                AND datconnlimit OPERATOR(pg_catalog.<>) (OPERATOR(pg_catalog.-) 2::pg_catalog.int4)
+                AND datconnlimit <> - 2
                 LIMIT 500",
             &[],
         )
@@ -67,7 +66,7 @@ pub async fn get_installed_extensions(
 
         let extensions: Vec<(String, String, i32)> = client
             .query(
-                "SELECT extname, extversion, extowner::pg_catalog.int4 FROM pg_catalog.pg_extension",
+                "SELECT extname, extversion, extowner::integer FROM pg_catalog.pg_extension",
                 &[],
             )
             .await?
@@ -120,7 +119,3 @@ pub async fn get_installed_extensions(
         extensions: extensions_map.into_values().collect(),
     })
 }
-
-pub fn initialize_metrics() {
-    Lazy::force(&INSTALLED_EXTENSIONS);
-}

compute_tools/src/lib.rs

@@ -16,7 +16,6 @@ pub mod compute_prewarm;
 pub mod compute_promote;
 pub mod disk_quota;
 pub mod extension_server;
-pub mod hadron_metrics;
 pub mod installed_extensions;
 pub mod local_proxy;
 pub mod lsn_lease;
@@ -25,7 +24,6 @@ mod migration;
 pub mod monitor;
 pub mod params;
 pub mod pg_helpers;
-pub mod pg_isready;
 pub mod pgbouncer;
 pub mod rsyslog;
 pub mod spec;

compute_tools/src/logger.rs

@@ -1,10 +1,7 @@
 use std::collections::HashMap;
-use std::sync::{LazyLock, RwLock};
-use tracing::Subscriber;
 use tracing::info;
-use tracing_appender;
+use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::prelude::*;
-use tracing_subscriber::{fmt, layer::SubscriberExt, registry::LookupSpan};
 
 /// Initialize logging to stderr, and OpenTelemetry tracing and exporter.
 ///
@@ -18,44 +15,16 @@ use tracing_subscriber::{fmt, layer::SubscriberExt, registry::LookupSpan};
 ///
 pub fn init_tracing_and_logging(
     default_log_level: &str,
-    log_dir_opt: &Option<String>,
-) -> anyhow::Result<(
-    Option<tracing_utils::Provider>,
-    Option<tracing_appender::non_blocking::WorkerGuard>,
-)> {
+) -> anyhow::Result<Option<tracing_utils::Provider>> {
     // Initialize Logging
     let env_filter = tracing_subscriber::EnvFilter::try_from_default_env()
         .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(default_log_level));
 
-    // Standard output streams
     let fmt_layer = tracing_subscriber::fmt::layer()
         .with_ansi(false)
         .with_target(false)
         .with_writer(std::io::stderr);
 
-    // Logs with file rotation. Files in `$log_dir/pgcctl.yyyy-MM-dd`
-    let (json_to_file_layer, _file_logs_guard) = if let Some(log_dir) = log_dir_opt {
-        std::fs::create_dir_all(log_dir)?;
-        let file_logs_appender = tracing_appender::rolling::RollingFileAppender::builder()
-            .rotation(tracing_appender::rolling::Rotation::DAILY)
-            .filename_prefix("pgcctl")
-            // Lib appends to existing files, so we will keep files for up to 2 days even on restart loops.
-            // At minimum, log-daemon will have 1 day to detect and upload a file (if created right before midnight).
-            .max_log_files(2)
-            .build(log_dir)
-            .expect("Initializing rolling file appender should succeed");
-        let (file_logs_writer, _file_logs_guard) =
-            tracing_appender::non_blocking(file_logs_appender);
-        let json_to_file_layer = tracing_subscriber::fmt::layer()
-            .with_ansi(false)
-            .with_target(false)
-            .event_format(PgJsonLogShapeFormatter)
-            .with_writer(file_logs_writer);
-        (Some(json_to_file_layer), Some(_file_logs_guard))
-    } else {
-        (None, None)
-    };
-
     // Initialize OpenTelemetry
     let provider =
         tracing_utils::init_tracing("compute_ctl", tracing_utils::ExportConfig::default());
@@ -66,13 +35,12 @@ pub fn init_tracing_and_logging(
         .with(env_filter)
         .with(otlp_layer)
         .with(fmt_layer)
-        .with(json_to_file_layer)
         .init();
     tracing::info!("logging and tracing started");
 
     utils::logging::replace_panic_hook_with_tracing_panic_hook().forget();
 
-    Ok((provider, _file_logs_guard))
+    Ok(provider)
 }
 
 /// Replace all newline characters with a special character to make it
@@ -127,157 +95,3 @@ pub fn startup_context_from_env() -> Option<opentelemetry::Context> {
         None
     }
 }
-
-/// Track relevant id's
-const UNKNOWN_IDS: &str = r#""pg_instance_id": "", "pg_compute_id": """#;
-static IDS: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new(UNKNOWN_IDS.to_string()));
-
-pub fn update_ids(instance_id: &Option<String>, compute_id: &Option<String>) -> anyhow::Result<()> {
-    let ids = format!(
-        r#""pg_instance_id": "{}", "pg_compute_id": "{}""#,
-        instance_id.as_ref().map(|s| s.as_str()).unwrap_or_default(),
-        compute_id.as_ref().map(|s| s.as_str()).unwrap_or_default()
-    );
-    let mut guard = IDS
-        .write()
-        .map_err(|e| anyhow::anyhow!("Log set id's rwlock poisoned: {}", e))?;
-    *guard = ids;
-    Ok(())
-}
-
-/// Massage compute_ctl logs into PG json log shape so we can use the same Lumberjack setup.
-struct PgJsonLogShapeFormatter;
-impl<S, N> fmt::format::FormatEvent<S, N> for PgJsonLogShapeFormatter
-where
-    S: Subscriber + for<'a> LookupSpan<'a>,
-    N: for<'a> fmt::format::FormatFields<'a> + 'static,
-{
-    fn format_event(
-        &self,
-        ctx: &fmt::FmtContext<'_, S, N>,
-        mut writer: fmt::format::Writer<'_>,
-        event: &tracing::Event<'_>,
-    ) -> std::fmt::Result {
-        // Format values from the event's metadata, and open message string
-        let metadata = event.metadata();
-        {
-            let ids_guard = IDS.read();
-            let ids = ids_guard
-                .as_ref()
-                .map(|guard| guard.as_str())
-                // Surpress so that we don't lose all uploaded/ file logs if something goes super wrong. We would notice the missing id's.
-                .unwrap_or(UNKNOWN_IDS);
-            write!(
-                &mut writer,
-                r#"{{"timestamp": "{}", "error_severity": "{}", "file_name": "{}", "backend_type": "compute_ctl_self", {}, "message": "#,
-                chrono::Utc::now().format("%Y-%m-%d %H:%M:%S%.3f GMT"),
-                metadata.level(),
-                metadata.target(),
-                ids
-            )?;
-        }
-
-        let mut message = String::new();
-        let message_writer = fmt::format::Writer::new(&mut message);
-
-        // Gather the message
-        ctx.field_format().format_fields(message_writer, event)?;
-
-        // TODO: any better options than to copy-paste this OSS span formatter?
-        // impl<S, N, T> FormatEvent<S, N> for Format<Full, T>
-        // https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/trait.FormatEvent.html#impl-FormatEvent%3CS,+N%3E-for-Format%3CFull,+T%3E
-
-        // write message, close bracket, and new line
-        writeln!(writer, "{}}}", serde_json::to_string(&message).unwrap())
-    }
-}
-
-#[cfg(feature = "testing")]
-#[cfg(test)]
-mod test {
-    use super::*;
-    use std::{cell::RefCell, io};
-
-    // Use thread_local! instead of Mutex for test isolation
-    thread_local! {
-        static WRITER_OUTPUT: RefCell<String> = const { RefCell::new(String::new()) };
-    }
-
-    #[derive(Clone, Default)]
-    struct StaticStringWriter;
-
-    impl io::Write for StaticStringWriter {
-        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
-            let output = String::from_utf8(buf.to_vec()).expect("Invalid UTF-8 in test output");
-            WRITER_OUTPUT.with(|s| s.borrow_mut().push_str(&output));
-            Ok(buf.len())
-        }
-
-        fn flush(&mut self) -> io::Result<()> {
-            Ok(())
-        }
-    }
-
-    impl fmt::MakeWriter<'_> for StaticStringWriter {
-        type Writer = Self;
-
-        fn make_writer(&self) -> Self::Writer {
-            Self
-        }
-    }
-
-    #[test]
-    fn test_log_pg_json_shape_formatter() {
-        // Use a scoped subscriber to prevent global state pollution
-        let subscriber = tracing_subscriber::registry().with(
-            tracing_subscriber::fmt::layer()
-                .with_ansi(false)
-                .with_target(false)
-                .event_format(PgJsonLogShapeFormatter)
-                .with_writer(StaticStringWriter),
-        );
-
-        let _ = update_ids(&Some("000".to_string()), &Some("111".to_string()));
-
-        // Clear any previous test state
-        WRITER_OUTPUT.with(|s| s.borrow_mut().clear());
-
-        let messages = [
-            "test message",
-            r#"json escape check:  name="BatchSpanProcessor.Flush.ExportError" reason="Other(reqwest::Error { kind: Request, url: \"http://localhost:4318/v1/traces\", source: hyper_
-            util::client::legacy::Error(Connect, ConnectError(\"tcp connect error\", Os { code: 111, kind: ConnectionRefused, message: \"Connection refused\" })) })" Failed during the export process"#,
-        ];
-
-        tracing::subscriber::with_default(subscriber, || {
-            for message in messages {
-                tracing::info!(message);
-            }
-        });
-        tracing::info!("not test message");
-
-        // Get captured output
-        let output = WRITER_OUTPUT.with(|s| s.borrow().clone());
-
-        let json_strings: Vec<&str> = output.lines().collect();
-        assert_eq!(
-            json_strings.len(),
-            messages.len(),
-            "Log didn't have the expected number of json strings."
-        );
-
-        let json_string_shape_regex = regex::Regex::new(
-            r#"\{"timestamp": "\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} GMT", "error_severity": "INFO", "file_name": ".+", "backend_type": "compute_ctl_self", "pg_instance_id": "000", "pg_compute_id": "111", "message": ".+"\}"#
-        ).unwrap();
-
-        for (i, expected_message) in messages.iter().enumerate() {
-            let json_string = json_strings[i];
-            assert!(
-                json_string_shape_regex.is_match(json_string),
-                "Json log didn't match expected pattern:\n{json_string}",
-            );
-            let parsed_json: serde_json::Value = serde_json::from_str(json_string).unwrap();
-            let actual_message = parsed_json["message"].as_str().unwrap();
-            assert_eq!(*expected_message, actual_message);
-        }
-    }
-}

compute_tools/src/migration.rs

@@ -76,7 +76,7 @@ impl<'m> MigrationRunner<'m> {
         self.client
             .simple_query("CREATE SCHEMA IF NOT EXISTS neon_migration")
             .await?;
-        self.client.simple_query("CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key pg_catalog.int4 NOT NULL PRIMARY KEY, id pg_catalog.int8 NOT NULL DEFAULT 0)").await?;
+        self.client.simple_query("CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key INT NOT NULL PRIMARY KEY, id bigint NOT NULL DEFAULT 0)").await?;
         self.client
             .simple_query(
                 "INSERT INTO neon_migration.migration_id VALUES (0, 0) ON CONFLICT DO NOTHING",

compute_tools/src/migrations/0002-alter_roles.sql

@@ -15,17 +15,17 @@ DO $$
 DECLARE
     role_name text;
 BEGIN
-    FOR role_name IN SELECT rolname FROM pg_catalog.pg_roles WHERE pg_catalog.pg_has_role(rolname, '{privileged_role_name}', 'member')
+    FOR role_name IN SELECT rolname FROM pg_roles WHERE pg_has_role(rolname, '{privileged_role_name}', 'member')
     LOOP
-        RAISE NOTICE 'EXECUTING ALTER ROLE % INHERIT', pg_catalog.quote_ident(role_name);
-        EXECUTE pg_catalog.format('ALTER ROLE %I INHERIT;', role_name);
+        RAISE NOTICE 'EXECUTING ALTER ROLE % INHERIT', quote_ident(role_name);
+        EXECUTE 'ALTER ROLE ' || quote_ident(role_name) || ' INHERIT';
     END LOOP;
 
-    FOR role_name IN SELECT rolname FROM pg_catalog.pg_roles
+    FOR role_name IN SELECT rolname FROM pg_roles
         WHERE
-            NOT pg_catalog.pg_has_role(rolname, '{privileged_role_name}', 'member') AND NOT pg_catalog.starts_with(rolname, 'pg_')
+            NOT pg_has_role(rolname, '{privileged_role_name}', 'member') AND NOT starts_with(rolname, 'pg_')
     LOOP
-        RAISE NOTICE 'EXECUTING ALTER ROLE % NOBYPASSRLS', pg_catalog.quote_ident(role_name);
-        EXECUTE pg_catalog.format('ALTER ROLE %I NOBYPASSRLS;', role_name);
+        RAISE NOTICE 'EXECUTING ALTER ROLE % NOBYPASSRLS', quote_ident(role_name);
+        EXECUTE 'ALTER ROLE ' || quote_ident(role_name) || ' NOBYPASSRLS';
     END LOOP;
 END $$;

compute_tools/src/migrations/0003-grant_pg_create_subscription_to_privileged_role.sql

@@ -1,6 +1,6 @@
 DO $$
 BEGIN
-    IF (SELECT setting::pg_catalog.numeric >= 160000 FROM pg_catalog.pg_settings WHERE name = 'server_version_num') THEN
+    IF (SELECT setting::numeric >= 160000 FROM pg_settings WHERE name = 'server_version_num') THEN
         EXECUTE 'GRANT pg_create_subscription TO {privileged_role_name}';
     END IF;
 END $$;

compute_tools/src/migrations/0009-revoke_replication_for_previously_allowed_roles.sql

@@ -5,9 +5,9 @@ DO $$
 DECLARE
     role_name TEXT;
 BEGIN
-    FOR role_name IN SELECT rolname FROM pg_catalog.pg_roles WHERE rolreplication IS TRUE
+    FOR role_name IN SELECT rolname FROM pg_roles WHERE rolreplication IS TRUE
     LOOP
-        RAISE NOTICE 'EXECUTING ALTER ROLE % NOREPLICATION', pg_catalog.quote_ident(role_name);
-        EXECUTE pg_catalog.format('ALTER ROLE %I NOREPLICATION;', role_name);
+        RAISE NOTICE 'EXECUTING ALTER ROLE % NOREPLICATION', quote_ident(role_name);
+        EXECUTE 'ALTER ROLE ' || quote_ident(role_name) || ' NOREPLICATION';
     END LOOP;
 END $$;

compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_privileged_role.sql

@@ -1,6 +1,6 @@
 DO $$
 BEGIN
-    IF (SELECT setting::pg_catalog.numeric >= 160000 FROM pg_catalog.pg_settings WHERE name OPERATOR(pg_catalog.=) 'server_version_num'::pg_catalog.text) THEN
+    IF (SELECT setting::numeric >= 160000 FROM pg_settings WHERE name = 'server_version_num') THEN
        EXECUTE 'GRANT EXECUTE ON FUNCTION pg_export_snapshot TO {privileged_role_name}';
        EXECUTE 'GRANT EXECUTE ON FUNCTION pg_log_standby_snapshot TO {privileged_role_name}';
     END IF;

compute_tools/src/migrations/tests/0001-add_bypass_rls_to_privileged_role.sql

@@ -2,7 +2,7 @@ DO $$
 DECLARE
     bypassrls boolean;
 BEGIN
-    SELECT rolbypassrls INTO bypassrls FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser';
+    SELECT rolbypassrls INTO bypassrls FROM pg_roles WHERE rolname = 'neon_superuser';
     IF NOT bypassrls THEN
         RAISE EXCEPTION 'neon_superuser cannot bypass RLS';
     END IF;

compute_tools/src/migrations/tests/0002-alter_roles.sql

@@ -4,8 +4,8 @@ DECLARE
 BEGIN
     FOR role IN
         SELECT rolname AS name, rolinherit AS inherit
-        FROM pg_catalog.pg_roles
-        WHERE pg_catalog.pg_has_role(rolname, 'neon_superuser', 'member')
+        FROM pg_roles
+        WHERE pg_has_role(rolname, 'neon_superuser', 'member')
     LOOP
         IF NOT role.inherit THEN
             RAISE EXCEPTION '% cannot inherit', quote_ident(role.name);
@@ -14,12 +14,12 @@ BEGIN
 
     FOR role IN
         SELECT rolname AS name, rolbypassrls AS bypassrls
-        FROM pg_catalog.pg_roles
-        WHERE NOT pg_catalog.pg_has_role(rolname, 'neon_superuser', 'member')
-            AND NOT pg_catalog.starts_with(rolname, 'pg_')
+        FROM pg_roles
+        WHERE NOT pg_has_role(rolname, 'neon_superuser', 'member')
+            AND NOT starts_with(rolname, 'pg_')
     LOOP
         IF role.bypassrls THEN
-            RAISE EXCEPTION  '% can bypass RLS', pg_catalog.quote_ident(role.name);
+            RAISE EXCEPTION  '% can bypass RLS', quote_ident(role.name);
         END IF;
     END LOOP;
 END $$;

compute_tools/src/migrations/tests/0003-grant_pg_create_subscription_to_privileged_role.sql

@@ -1,10 +1,10 @@
 DO $$
 BEGIN
-    IF (SELECT pg_catalog.current_setting('server_version_num')::pg_catalog.numeric < 160000) THEN
+    IF (SELECT current_setting('server_version_num')::numeric < 160000) THEN
         RETURN;
     END IF;
 
-    IF NOT (SELECT pg_catalog.pg_has_role('neon_superuser', 'pg_create_subscription', 'member')) THEN
+    IF NOT (SELECT pg_has_role('neon_superuser', 'pg_create_subscription', 'member')) THEN
         RAISE EXCEPTION 'neon_superuser cannot execute pg_create_subscription';
     END IF;
 END $$;

compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_privileged_role.sql

@@ -2,12 +2,12 @@ DO $$
 DECLARE
     monitor record;
 BEGIN
-    SELECT pg_catalog.pg_has_role('neon_superuser', 'pg_monitor', 'member') AS member,
+    SELECT pg_has_role('neon_superuser', 'pg_monitor', 'member') AS member,
             admin_option AS admin
         INTO monitor
-        FROM pg_catalog.pg_auth_members
-        WHERE roleid = 'pg_monitor'::pg_catalog.regrole
-            AND member = 'neon_superuser'::pg_catalog.regrole;
+        FROM pg_auth_members
+        WHERE roleid = 'pg_monitor'::regrole
+            AND member = 'neon_superuser'::regrole;
 
     IF monitor IS NULL THEN
         RAISE EXCEPTION 'no entry in pg_auth_members for neon_superuser and pg_monitor';

compute_tools/src/migrations/tests/0010-grant_snapshot_synchronization_funcs_to_privileged_role.sql

@@ -2,11 +2,11 @@ DO $$
 DECLARE
     can_execute boolean;
 BEGIN
-    SELECT pg_catalog.bool_and(pg_catalog.has_function_privilege('neon_superuser', oid, 'execute'))
+    SELECT bool_and(has_function_privilege('neon_superuser', oid, 'execute'))
        INTO can_execute
-       FROM pg_catalog.pg_proc
+       FROM pg_proc
        WHERE proname IN ('pg_export_snapshot', 'pg_log_standby_snapshot')
-           AND pronamespace = 'pg_catalog'::pg_catalog.regnamespace;
+           AND pronamespace = 'pg_catalog'::regnamespace;
     IF NOT can_execute THEN
         RAISE EXCEPTION 'neon_superuser cannot execute both pg_export_snapshot and pg_log_standby_snapshot';
     END IF;

compute_tools/src/migrations/tests/0011-grant_pg_show_replication_origin_status_to_privileged_role.sql

@@ -2,9 +2,9 @@ DO $$
 DECLARE
     can_execute boolean;
 BEGIN
-    SELECT pg_catalog.has_function_privilege('neon_superuser', oid, 'execute')
+    SELECT has_function_privilege('neon_superuser', oid, 'execute')
        INTO can_execute
-       FROM pg_catalog.pg_proc
+       FROM pg_proc
        WHERE proname = 'pg_show_replication_origin_status'
            AND pronamespace = 'pg_catalog'::regnamespace;
     IF NOT can_execute THEN

compute_tools/src/migrations/tests/0012-grant_pg_signal_backend_to_privileged_role.sql

@@ -2,10 +2,10 @@ DO $$
 DECLARE
     signal_backend record;
 BEGIN
-    SELECT pg_catalog.pg_has_role('neon_superuser', 'pg_signal_backend', 'member') AS member,
+    SELECT pg_has_role('neon_superuser', 'pg_signal_backend', 'member') AS member,
             admin_option AS admin
         INTO signal_backend
-        FROM pg_catalog.pg_auth_members
+        FROM pg_auth_members
         WHERE roleid = 'pg_signal_backend'::regrole
             AND member = 'neon_superuser'::regrole;
 

compute_tools/src/monitor.rs

@@ -407,9 +407,9 @@ fn get_database_stats(cli: &mut Client) -> anyhow::Result<(f64, i64)> {
     // like `postgres_exporter` use it to query Postgres statistics.
     // Use explicit 8 bytes type casts to match Rust types.
     let stats = cli.query_one(
-        "SELECT pg_catalog.coalesce(pg_catalog.sum(active_time), 0.0)::pg_catalog.float8 AS total_active_time,
-            pg_catalog.coalesce(pg_catalog.sum(sessions), 0)::pg_catalog.bigint AS total_sessions
-        FROM pg_catalog.pg_stat_database
+        "SELECT coalesce(sum(active_time), 0.0)::float8 AS total_active_time,
+            coalesce(sum(sessions), 0)::bigint AS total_sessions
+        FROM pg_stat_database
         WHERE datname NOT IN (
                 'postgres',
                 'template0',
@@ -445,11 +445,11 @@ fn get_backends_state_change(cli: &mut Client) -> anyhow::Result<Option<DateTime
     let mut last_active: Option<DateTime<Utc>> = None;
     // Get all running client backends except ourself, use RFC3339 DateTime format.
     let backends = cli.query(
-        "SELECT state, pg_catalog.to_char(state_change, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"'::pg_catalog.text) AS state_change
+        "SELECT state, to_char(state_change, 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') AS state_change
                 FROM pg_stat_activity
-                    WHERE backend_type OPERATOR(pg_catalog.=) 'client backend'::pg_catalog.text
-                    AND pid OPERATOR(pg_catalog.!=) pg_catalog.pg_backend_pid()
-                    AND usename OPERATOR(pg_catalog.!=) 'cloud_admin'::pg_catalog.name;", // XXX: find a better way to filter other monitors?
+                    WHERE backend_type = 'client backend'
+                    AND pid != pg_backend_pid()
+                    AND usename != 'cloud_admin';", // XXX: find a better way to filter other monitors?
         &[],
     );
 

compute_tools/src/pg_helpers.rs

@@ -299,9 +299,9 @@ pub async fn get_existing_dbs_async(
         .query_raw::<str, &String, &[String; 0]>(
             "SELECT
                 datname AS name,
-                (SELECT rolname FROM pg_catalog.pg_roles WHERE oid OPERATOR(pg_catalog.=) datdba) AS owner,
+                (SELECT rolname FROM pg_roles WHERE oid = datdba) AS owner,
                 NOT datallowconn AS restrict_conn,
-                datconnlimit OPERATOR(pg_catalog.=) (OPERATOR(pg_catalog.-) 2) AS invalid
+                datconnlimit = - 2 AS invalid
             FROM
                 pg_catalog.pg_database;",
             &[],

compute_tools/src/pg_isready.rs

@@ -1,30 +0,0 @@
-use anyhow::{Context, anyhow};
-
-// Run `/usr/local/bin/pg_isready -p {port}`
-// Check the connectivity of PG
-// Success means PG is listening on the port and accepting connections
-// Note that PG does not need to authenticate the connection, nor reserve a connection quota for it.
-// See https://www.postgresql.org/docs/current/app-pg-isready.html
-pub fn pg_isready(bin: &str, port: u16) -> anyhow::Result<()> {
-    let child_result = std::process::Command::new(bin)
-        .arg("-p")
-        .arg(port.to_string())
-        .spawn();
-
-    child_result
-        .context("spawn() failed")
-        .and_then(|mut child| child.wait().context("wait() failed"))
-        .and_then(|status| match status.success() {
-            true => Ok(()),
-            false => Err(anyhow!("process exited with {status}")),
-        })
-        // wrap any prior error with the overall context that we couldn't run the command
-        .with_context(|| format!("could not run `{bin} --port {port}`"))
-}
-
-// It's safe to assume pg_isready is under the same directory with postgres,
-// because it is a PG util bin installed along with postgres
-pub fn get_pg_isready_bin(pgbin: &str) -> String {
-    let split = pgbin.split("/").collect::<Vec<&str>>();
-    split[0..split.len() - 1].join("/") + "/pg_isready"
-}

compute_tools/src/spec.rs

@@ -142,7 +142,7 @@ pub fn update_pg_hba(pgdata_path: &Path, databricks_pg_hba: Option<&String>) ->
     // Update pg_hba to contains databricks specfic settings before adding neon settings
     // PG uses the first record that matches to perform authentication, so we need to have
     // our rules before the default ones from neon.
-    // See https://www.postgresql.org/docs/current/auth-pg-hba-conf.html
+    // See https://www.postgresql.org/docs/16/auth-pg-hba-conf.html
     if let Some(databricks_pg_hba) = databricks_pg_hba {
         if config::line_in_file(
             &pghba_path,

compute_tools/src/spec_apply.rs

@@ -13,19 +13,17 @@ use tokio_postgres::Client;
 use tokio_postgres::error::SqlState;
 use tracing::{Instrument, debug, error, info, info_span, instrument, warn};
 
-use crate::compute::{ComputeNode, ComputeNodeParams, ComputeState, create_databricks_roles};
-use crate::hadron_metrics::COMPUTE_CONFIGURE_STATEMENT_TIMEOUT_ERRORS;
+use crate::compute::{ComputeNode, ComputeNodeParams, ComputeState};
 use crate::pg_helpers::{
     DatabaseExt, Escaping, GenericOptionsSearch, RoleExt, get_existing_dbs_async,
     get_existing_roles_async,
 };
 use crate::spec_apply::ApplySpecPhase::{
-    AddDatabricksGrants, AlterDatabricksRoles, CreateAndAlterDatabases, CreateAndAlterRoles,
-    CreateAvailabilityCheck, CreateDatabricksMisc, CreateDatabricksRoles, CreatePgauditExtension,
+    CreateAndAlterDatabases, CreateAndAlterRoles, CreateAvailabilityCheck, CreatePgauditExtension,
     CreatePgauditlogtofileExtension, CreatePrivilegedRole, CreateSchemaNeon,
     DisablePostgresDBPgAudit, DropInvalidDatabases, DropRoles, FinalizeDropLogicalSubscriptions,
-    HandleDatabricksAuthExtension, HandleNeonExtension, HandleOtherExtensions,
-    RenameAndDeleteDatabases, RenameRoles, RunInEachDatabase,
+    HandleNeonExtension, HandleOtherExtensions, RenameAndDeleteDatabases, RenameRoles,
+    RunInEachDatabase,
 };
 use crate::spec_apply::PerDatabasePhase::{
     ChangeSchemaPerms, DeleteDBRoleReferences, DropLogicalSubscriptions,
@@ -82,7 +80,7 @@ impl ComputeNode {
                 info!("Checking if drop subscription operation was already performed for timeline_id: {}", timeline_id);
 
                 drop_subscriptions_done = match
-                    client.query("select 1 from neon.drop_subscriptions_done where timeline_id OPERATOR(pg_catalog.=) $1", &[&timeline_id.to_string()]).await {
+                    client.query("select 1 from neon.drop_subscriptions_done where timeline_id = $1", &[&timeline_id.to_string()]).await {
                     Ok(result) => !result.is_empty(),
                     Err(e) =>
                     {
@@ -168,7 +166,6 @@ impl ComputeNode {
                         concurrency_token.clone(),
                         db,
                         [DropLogicalSubscriptions].to_vec(),
-                        self.params.lakebase_mode,
                     );
 
                     Ok(tokio::spawn(fut))
@@ -189,33 +186,15 @@ impl ComputeNode {
                 };
             }
 
-            let phases = if self.params.lakebase_mode {
-                vec![
-                    CreatePrivilegedRole,
-                // BEGIN_HADRON
-                CreateDatabricksRoles,
-                AlterDatabricksRoles,
-                // END_HADRON
+            for phase in [
+                CreatePrivilegedRole,
                 DropInvalidDatabases,
                 RenameRoles,
                 CreateAndAlterRoles,
                 RenameAndDeleteDatabases,
                 CreateAndAlterDatabases,
                 CreateSchemaNeon,
-            ]
-            } else {
-                vec![
-                    CreatePrivilegedRole,
-                DropInvalidDatabases,
-                RenameRoles,
-                CreateAndAlterRoles,
-                RenameAndDeleteDatabases,
-                CreateAndAlterDatabases,
-                CreateSchemaNeon,
-            ]
-            };
-
-            for phase in phases {
+            ] {
                 info!("Applying phase {:?}", &phase);
                 apply_operations(
                     params.clone(),
@@ -224,7 +203,6 @@ impl ComputeNode {
                     jwks_roles.clone(),
                     phase,
                     || async { Ok(&client) },
-                    self.params.lakebase_mode,
                 )
                 .await?;
             }
@@ -276,7 +254,6 @@ impl ComputeNode {
                         concurrency_token.clone(),
                         db,
                         phases,
-                        self.params.lakebase_mode,
                     );
 
                     Ok(tokio::spawn(fut))
@@ -288,28 +265,12 @@ impl ComputeNode {
                 handle.await??;
             }
 
-            let mut phases = if self.params.lakebase_mode {
-                vec![
-                HandleOtherExtensions,
-                HandleNeonExtension, // This step depends on CreateSchemaNeon
-                // BEGIN_HADRON
-                HandleDatabricksAuthExtension,
-                // END_HADRON
-                CreateAvailabilityCheck,
-                DropRoles,
-                // BEGIN_HADRON
-                AddDatabricksGrants,
-                CreateDatabricksMisc,
-                // END_HADRON
-            ]
-            } else {
-                vec![
+            let mut phases = vec![
                 HandleOtherExtensions,
                 HandleNeonExtension, // This step depends on CreateSchemaNeon
                 CreateAvailabilityCheck,
                 DropRoles,
-            ]
-            };
+            ];
 
             // This step depends on CreateSchemaNeon
             if spec.drop_subscriptions_before_start && !drop_subscriptions_done {
@@ -342,7 +303,6 @@ impl ComputeNode {
                     jwks_roles.clone(),
                     phase,
                     || async { Ok(&client) },
-                    self.params.lakebase_mode,
                 )
                 .await?;
             }
@@ -368,7 +328,6 @@ impl ComputeNode {
         concurrency_token: Arc<tokio::sync::Semaphore>,
         db: DB,
         subphases: Vec<PerDatabasePhase>,
-        lakebase_mode: bool,
     ) -> Result<()> {
         let _permit = concurrency_token.acquire().await?;
 
@@ -396,7 +355,6 @@ impl ComputeNode {
                     let client = client_conn.as_ref().unwrap();
                     Ok(client)
                 },
-                lakebase_mode,
             )
             .await?;
         }
@@ -519,10 +477,6 @@ pub enum PerDatabasePhase {
 #[derive(Clone, Debug)]
 pub enum ApplySpecPhase {
     CreatePrivilegedRole,
-    // BEGIN_HADRON
-    CreateDatabricksRoles,
-    AlterDatabricksRoles,
-    // END_HADRON
     DropInvalidDatabases,
     RenameRoles,
     CreateAndAlterRoles,
@@ -535,14 +489,7 @@ pub enum ApplySpecPhase {
     DisablePostgresDBPgAudit,
     HandleOtherExtensions,
     HandleNeonExtension,
-    // BEGIN_HADRON
-    HandleDatabricksAuthExtension,
-    // END_HADRON
     CreateAvailabilityCheck,
-    // BEGIN_HADRON
-    AddDatabricksGrants,
-    CreateDatabricksMisc,
-    // END_HADRON
     DropRoles,
     FinalizeDropLogicalSubscriptions,
 }
@@ -578,7 +525,6 @@ pub async fn apply_operations<'a, Fut, F>(
     jwks_roles: Arc<HashSet<String>>,
     apply_spec_phase: ApplySpecPhase,
     client: F,
-    lakebase_mode: bool,
 ) -> Result<()>
 where
     F: FnOnce() -> Fut,
@@ -625,23 +571,6 @@ where
                         },
                         query
                     );
-                    if !lakebase_mode {
-                        return res;
-                    }
-                    // BEGIN HADRON
-                    if let Err(e) = res.as_ref() {
-                        if let Some(sql_state) = e.code() {
-                            if sql_state.code() == "57014" {
-                                // SQL State 57014 (ERRCODE_QUERY_CANCELED) is used for statement timeouts.
-                                // Increment the counter whenever a statement timeout occurs. Timeouts on
-                                // this configuration path can only occur due to PS connectivity problems that
-                                // Postgres failed to recover from.
-                                COMPUTE_CONFIGURE_STATEMENT_TIMEOUT_ERRORS.inc();
-                            }
-                        }
-                    }
-                    // END HADRON
-
                     res
                 }
                 .instrument(inspan)
@@ -679,44 +608,10 @@ async fn get_operations<'a>(
         ApplySpecPhase::CreatePrivilegedRole => Ok(Box::new(once(Operation {
             query: format!(
                 include_str!("sql/create_privileged_role.sql"),
-                privileged_role_name = params.privileged_role_name,
-                privileges = if params.lakebase_mode {
-                    "CREATEDB CREATEROLE NOLOGIN BYPASSRLS"
-                } else {
-                    "CREATEDB CREATEROLE NOLOGIN REPLICATION BYPASSRLS"
-                }
+                privileged_role_name = params.privileged_role_name
             ),
             comment: None,
         }))),
-        // BEGIN_HADRON
-        // New Hadron phase
-        ApplySpecPhase::CreateDatabricksRoles => {
-            let queries = create_databricks_roles();
-            let operations = queries.into_iter().map(|query| Operation {
-                query,
-                comment: None,
-            });
-            Ok(Box::new(operations))
-        }
-
-        // Backfill existing databricks_reader_* roles with statement timeout from GUC
-        ApplySpecPhase::AlterDatabricksRoles => {
-            let query = String::from(include_str!(
-                "sql/alter_databricks_reader_roles_timeout.sql"
-            ));
-
-            let operations = once(Operation {
-                query,
-                comment: Some(
-                    "Backfill existing databricks_reader_* roles with statement timeout"
-                        .to_string(),
-                ),
-            });
-
-            Ok(Box::new(operations))
-        }
-        // End of new Hadron Phase
-        // END_HADRON
         ApplySpecPhase::DropInvalidDatabases => {
             let mut ctx = ctx.write().await;
             let databases = &mut ctx.dbs;
@@ -1086,10 +981,7 @@ async fn get_operations<'a>(
                                         // N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`
                                         role_name = escaped_role,
                                         outer_tag = outer_tag,
-                                    )
-                                    // HADRON change:
-                                    .replace("neon_superuser", &params.privileged_role_name),
-                                    // HADRON change end                                    ,
+                                    ),
                                     comment: None,
                                 },
                                 // This now will only drop privileges of the role
@@ -1125,8 +1017,7 @@ async fn get_operations<'a>(
                             comment: None,
                         },
                         Operation {
-                            query: String::from(include_str!("sql/default_grants.sql"))
-                                .replace("neon_superuser", &params.privileged_role_name),
+                            query: String::from(include_str!("sql/default_grants.sql")),
                             comment: None,
                         },
                     ]
@@ -1142,9 +1033,7 @@ async fn get_operations<'a>(
             if let Some(libs) = spec.cluster.settings.find("shared_preload_libraries") {
                 if libs.contains("pg_stat_statements") {
                     return Ok(Box::new(once(Operation {
-                        query: String::from(
-                            "CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA public",
-                        ),
+                        query: String::from("CREATE EXTENSION IF NOT EXISTS pg_stat_statements"),
                         comment: Some(String::from("create system extensions")),
                     })));
                 }
@@ -1152,13 +1041,11 @@ async fn get_operations<'a>(
             Ok(Box::new(empty()))
         }
         ApplySpecPhase::CreatePgauditExtension => Ok(Box::new(once(Operation {
-            query: String::from("CREATE EXTENSION IF NOT EXISTS pgaudit WITH SCHEMA public"),
+            query: String::from("CREATE EXTENSION IF NOT EXISTS pgaudit"),
             comment: Some(String::from("create pgaudit extensions")),
         }))),
         ApplySpecPhase::CreatePgauditlogtofileExtension => Ok(Box::new(once(Operation {
-            query: String::from(
-                "CREATE EXTENSION IF NOT EXISTS pgauditlogtofile WITH SCHEMA public",
-            ),
+            query: String::from("CREATE EXTENSION IF NOT EXISTS pgauditlogtofile"),
             comment: Some(String::from("create pgauditlogtofile extensions")),
         }))),
         // Disable pgaudit logging for postgres database.
@@ -1182,7 +1069,7 @@ async fn get_operations<'a>(
                 },
                 Operation {
                     query: String::from(
-                        "UPDATE pg_catalog.pg_extension SET extrelocatable = true WHERE extname OPERATOR(pg_catalog.=) 'neon'::pg_catalog.name AND extrelocatable OPERATOR(pg_catalog.=) false",
+                        "UPDATE pg_extension SET extrelocatable = true WHERE extname = 'neon'",
                     ),
                     comment: Some(String::from("compat/fix: make neon relocatable")),
                 },
@@ -1199,28 +1086,6 @@ async fn get_operations<'a>(
 
             Ok(Box::new(operations))
         }
-        // BEGIN_HADRON
-        // Note: we may want to version the extension someday, but for now we just drop it and recreate it.
-        ApplySpecPhase::HandleDatabricksAuthExtension => {
-            let operations = vec![
-                Operation {
-                    query: String::from("DROP EXTENSION IF EXISTS databricks_auth"),
-                    comment: Some(String::from("dropping existing databricks_auth extension")),
-                },
-                Operation {
-                    query: String::from("CREATE EXTENSION databricks_auth"),
-                    comment: Some(String::from("creating databricks_auth extension")),
-                },
-                Operation {
-                    query: String::from("GRANT SELECT ON databricks_auth_metrics TO pg_monitor"),
-                    comment: Some(String::from("grant select on databricks auth counters")),
-                },
-            ]
-            .into_iter();
-
-            Ok(Box::new(operations))
-        }
-        // END_HADRON
         ApplySpecPhase::CreateAvailabilityCheck => Ok(Box::new(once(Operation {
             query: String::from(include_str!("sql/add_availabilitycheck_tables.sql")),
             comment: None,
@@ -1238,63 +1103,6 @@ async fn get_operations<'a>(
 
             Ok(Box::new(operations))
         }
-
-        // BEGIN_HADRON
-        // New Hadron phases
-        //
-        // Grants permissions to roles that are used by Databricks.
-        ApplySpecPhase::AddDatabricksGrants => {
-            let operations = vec![
-                Operation {
-                    query: String::from("GRANT USAGE ON SCHEMA neon TO databricks_monitor"),
-                    comment: Some(String::from(
-                        "Permissions needed to execute neon.* functions (in the postgres database)",
-                    )),
-                },
-                Operation {
-                    query: String::from(
-                        "GRANT SELECT, INSERT, UPDATE ON health_check TO databricks_monitor",
-                    ),
-                    comment: Some(String::from("Permissions needed for read and write probes")),
-                },
-                Operation {
-                    query: String::from(
-                        "GRANT EXECUTE ON FUNCTION pg_ls_dir(text) TO databricks_monitor",
-                    ),
-                    comment: Some(String::from(
-                        "Permissions needed to monitor .snap file counts",
-                    )),
-                },
-                Operation {
-                    query: String::from(
-                        "GRANT SELECT ON neon.neon_perf_counters TO databricks_monitor",
-                    ),
-                    comment: Some(String::from(
-                        "Permissions needed to access neon performance counters view",
-                    )),
-                },
-                Operation {
-                    query: String::from(
-                        "GRANT EXECUTE ON FUNCTION neon.get_perf_counters() TO databricks_monitor",
-                    ),
-                    comment: Some(String::from(
-                        "Permissions needed to execute the underlying performance counters function",
-                    )),
-                },
-            ]
-            .into_iter();
-
-            Ok(Box::new(operations))
-        }
-        // Creates minor objects that are used by Databricks.
-        ApplySpecPhase::CreateDatabricksMisc => Ok(Box::new(once(Operation {
-            query: String::from(include_str!("sql/create_databricks_misc.sql")),
-            comment: Some(String::from(
-                "The function databricks_monitor uses to convert exception to 0 or 1",
-            )),
-        }))),
-        // End of new Hadron phases
-        // END_HADRON
         ApplySpecPhase::FinalizeDropLogicalSubscriptions => Ok(Box::new(once(Operation {
             query: String::from(include_str!("sql/finalize_drop_subscriptions.sql")),
             comment: None,

compute_tools/src/sql/add_availabilitycheck_tables.sql

@@ -3,17 +3,16 @@ BEGIN
     IF NOT EXISTS(
         SELECT 1
         FROM pg_catalog.pg_tables
-        WHERE tablename::pg_catalog.name OPERATOR(pg_catalog.=) 'health_check'::pg_catalog.name
-        AND schemaname::pg_catalog.name OPERATOR(pg_catalog.=) 'public'::pg_catalog.name
+        WHERE tablename = 'health_check'
     )
     THEN
-    CREATE TABLE public.health_check (
-        id pg_catalog.int4 primary key generated by default as identity,
-        updated_at pg_catalog.timestamptz default pg_catalog.now()
+    CREATE TABLE health_check (
+        id serial primary key,
+        updated_at timestamptz default now()
     );
-    INSERT INTO public.health_check VALUES (1, pg_catalog.now())
+    INSERT INTO health_check VALUES (1, now())
         ON CONFLICT (id) DO UPDATE
-         SET updated_at = pg_catalog.now();
+         SET updated_at = now();
     END IF;
 END
 $$

compute_tools/src/sql/alter_databricks_reader_roles_timeout.sql

@@ -1,25 +0,0 @@
-DO $$
-DECLARE
-    reader_role RECORD;
-    timeout_value TEXT;
-BEGIN
-    -- Get the current GUC setting for reader statement timeout
-    SELECT current_setting('databricks.reader_statement_timeout', true) INTO timeout_value;
-    
-    -- Only proceed if timeout_value is not null/empty and not '0' (disabled)
-    IF timeout_value IS NOT NULL AND timeout_value != '' AND timeout_value != '0' THEN
-        -- Find all databricks_reader_* roles and update their statement_timeout
-        FOR reader_role IN 
-            SELECT r.rolname
-            FROM pg_roles r
-            WHERE r.rolname ~ '^databricks_reader_\d+$'
-        LOOP
-            -- Apply the timeout setting to the role (will overwrite existing setting)
-            EXECUTE format('ALTER ROLE %I SET statement_timeout = %L', 
-                         reader_role.rolname, timeout_value);
-            
-            RAISE LOG 'Updated statement_timeout = % for role %', timeout_value, reader_role.rolname;
-        END LOOP;
-    END IF;
-END
-$$;

compute_tools/src/sql/anon_ext_fn_reassign.sql

@@ -0,0 +1,12 @@
+DO $$
+DECLARE
+    query varchar;
+BEGIN
+    FOR query IN SELECT 'ALTER FUNCTION '||nsp.nspname||'.'||p.proname||'('||pg_get_function_identity_arguments(p.oid)||') OWNER TO {db_owner};'
+    FROM pg_proc p
+        JOIN pg_namespace nsp ON p.pronamespace = nsp.oid
+    WHERE nsp.nspname = 'anon' LOOP
+        EXECUTE query;
+    END LOOP;
+END
+$$;

compute_tools/src/sql/create_databricks_misc.sql

@@ -1,15 +0,0 @@
-ALTER ROLE databricks_monitor SET statement_timeout = '60s';
-
-CREATE OR REPLACE FUNCTION health_check_write_succeeds()
-RETURNS INTEGER AS $$
-BEGIN
-INSERT INTO health_check VALUES (1, now())
-ON CONFLICT (id) DO UPDATE
-    SET updated_at = now();
-
-RETURN 1;
-EXCEPTION WHEN OTHERS THEN
-RAISE EXCEPTION '[DATABRICKS_SMGR] health_check failed: [%] %', SQLSTATE, SQLERRM;
-RETURN 0;
-END;
-$$ LANGUAGE plpgsql;

compute_tools/src/sql/create_privileged_role.sql

@@ -1,8 +1,8 @@
 DO $$
     BEGIN
-        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname OPERATOR(pg_catalog.=) '{privileged_role_name}'::pg_catalog.name)
+        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{privileged_role_name}')
         THEN
-            CREATE ROLE {privileged_role_name} {privileges} IN ROLE pg_read_all_data, pg_write_all_data;
+            CREATE ROLE {privileged_role_name} CREATEDB CREATEROLE NOLOGIN REPLICATION BYPASSRLS IN ROLE pg_read_all_data, pg_write_all_data;
         END IF;
     END
 $$;

compute_tools/src/sql/default_grants.sql

@@ -4,14 +4,14 @@ $$
         IF EXISTS(
             SELECT nspname
             FROM pg_catalog.pg_namespace
-            WHERE nspname OPERATOR(pg_catalog.=) 'public'
+            WHERE nspname = 'public'
         ) AND
-           pg_catalog.current_setting('server_version_num')::int OPERATOR(pg_catalog./) 10000 OPERATOR(pg_catalog.>=) 15
+           current_setting('server_version_num')::int / 10000 >= 15
         THEN
             IF EXISTS(
                 SELECT rolname
                 FROM pg_catalog.pg_roles
-                WHERE rolname OPERATOR(pg_catalog.=) 'web_access'
+                WHERE rolname = 'web_access'
             )
             THEN
                 GRANT CREATE ON SCHEMA public TO web_access;
@@ -20,7 +20,7 @@ $$
         IF EXISTS(
             SELECT nspname
             FROM pg_catalog.pg_namespace
-            WHERE nspname OPERATOR(pg_catalog.=) 'public'
+            WHERE nspname = 'public'
         )
         THEN
             ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO neon_superuser WITH GRANT OPTION;

compute_tools/src/sql/drop_subscriptions.sql

@@ -2,17 +2,11 @@ DO ${outer_tag}$
 DECLARE
     subname TEXT;
 BEGIN
-    LOCK TABLE pg_catalog.pg_subscription IN ACCESS EXCLUSIVE MODE;
-    FOR subname IN
-        SELECT pg_subscription.subname
-        FROM pg_catalog.pg_subscription
-        WHERE subdbid OPERATOR(pg_catalog.=) (
-            SELECT oid FROM pg_database WHERE datname OPERATOR(pg_catalog.=) {datname_str}::pg_catalog.name
-        )
-    LOOP
-        EXECUTE pg_catalog.format('ALTER SUBSCRIPTION %I DISABLE;', subname);
-        EXECUTE pg_catalog.format('ALTER SUBSCRIPTION %I SET (slot_name = NONE);', subname);
-        EXECUTE pg_catalog.format('DROP SUBSCRIPTION %I;', subname);
+    LOCK TABLE pg_subscription IN ACCESS EXCLUSIVE MODE;
+    FOR subname IN SELECT pg_subscription.subname FROM pg_subscription WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = {datname_str}) LOOP
+        EXECUTE format('ALTER SUBSCRIPTION %I DISABLE;', subname);
+        EXECUTE format('ALTER SUBSCRIPTION %I SET (slot_name = NONE);', subname);
+        EXECUTE format('DROP SUBSCRIPTION %I;', subname);
     END LOOP;
 END;
 ${outer_tag}$;

compute_tools/src/sql/finalize_drop_subscriptions.sql

@@ -3,19 +3,19 @@ BEGIN
     IF NOT EXISTS(
         SELECT 1
         FROM pg_catalog.pg_tables
-        WHERE tablename OPERATOR(pg_catalog.=) 'drop_subscriptions_done'::pg_catalog.name
-        AND schemaname OPERATOR(pg_catalog.=) 'neon'::pg_catalog.name
+        WHERE tablename = 'drop_subscriptions_done'
+        AND schemaname = 'neon'
     )
     THEN
         CREATE TABLE neon.drop_subscriptions_done
-        (id pg_catalog.int4 primary key generated by default as identity, timeline_id pg_catalog.text);
+        (id serial primary key, timeline_id text);
     END IF;
 
     -- preserve the timeline_id of the last drop_subscriptions run
     -- to ensure that the cleanup of a timeline is executed only once.
     -- use upsert to avoid the table bloat in case of cascade branching (branch of a branch)
-    INSERT INTO neon.drop_subscriptions_done VALUES (1, pg_catalog.current_setting('neon.timeline_id'))
+    INSERT INTO neon.drop_subscriptions_done VALUES (1, current_setting('neon.timeline_id'))
     ON CONFLICT (id) DO UPDATE
-    SET timeline_id = pg_catalog.current_setting('neon.timeline_id')::pg_catalog.text;
+    SET timeline_id = current_setting('neon.timeline_id');
 END
 $$

compute_tools/src/sql/pre_drop_role_revoke_privileges.sql

@@ -15,15 +15,15 @@ BEGIN
         WHERE schema_name IN ('public')
     LOOP
         FOR grantor IN EXECUTE
-            pg_catalog.format(
-                'SELECT DISTINCT rtg.grantor FROM information_schema.role_table_grants AS rtg WHERE grantee OPERATOR(pg_catalog.=) %s',
+            format(
+                'SELECT DISTINCT rtg.grantor FROM information_schema.role_table_grants AS rtg WHERE grantee = %s',
                 -- N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`
                 quote_literal({role_name})
             )
         LOOP
-            EXECUTE pg_catalog.format('SET LOCAL ROLE %I', grantor);
+            EXECUTE format('SET LOCAL ROLE %I', grantor);
 
-            revoke_query := pg_catalog.format(
+            revoke_query := format(
                 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM %I GRANTED BY %I',
                 schema,
                 -- N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`

compute_tools/src/sql/set_public_schema_owner.sql

@@ -5,17 +5,17 @@ DO ${outer_tag}$
         IF EXISTS(
             SELECT nspname
             FROM pg_catalog.pg_namespace
-            WHERE nspname OPERATOR(pg_catalog.=) 'public'::pg_catalog.name
+            WHERE nspname = 'public'
         )
         THEN
             SELECT nspowner::regrole::text
             FROM pg_catalog.pg_namespace
-            WHERE nspname OPERATOR(pg_catalog.=) 'public'::pg_catalog.text
+            WHERE nspname = 'public'
             INTO schema_owner;
 
-            IF schema_owner OPERATOR(pg_catalog.=) 'cloud_admin'::pg_catalog.text OR schema_owner OPERATOR(pg_catalog.=) 'zenith_admin'::pg_catalog.text
+            IF schema_owner = 'cloud_admin' OR schema_owner = 'zenith_admin'
             THEN
-                EXECUTE pg_catalog.format('ALTER SCHEMA public OWNER TO %I', {db_owner});
+                EXECUTE format('ALTER SCHEMA public OWNER TO %I', {db_owner});
             END IF;
         END IF;
     END

compute_tools/src/sql/unset_template_for_drop_dbs.sql

@@ -3,10 +3,10 @@ DO ${outer_tag}$
         IF EXISTS(
             SELECT 1
             FROM pg_catalog.pg_database
-            WHERE datname OPERATOR(pg_catalog.=) {datname}::pg_catalog.name
+            WHERE datname = {datname}
         )
         THEN
-            EXECUTE pg_catalog.format('ALTER DATABASE %I is_template false', {datname});
+            EXECUTE format('ALTER DATABASE %I is_template false', {datname});
         END IF;
     END
 ${outer_tag}$;

control_plane/src/endpoint.rs

@@ -79,7 +79,7 @@ use spki::der::Decode;
 use spki::{SubjectPublicKeyInfo, SubjectPublicKeyInfoRef};
 use tracing::debug;
 use utils::id::{NodeId, TenantId, TimelineId};
-use utils::shard::{ShardCount, ShardIndex, ShardNumber};
+use utils::shard::{ShardIndex, ShardNumber};
 
 use pageserver_api::config::DEFAULT_GRPC_LISTEN_PORT as DEFAULT_PAGESERVER_GRPC_PORT;
 use postgres_connection::parse_host_port;
@@ -728,13 +728,14 @@ impl Endpoint {
 
         // For the sake of backwards-compatibility, also fill in 'pageserver_connstring'
         //
-        // XXX: I believe this is not really needed, except to make
-        // test_forward_compatibility happy.
-        //
         // Use a closure so that we can conviniently return None in the middle of the
         // loop.
         let pageserver_connstring: Option<String> = (|| {
-            let num_shards = args.pageserver_conninfo.shard_count.count();
+            let num_shards = if args.pageserver_conninfo.shard_count.is_unsharded() {
+                1
+            } else {
+                args.pageserver_conninfo.shard_count.0
+            };
             let mut connstrings = Vec::new();
             for shard_no in 0..num_shards {
                 let shard_index = ShardIndex {
@@ -826,7 +827,6 @@ impl Endpoint {
                 autoprewarm: args.autoprewarm,
                 offload_lfc_interval_seconds: args.offload_lfc_interval_seconds,
                 suspend_timeout_seconds: -1, // Only used in neon_local.
-                databricks_settings: None,
             };
 
             // this strange code is needed to support respec() in tests
@@ -971,9 +971,7 @@ impl Endpoint {
                         | ComputeStatus::Configuration
                         | ComputeStatus::TerminationPendingFast
                         | ComputeStatus::TerminationPendingImmediate
-                        | ComputeStatus::Terminated
-                        | ComputeStatus::RefreshConfigurationPending
-                        | ComputeStatus::RefreshConfiguration => {
+                        | ComputeStatus::Terminated => {
                             bail!("unexpected compute status: {:?}", state.status)
                         }
                     }
@@ -996,27 +994,6 @@ impl Endpoint {
         Ok(())
     }
 
-    // Update the pageservers in the spec file of the endpoint. This is useful to test the spec refresh scenario.
-    pub async fn update_pageservers_in_config(
-        &self,
-        pageserver_conninfo: &PageserverConnectionInfo,
-    ) -> Result<()> {
-        let config_path = self.endpoint_path().join("config.json");
-        let mut config: ComputeConfig = {
-            let file = std::fs::File::open(&config_path)?;
-            serde_json::from_reader(file)?
-        };
-
-        let mut spec = config.spec.unwrap();
-        spec.pageserver_connection_info = Some(pageserver_conninfo.clone());
-        config.spec = Some(spec);
-
-        let file = std::fs::File::create(&config_path)?;
-        serde_json::to_writer_pretty(file, &config)?;
-
-        Ok(())
-    }
-
     // Call the /status HTTP API
     pub async fn get_status(&self) -> Result<ComputeStatusResponse> {
         let client = reqwest::Client::new();
@@ -1179,33 +1156,6 @@ impl Endpoint {
         Ok(response)
     }
 
-    pub async fn refresh_configuration(&self) -> Result<()> {
-        let client = reqwest::Client::builder()
-            .timeout(Duration::from_secs(30))
-            .build()
-            .unwrap();
-        let response = client
-            .post(format!(
-                "http://{}:{}/refresh_configuration",
-                self.internal_http_address.ip(),
-                self.internal_http_address.port()
-            ))
-            .send()
-            .await?;
-
-        let status = response.status();
-        if !(status.is_client_error() || status.is_server_error()) {
-            Ok(())
-        } else {
-            let url = response.url().to_owned();
-            let msg = match response.text().await {
-                Ok(err_body) => format!("Error: {err_body}"),
-                Err(_) => format!("Http error ({}) at {}.", status.as_u16(), url),
-            };
-            Err(anyhow::anyhow!(msg))
-        }
-    }
-
     pub fn connstr(&self, user: &str, db_name: &str) -> String {
         format!(
             "postgresql://{}@{}:{}/{}",
@@ -1217,11 +1167,9 @@ impl Endpoint {
     }
 }
 
-/// If caller is telling us what pageserver to use, this is not a tenant which is
-/// fully managed by storage controller, therefore not sharded.
-pub fn local_pageserver_conf_to_conn_info(
+pub fn pageserver_conf_to_shard_conn_info(
     conf: &crate::local_env::PageServerConf,
-) -> Result<PageserverConnectionInfo> {
+) -> Result<PageserverShardConnectionInfo> {
     let libpq_url = {
         let (host, port) = parse_host_port(&conf.listen_pg_addr)?;
         let port = port.unwrap_or(5432);
@@ -1234,24 +1182,10 @@ pub fn local_pageserver_conf_to_conn_info(
     } else {
         None
     };
-    let ps_conninfo = PageserverShardConnectionInfo {
-        id: Some(conf.id),
+    Ok(PageserverShardConnectionInfo {
+        id: Some(conf.id.to_string()),
         libpq_url,
         grpc_url,
-    };
-
-    let shard_info = PageserverShardInfo {
-        pageservers: vec![ps_conninfo],
-    };
-
-    let shards: HashMap<_, _> = vec![(ShardIndex::unsharded(), shard_info)]
-        .into_iter()
-        .collect();
-    Ok(PageserverConnectionInfo {
-        shard_count: ShardCount::unsharded(),
-        stripe_size: None,
-        shards,
-        prefer_protocol: PageserverProtocol::default(),
     })
 }
 
@@ -1276,7 +1210,7 @@ pub fn tenant_locate_response_to_conn_info(
 
         let shard_info = PageserverShardInfo {
             pageservers: vec![PageserverShardConnectionInfo {
-                id: Some(shard.node_id),
+                id: Some(shard.node_id.to_string()),
                 libpq_url,
                 grpc_url,
             }],
@@ -1288,7 +1222,7 @@ pub fn tenant_locate_response_to_conn_info(
     let stripe_size = if response.shard_params.count.is_unsharded() {
         None
     } else {
-        Some(response.shard_params.stripe_size)
+        Some(response.shard_params.stripe_size.0)
     };
     Ok(PageserverConnectionInfo {
         shard_count: response.shard_params.count,

control_plane/src/local_env.rs

@@ -1074,7 +1074,7 @@ fn generate_auth_keys(private_key_path: &Path, public_key_path: &Path) -> anyhow
 }
 
 fn generate_ssl_ca_cert(cert_path: &Path, key_path: &Path) -> anyhow::Result<()> {
-    // openssl req -x509 -newkey ed25519 -nodes -subj "/CN=Neon Local CA" -days 36500 \
+    // openssl req -x509 -newkey rsa:2048 -nodes -subj "/CN=Neon Local CA" -days 36500 \
     // -out rootCA.crt -keyout rootCA.key
     let keygen_output = Command::new("openssl")
         .args([
@@ -1104,7 +1104,7 @@ fn generate_ssl_cert(
     let mut csr_path = cert_path.to_path_buf();
     csr_path.set_extension(".csr");
 
-    // openssl req -new -nodes -newkey ed25519 -keyout server.key -out server.csr \
+    // openssl req -new -nodes -newkey rsa:2048 -keyout server.key -out server.csr \
     // -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
     let keygen_output = Command::new("openssl")
         .args(["req", "-new", "-nodes"])