pageserver: trace slow requests when compact_l0

Signed-off-by: Alex Chi Z <chi@neon.tech>
Co-authored-by: Christian Schwarz <christian@neon.tech>

.github/actionlint.yml

@@ -4,7 +4,6 @@ self-hosted-runner:
     - large
     - large-arm64
     - small
-    - small-metal
     - small-arm64
     - us-east-2
 config-variables:

.github/actions/neon-project-create/action.yml

@@ -17,31 +17,6 @@ inputs:
   compute_units:
     description: '[Min, Max] compute units'
     default: '[1, 1]'
-  # settings below only needed if you want the project to be sharded from the beginning
-  shard_split_project:
-    description: 'by default new projects are not shard-split, specify true to shard-split'
-    required: false
-    default: 'false'
-  admin_api_key:
-    description: 'Admin API Key needed for shard-splitting. Must be specified if shard_split_project is true'
-    required: false
-  shard_count:
-    description: 'Number of shards to split the project into, only applies if shard_split_project is true'
-    required: false
-    default: '8'
-  stripe_size:
-    description: 'Stripe size, optional, in 8kiB pages.  e.g. set 2048 for 16MB stripes. Default is 128 MiB, only applies if shard_split_project is true'
-    required: false
-    default: '32768'
-  psql_path:
-    description: 'Path to psql binary - it is caller responsibility to provision the psql binary'
-    required: false
-    default: '/tmp/neon/pg_install/v16/bin/psql'
-  libpq_lib_path:
-    description: 'Path to directory containing libpq library - it is caller responsibility to provision the libpq library'
-    required: false
-    default: '/tmp/neon/pg_install/v16/lib'
-  
 
 outputs:
   dsn:
@@ -88,23 +63,6 @@ runs:
         echo "project_id=${project_id}" >> $GITHUB_OUTPUT
 
         echo "Project ${project_id} has been created"
-
-        if [ "${SHARD_SPLIT_PROJECT}" = "true" ]; then
-          # determine tenant ID
-          TENANT_ID=`${PSQL} ${dsn} -t -A -c "SHOW neon.tenant_id"`
-          
-          echo "Splitting project ${project_id} with tenant_id ${TENANT_ID} into $((SHARD_COUNT)) shards with stripe size $((STRIPE_SIZE))"
-
-          echo "Sending PUT request to https://${API_HOST}/regions/${REGION_ID}/api/v1/admin/storage/proxy/control/v1/tenant/${TENANT_ID}/shard_split"
-          echo "with body {\"new_shard_count\": $((SHARD_COUNT)), \"new_stripe_size\": $((STRIPE_SIZE))}"
-          
-          # we need an ADMIN API KEY to invoke storage controller API for shard splitting (bash -u above checks that the variable is set)
-          curl -X PUT \
-            "https://${API_HOST}/regions/${REGION_ID}/api/v1/admin/storage/proxy/control/v1/tenant/${TENANT_ID}/shard_split" \
-            -H "Accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer ${ADMIN_API_KEY}" \
-            -d "{\"new_shard_count\": $SHARD_COUNT, \"new_stripe_size\": $STRIPE_SIZE}"
-        fi
-
       env:
         API_HOST: ${{ inputs.api_host }}
         API_KEY: ${{ inputs.api_key }}
@@ -112,9 +70,3 @@ runs:
         POSTGRES_VERSION: ${{ inputs.postgres_version }}
         MIN_CU: ${{ fromJSON(inputs.compute_units)[0] }}
         MAX_CU: ${{ fromJSON(inputs.compute_units)[1] }}
-        SHARD_SPLIT_PROJECT: ${{ inputs.shard_split_project }}
-        ADMIN_API_KEY: ${{ inputs.admin_api_key }}
-        SHARD_COUNT: ${{ inputs.shard_count }}
-        STRIPE_SIZE: ${{ inputs.stripe_size }}
-        PSQL: ${{ inputs.psql_path }}
-        LD_LIBRARY_PATH: ${{ inputs.libpq_lib_path }}

.github/workflows/_benchmarking_preparation.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ aws-rds-postgres, aws-aurora-serverless-v2-postgres, neon, neon_pg17 ]
+        platform: [ aws-rds-postgres, aws-aurora-serverless-v2-postgres, neon ]
         database: [ clickbench, tpch, userexample ]
 
     env:
@@ -41,9 +41,6 @@ jobs:
           neon)
             CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR }}
             ;;
-          neon_pg17)
-            CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR_PG17 }}
-            ;;
           aws-rds-postgres)
             CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CONNSTR }}
             ;;

.github/workflows/_build-and-test-locally.yml

@@ -158,6 +158,8 @@ jobs:
 
       - name: Run cargo build
         run: |
+          PQ_LIB_DIR=$(pwd)/pg_install/v16/lib
+          export PQ_LIB_DIR
           ${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests
 
       # Do install *before* running rust tests because they might recompile the
@@ -215,6 +217,8 @@ jobs:
         env:
           NEXTEST_RETRIES: 3
         run: |
+          PQ_LIB_DIR=$(pwd)/pg_install/v16/lib
+          export PQ_LIB_DIR
           LD_LIBRARY_PATH=$(pwd)/pg_install/v17/lib
           export LD_LIBRARY_PATH
 

.github/workflows/benchmarking.yml

@@ -63,15 +63,11 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - PG_VERSION: 16
+          - DEFAULT_PG_VERSION: 16
             PLATFORM: "neon-staging"
             region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
             RUNNER: [ self-hosted, us-east-2, x64 ]
-          - PG_VERSION: 17
-            PLATFORM: "neon-staging"
-            region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
-            RUNNER: [ self-hosted, us-east-2, x64 ]
-          - PG_VERSION: 16
+          - DEFAULT_PG_VERSION: 16
             PLATFORM: "azure-staging"
             region_id: 'azure-eastus2'
             RUNNER: [ self-hosted, eastus2, x64 ]
@@ -79,7 +75,7 @@ jobs:
       TEST_PG_BENCH_DURATIONS_MATRIX: "300"
       TEST_PG_BENCH_SCALES_MATRIX: "10,100"
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      PG_VERSION: ${{ matrix.PG_VERSION }}
+      DEFAULT_PG_VERSION: ${{ matrix.DEFAULT_PG_VERSION }}
       TEST_OUTPUT: /tmp/test_output
       BUILD_TYPE: remote
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
@@ -116,7 +112,7 @@ jobs:
       uses: ./.github/actions/neon-project-create
       with:
         region_id: ${{ matrix.region_id }}
-        postgres_version: ${{ env.PG_VERSION }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}
         api_key: ${{ secrets.NEON_STAGING_API_KEY }}
 
     - name: Run benchmark
@@ -126,7 +122,7 @@ jobs:
         test_selection: performance
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
         # Set --sparse-ordering option of pytest-order plugin
         # to ensure tests are running in order of appears in the file.
@@ -317,11 +313,7 @@ jobs:
                       { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"',   "image": "neondatabase/build-tools:pinned-bookworm" },
                       { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-new",      "db_size": "10gb","runner": '"$runner_azure"',   "image": "neondatabase/build-tools:pinned-bookworm" },
                       { "pg_version": 16, "region_id": "azure-eastus2",          "platform": "neonvm-azure-captest-new",      "db_size": "50gb","runner": '"$runner_azure"',   "image": "neondatabase/build-tools:pinned-bookworm" },
-                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier",       "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" }, 
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]
+                      { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]
         }'
 
         if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
@@ -337,15 +329,12 @@ jobs:
         matrix='{
           "platform": [
             "neonvm-captest-reuse"
-          ],
-          "pg_version" : [
-            16,17
-          ],
+          ]
         }'
 
         if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
-          matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 16, "platform": "rds-postgres" },
-                                                     { "pg_version": 16, "platform": "rds-aurora"   }]')
+          matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres" },
+                                                     { "platform": "rds-aurora"   }]')
         fi
 
         echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
@@ -357,14 +346,14 @@ jobs:
           "platform": [
             "neonvm-captest-reuse"
           ],
-          "pg_version" : [
-            16,17
+          "scale": [
+            "10"
           ]
         }'
 
         if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
-          matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 16, "platform": "rds-postgres" },
-                                                     { "pg_version": 16, "platform": "rds-aurora"   }]')
+          matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres", "scale": "10" },
+                                                     { "platform": "rds-aurora",   "scale": "10" }]')
         fi
 
         echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
@@ -389,7 +378,7 @@ jobs:
       TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
       TEST_PG_BENCH_SCALES_MATRIX: ${{ matrix.db_size }}
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      PG_VERSION: ${{ matrix.pg_version }}
+      DEFAULT_PG_VERSION: ${{ matrix.pg_version }}
       TEST_OUTPUT: /tmp/test_output
       BUILD_TYPE: remote
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
@@ -427,7 +416,7 @@ jobs:
       uses: ./.github/actions/neon-project-create
       with:
         region_id: ${{ matrix.region_id }}
-        postgres_version: ${{ env.PG_VERSION }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}
         api_key: ${{ secrets.NEON_STAGING_API_KEY }}
         compute_units: ${{ (contains(matrix.platform, 'captest-freetier') && '[0.25, 0.25]') || '[1, 1]' }}
 
@@ -470,7 +459,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_perf_many_relations
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
@@ -486,7 +475,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
@@ -501,7 +490,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
@@ -516,7 +505,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
@@ -560,19 +549,14 @@ jobs:
         include:
           - PLATFORM: "neonvm-captest-pgvector"
             RUNNER: [ self-hosted, us-east-2, x64 ]
-            postgres_version: 16
-          - PLATFORM: "neonvm-captest-pgvector-pg17"
-            RUNNER: [ self-hosted, us-east-2, x64 ]
-            postgres_version: 17
           - PLATFORM: "azure-captest-pgvector"
             RUNNER: [ self-hosted, eastus2, x64 ]
-            postgres_version: 16
 
     env:
       TEST_PG_BENCH_DURATIONS_MATRIX: "15m"
       TEST_PG_BENCH_SCALES_MATRIX: "1"
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      PG_VERSION: ${{ matrix.postgres_version }}
+      DEFAULT_PG_VERSION: 16
       TEST_OUTPUT: /tmp/test_output
       BUILD_TYPE: remote
 
@@ -606,13 +590,9 @@ jobs:
         dpkg -x postgresql-client-16_16.6-1.pgdg120+1_${arch}.deb pg
 
         mkdir -p /tmp/neon/pg_install/v16/bin
-        mkdir -p /tmp/neon/pg_install/v17/bin
         ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench
         ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/psql    /tmp/neon/pg_install/v16/bin/psql
         ln -s /home/nonroot/pg/usr/lib/$(uname -m)-linux-gnu     /tmp/neon/pg_install/v16/lib
-        ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/pgbench /tmp/neon/pg_install/v17/bin/pgbench
-        ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/psql    /tmp/neon/pg_install/v17/bin/psql
-        ln -s /home/nonroot/pg/usr/lib/$(uname -m)-linux-gnu     /tmp/neon/pg_install/v17/lib
 
         LD_LIBRARY_PATH="/home/nonroot/pg/usr/lib/$(uname -m)-linux-gnu:${LD_LIBRARY_PATH:-}"
         export LD_LIBRARY_PATH
@@ -628,9 +608,6 @@ jobs:
           neonvm-captest-pgvector)
             CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }}
             ;;
-          neonvm-captest-pgvector-pg17)
-            CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_PG17 }}
-            ;;
           azure-captest-pgvector)
             CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_AZURE }}
             ;;
@@ -657,7 +634,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_pgvector_indexing
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
@@ -672,7 +649,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
@@ -719,7 +696,7 @@ jobs:
 
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      PG_VERSION: ${{ matrix.pg_version }}
+      DEFAULT_PG_VERSION: 16
       TEST_OUTPUT: /tmp/test_output
       TEST_OLAP_COLLECT_EXPLAIN: ${{ github.event.inputs.collect_olap_explain }}
       TEST_OLAP_COLLECT_PG_STAT_STATEMENTS: ${{ github.event.inputs.collect_pg_stat_statements }}
@@ -762,18 +739,7 @@ jobs:
       run: |
         case "${PLATFORM}" in
           neonvm-captest-reuse)
-            case "${PG_VERSION}" in
-              16)
-                CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_10M_CONNSTR_V16 }}
-                ;;
-              17)
-                CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR_PG17 }}
-                ;;
-              *)
-                echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
-                exit 1
-                ;;
-            esac
+            CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_10M_CONNSTR }}
             ;;
           rds-aurora)
             CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CLICKBENCH_10M_CONNSTR }}
@@ -797,7 +763,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 43200 -k test_clickbench
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
@@ -846,11 +812,12 @@ jobs:
 
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      PG_VERSION: ${{ matrix.pg_version }}
+      DEFAULT_PG_VERSION: 16
       TEST_OUTPUT: /tmp/test_output
       BUILD_TYPE: remote
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
       PLATFORM: ${{ matrix.platform }}
+      TEST_OLAP_SCALE: ${{ matrix.scale }}
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
@@ -882,31 +849,21 @@ jobs:
       run: |
         case "${PLATFORM}" in
           neonvm-captest-reuse)
-            case "${PG_VERSION}" in
-              16)
-                CONNSTR_SECRET_NAME="BENCHMARK_CAPTEST_TPCH_S10_CONNSTR"
-                ;;
-              17)
-                CONNSTR_SECRET_NAME="BENCHMARK_CAPTEST_CONNSTR_PG17"
-                ;;
-              *)
-                echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
-                exit 1
-                ;;
-            esac
+            ENV_PLATFORM=CAPTEST_TPCH
             ;;
           rds-aurora)
-            CONNSTR_SECRET_NAME="BENCHMARK_RDS_AURORA_TPCH_S10_CONNSTR"
+            ENV_PLATFORM=RDS_AURORA_TPCH
             ;;
           rds-postgres)
-            CONNSTR_SECRET_NAME="BENCHMARK_RDS_POSTGRES_TPCH_S10_CONNSTR"
+            ENV_PLATFORM=RDS_POSTGRES_TPCH
             ;;
           *)
             echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neonvm-captest-reuse', 'rds-aurora', or 'rds-postgres'"
             exit 1
             ;;
         esac
-        
+
+        CONNSTR_SECRET_NAME="BENCHMARK_${ENV_PLATFORM}_S${TEST_OLAP_SCALE}_CONNSTR"
         echo "CONNSTR_SECRET_NAME=${CONNSTR_SECRET_NAME}" >> $GITHUB_ENV
 
     - name: Set up Connection String
@@ -924,13 +881,13 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_tpch
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
-        TEST_OLAP_SCALE: 10
+        TEST_OLAP_SCALE: ${{ matrix.scale }}
 
     - name: Create Allure report
       id: create-allure-report
@@ -965,7 +922,7 @@ jobs:
 
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      PG_VERSION: ${{ matrix.pg_version }}
+      DEFAULT_PG_VERSION: 16
       TEST_OUTPUT: /tmp/test_output
       BUILD_TYPE: remote
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
@@ -1002,18 +959,7 @@ jobs:
       run: |
         case "${PLATFORM}" in
           neonvm-captest-reuse)
-            case "${PG_VERSION}" in
-              16)
-                CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_CAPTEST_CONNSTR }}
-                ;;
-              17)
-                CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR_PG17 }}
-                ;;
-              *)
-                echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
-                exit 1
-                ;;
-            esac
+            CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_CAPTEST_CONNSTR }}
             ;;
           rds-aurora)
             CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_AURORA_CONNSTR }}
@@ -1037,7 +983,7 @@ jobs:
         run_in_parallel: false
         save_perf_report: ${{ env.SAVE_PERF_REPORT }}
         extra_params: -m remote_cluster --timeout 21600 -k test_user_examples
-        pg_version: ${{ env.PG_VERSION }}
+        pg_version: ${{ env.DEFAULT_PG_VERSION }}
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"

.github/workflows/build-macos.yml

@@ -235,7 +235,7 @@ jobs:
           echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
 
       - name: Run cargo build (only for v17)
-        run: cargo build --all --release -j$(sysctl -n hw.ncpu)
+        run: PQ_LIB_DIR=$(pwd)/pg_install/v17/lib cargo build --all --release -j$(sysctl -n hw.ncpu)
 
       - name: Check that no warnings are produced (only for v17)
         run: ./run_clippy.sh

.github/workflows/build_and_test.yml

@@ -242,7 +242,7 @@ jobs:
       statuses: write
       contents: write
       pull-requests: write
-    runs-on: [ self-hosted, small-metal ]
+    runs-on: [ self-hosted, small ]
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
@@ -786,17 +786,6 @@ jobs:
           username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
-      - name: Get the last compute release tag
-        id: get-last-compute-release-tag
-        env:
-          GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
-        run: |
-          tag=$(gh api -q '[.[].tag_name | select(startswith("release-compute"))][0]'\
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            "/repos/${{ github.repository }}/releases")
-          echo tag=${tag} >> ${GITHUB_OUTPUT}
-
       # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
       # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
       # Regular pageserver version string looks like
@@ -828,20 +817,6 @@ jobs:
           TEST_VERSION_ONLY: ${{ matrix.pg_version }}
         run: ./docker-compose/docker_compose_test.sh
 
-      - name: Print logs and clean up docker-compose test
-        if: always()
-        run: |
-          docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml logs || true
-          docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml down
-
-      - name: Test extension upgrade
-        timeout-minutes: 20
-        if: ${{ needs.tag.outputs.build-tag == github.run_id }}
-        env:
-          NEWTAG: ${{ needs.tag.outputs.build-tag }}
-          OLDTAG: ${{ steps.get-last-compute-release-tag.outputs.tag }}
-        run: ./docker-compose/test_extensions_upgrade.sh
-
       - name: Print logs and clean up
         if: always()
         run: |
@@ -1075,7 +1050,6 @@ jobs:
           retries: 5
           script: |
             const tag = "${{ needs.tag.outputs.build-tag }}";
-            const branch = "${{ github.ref_name }}";
 
             try {
               const existingRef = await github.rest.git.getRef({
@@ -1118,48 +1092,12 @@ jobs:
               }
 
               console.log(`Release for tag ${tag} does not exist. Creating it...`);
-
-              // Find the PR number using the commit SHA
-              const pullRequests = await github.rest.pulls.list({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                state: 'closed',
-                base: branch,
-              });
-
-              const pr = pullRequests.data.find(pr => pr.merge_commit_sha === context.sha);
-              const prNumber = pr ? pr.number : null;
-
-              // Find the previous release on the branch
-              const releases = await github.rest.repos.listReleases({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                per_page: 100,
-              });
-
-              const branchReleases = releases.data
-                .filter((release) => {
-                  const regex = new RegExp(`^${branch}-\\d+$`);
-                  return regex.test(release.tag_name) && !release.draft && !release.prerelease;
-                })
-                .sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
-
-              const previousTag = branchReleases.length > 0 ? branchReleases[0].tag_name : null;
-
-              const releaseNotes = [
-                prNumber
-                  ? `Release PR https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}.`
-                  : 'Release PR not found.',
-                previousTag
-                  ? `Diff with the previous release https://github.com/${context.repo.owner}/${context.repo.repo}/compare/${previousTag}...${tag}.`
-                  : `No previous release found on branch ${branch}.`,
-              ].join('\n\n');
-
               await github.rest.repos.createRelease({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 tag_name: tag,
-                body: releaseNotes,
+                // TODO: Automate release notes properly
+                generate_release_notes: false,
               });
               console.log(`Release for tag ${tag} created successfully.`);
             }

.github/workflows/ingest_benchmark.yml

@@ -28,24 +28,7 @@ jobs:
     strategy:
       fail-fast: false # allow other variants to continue even if one fails
       matrix:
-        include:
-          - target_project: new_empty_project_stripe_size_2048 
-            stripe_size: 2048 # 16 MiB
-            postgres_version: 16
-          - target_project: new_empty_project_stripe_size_32768
-            stripe_size: 32768 # 256 MiB # note that this is different from null because using null will shard_split the project only if it reaches the threshold
-                               # while here it is sharded from the beginning with a shard size of 256 MiB
-            postgres_version: 16
-          - target_project: new_empty_project
-            stripe_size: null # run with neon defaults which will shard split only when reaching the threshold
-            postgres_version: 16
-          - target_project: new_empty_project
-            stripe_size: null # run with neon defaults which will shard split only when reaching the threshold
-            postgres_version: 17
-          - target_project: large_existing_project
-            stripe_size: null # cannot re-shared or choose different stripe size for existing, already sharded project
-            postgres_version: 16
-      max-parallel: 1 # we want to run each stripe size sequentially to be able to compare the results
+        target_project: [new_empty_project, large_existing_project]
     permissions:
       contents: write
       statuses: write
@@ -84,21 +67,17 @@ jobs:
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
 
     - name: Create Neon Project
-      if: ${{ startsWith(matrix.target_project, 'new_empty_project') }}
+      if: ${{ matrix.target_project == 'new_empty_project' }}
       id: create-neon-project-ingest-target
       uses: ./.github/actions/neon-project-create
       with:
         region_id: aws-us-east-2
-        postgres_version: ${{ matrix.postgres_version }}
+        postgres_version: 16
         compute_units: '[7, 7]' # we want to test large compute here to avoid compute-side bottleneck
         api_key: ${{ secrets.NEON_STAGING_API_KEY }}
-        shard_split_project: ${{ matrix.stripe_size != null && 'true' || 'false' }}
-        admin_api_key: ${{ secrets.NEON_STAGING_ADMIN_API_KEY }} 
-        shard_count: 8
-        stripe_size: ${{ matrix.stripe_size }}
 
     - name: Initialize Neon project
-      if: ${{ startsWith(matrix.target_project, 'new_empty_project') }}
+      if: ${{ matrix.target_project == 'new_empty_project' }}
       env:
           BENCHMARK_INGEST_TARGET_CONNSTR: ${{ steps.create-neon-project-ingest-target.outputs.dsn }}
           NEW_PROJECT_ID: ${{ steps.create-neon-project-ingest-target.outputs.project_id }}
@@ -151,7 +130,7 @@ jobs:
         test_selection: performance/test_perf_ingest_using_pgcopydb.py
         run_in_parallel: false
         extra_params: -s -m remote_cluster --timeout 86400 -k test_ingest_performance_using_pgcopydb
-        pg_version: v${{ matrix.postgres_version }}
+        pg_version: v16
         save_perf_report: true
         aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
       env:
@@ -167,7 +146,7 @@ jobs:
         ${PSQL} "${BENCHMARK_INGEST_TARGET_CONNSTR}" -c "\dt+"
 
     - name: Delete Neon Project
-      if: ${{ always() && startsWith(matrix.target_project, 'new_empty_project') }}
+      if: ${{ always() && matrix.target_project == 'new_empty_project' }}
       uses: ./.github/actions/neon-project-delete
       with:
         project_id: ${{ steps.create-neon-project-ingest-target.outputs.project_id }}

.github/workflows/neon_extra_builds.yml

@@ -114,7 +114,7 @@ jobs:
         run: make walproposer-lib -j$(nproc)
 
       - name: Produce the build stats
-        run: cargo build --all --release --timings -j$(nproc)
+        run: PQ_LIB_DIR=$(pwd)/pg_install/v17/lib cargo build --all --release --timings -j$(nproc)
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

Cargo.lock

@@ -179,7 +179,7 @@ dependencies = [
  "nom",
  "num-traits",
  "rusticata-macros",
- "thiserror 1.0.69",
+ "thiserror",
  "time",
 ]
 
@@ -718,14 +718,14 @@ dependencies = [
 
 [[package]]
 name = "axum"
-version = "0.8.1"
+version = "0.7.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8"
+checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
 dependencies = [
+ "async-trait",
  "axum-core",
  "base64 0.22.1",
  "bytes",
- "form_urlencoded",
  "futures-util",
  "http 1.1.0",
  "http-body 1.0.0",
@@ -733,7 +733,7 @@ dependencies = [
  "hyper 1.4.1",
  "hyper-util",
  "itoa",
- "matchit",
+ "matchit 0.7.0",
  "memchr",
  "mime",
  "percent-encoding",
@@ -746,7 +746,7 @@ dependencies = [
  "sha1",
  "sync_wrapper 1.0.1",
  "tokio",
- "tokio-tungstenite 0.26.1",
+ "tokio-tungstenite 0.24.0",
  "tower 0.5.2",
  "tower-layer",
  "tower-service",
@@ -755,10 +755,11 @@ dependencies = [
 
 [[package]]
 name = "axum-core"
-version = "0.5.0"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733"
+checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
 dependencies = [
+ "async-trait",
  "bytes",
  "futures-util",
  "http 1.1.0",
@@ -941,18 +942,6 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
-[[package]]
-name = "bb8"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8"
-dependencies = [
- "async-trait",
- "futures-util",
- "parking_lot 0.12.1",
- "tokio",
-]
-
 [[package]]
 name = "bcder"
 version = "0.7.4"
@@ -1129,7 +1118,7 @@ dependencies = [
  "log",
  "nix 0.25.1",
  "regex",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -1310,9 +1299,9 @@ dependencies = [
  "serde_with",
  "signal-hook",
  "tar",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-stream",
  "tokio-util",
  "tower 0.5.2",
@@ -1419,9 +1408,9 @@ dependencies = [
  "serde",
  "serde_json",
  "storage_broker",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-util",
  "toml",
  "toml_edit",
@@ -1797,24 +1786,11 @@ dependencies = [
  "chrono",
  "diesel_derives",
  "itoa",
+ "pq-sys",
+ "r2d2",
  "serde_json",
 ]
 
-[[package]]
-name = "diesel-async"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51a307ac00f7c23f526a04a77761a0519b9f0eb2838ebf5b905a58580095bdcb"
-dependencies = [
- "async-trait",
- "bb8",
- "diesel",
- "futures-util",
- "scoped-futures",
- "tokio",
- "tokio-postgres 0.7.12",
-]
-
 [[package]]
 name = "diesel_derives"
 version = "2.2.1"
@@ -2263,7 +2239,7 @@ dependencies = [
  "pin-project",
  "rand 0.8.5",
  "sha1",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "tokio-util",
 ]
@@ -3389,6 +3365,12 @@ dependencies = [
  "regex-automata 0.1.10",
 ]
 
+[[package]]
+name = "matchit"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
+
 [[package]]
 name = "matchit"
 version = "0.8.4"
@@ -3779,7 +3761,7 @@ dependencies = [
  "serde_json",
  "serde_path_to_error",
  "sha2",
- "thiserror 1.0.69",
+ "thiserror",
  "url",
 ]
 
@@ -3829,7 +3811,7 @@ dependencies = [
  "futures-sink",
  "js-sys",
  "pin-project-lite",
- "thiserror 1.0.69",
+ "thiserror",
  "tracing",
 ]
 
@@ -3861,7 +3843,7 @@ dependencies = [
  "opentelemetry_sdk",
  "prost",
  "reqwest",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -3897,7 +3879,7 @@ dependencies = [
  "percent-encoding",
  "rand 0.8.5",
  "serde_json",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "tokio-stream",
  "tracing",
@@ -4011,7 +3993,7 @@ dependencies = [
  "remote_storage",
  "serde_json",
  "svg_fmt",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "tokio-util",
  "utils",
@@ -4060,8 +4042,8 @@ dependencies = [
  "pageserver_compaction",
  "pin-project-lite",
  "postgres",
- "postgres-protocol 0.6.6",
- "postgres-types 0.2.6",
+ "postgres-protocol",
+ "postgres-types",
  "postgres_backend",
  "postgres_connection",
  "postgres_ffi",
@@ -4087,12 +4069,12 @@ dependencies = [
  "strum_macros",
  "sysinfo",
  "tenant_size_model",
- "thiserror 1.0.69",
+ "thiserror",
  "tikv-jemallocator",
  "tokio",
  "tokio-epoll-uring",
  "tokio-io-timeout",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-stream",
  "tokio-tar",
  "tokio-util",
@@ -4133,7 +4115,7 @@ dependencies = [
  "storage_broker",
  "strum",
  "strum_macros",
- "thiserror 1.0.69",
+ "thiserror",
  "utils",
 ]
 
@@ -4148,9 +4130,9 @@ dependencies = [
  "postgres",
  "reqwest",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-stream",
  "tokio-util",
  "utils",
@@ -4448,23 +4430,23 @@ dependencies = [
 
 [[package]]
 name = "postgres"
-version = "0.19.6"
-source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#8b44892f7851e705810b2cb54504325699966070"
+version = "0.19.4"
+source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#511f998c00148ab7c847bd7e6cfd3a906d0e7473"
 dependencies = [
  "bytes",
  "fallible-iterator",
  "futures-util",
  "log",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
 ]
 
 [[package]]
 name = "postgres-protocol"
-version = "0.6.6"
-source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#8b44892f7851e705810b2cb54504325699966070"
+version = "0.6.4"
+source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#511f998c00148ab7c847bd7e6cfd3a906d0e7473"
 dependencies = [
- "base64 0.21.1",
+ "base64 0.20.0",
  "byteorder",
  "bytes",
  "fallible-iterator",
@@ -4477,24 +4459,6 @@ dependencies = [
  "stringprep",
 ]
 
-[[package]]
-name = "postgres-protocol"
-version = "0.6.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23"
-dependencies = [
- "base64 0.22.1",
- "byteorder",
- "bytes",
- "fallible-iterator",
- "hmac",
- "md-5",
- "memchr",
- "rand 0.8.5",
- "sha2",
- "stringprep",
-]
-
 [[package]]
 name = "postgres-protocol2"
 version = "0.1.0"
@@ -4513,24 +4477,12 @@ dependencies = [
 
 [[package]]
 name = "postgres-types"
-version = "0.2.6"
-source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#8b44892f7851e705810b2cb54504325699966070"
-dependencies = [
- "bytes",
- "chrono",
- "fallible-iterator",
- "postgres-protocol 0.6.6",
-]
-
-[[package]]
-name = "postgres-types"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f"
+version = "0.2.4"
+source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#511f998c00148ab7c847bd7e6cfd3a906d0e7473"
 dependencies = [
  "bytes",
  "fallible-iterator",
- "postgres-protocol 0.6.7",
+ "postgres-protocol",
 ]
 
 [[package]]
@@ -4553,9 +4505,9 @@ dependencies = [
  "rustls 0.23.18",
  "rustls-pemfile 2.1.1",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-postgres-rustls",
  "tokio-rustls 0.26.0",
  "tokio-util",
@@ -4570,7 +4522,7 @@ dependencies = [
  "itertools 0.10.5",
  "once_cell",
  "postgres",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "url",
 ]
 
@@ -4591,7 +4543,7 @@ dependencies = [
  "pprof",
  "regex",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
  "tracing",
  "utils",
 ]
@@ -4602,7 +4554,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "camino",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "workspace_hack",
 ]
@@ -4635,7 +4587,7 @@ dependencies = [
  "smallvec",
  "symbolic-demangle",
  "tempfile",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -4657,6 +4609,15 @@ version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
+[[package]]
+name = "pq-sys"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6cc05d7ea95200187117196eee9edd0644424911821aeb28a18ce60ea0b8793"
+dependencies = [
+ "vcpkg",
+]
+
 [[package]]
 name = "pq_proto"
 version = "0.1.0"
@@ -4664,10 +4625,10 @@ dependencies = [
  "byteorder",
  "bytes",
  "itertools 0.10.5",
- "postgres-protocol 0.6.6",
+ "postgres-protocol",
  "rand 0.8.5",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
 ]
 
@@ -4738,7 +4699,7 @@ dependencies = [
  "memchr",
  "parking_lot 0.12.1",
  "procfs",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -4908,11 +4869,11 @@ dependencies = [
  "strum",
  "strum_macros",
  "subtle",
- "thiserror 1.0.69",
+ "thiserror",
  "tikv-jemalloc-ctl",
  "tikv-jemallocator",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-postgres2",
  "tokio-rustls 0.26.0",
  "tokio-tungstenite 0.21.0",
@@ -4969,6 +4930,17 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "r2d2"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
+dependencies = [
+ "log",
+ "parking_lot 0.12.1",
+ "scheduled-thread-pool",
+]
+
 [[package]]
 name = "rand"
 version = "0.7.3"
@@ -5305,7 +5277,7 @@ dependencies = [
  "http 1.1.0",
  "reqwest",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
  "tower-service",
 ]
 
@@ -5325,7 +5297,7 @@ dependencies = [
  "reqwest",
  "reqwest-middleware",
  "retry-policies",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "tracing",
  "wasm-timer",
@@ -5341,7 +5313,7 @@ dependencies = [
  "async-trait",
  "getrandom 0.2.11",
  "http 1.1.0",
- "matchit",
+ "matchit 0.8.4",
  "opentelemetry",
  "reqwest",
  "reqwest-middleware",
@@ -5700,7 +5672,7 @@ dependencies = [
  "pageserver_api",
  "parking_lot 0.12.1",
  "postgres",
- "postgres-protocol 0.6.6",
+ "postgres-protocol",
  "postgres_backend",
  "postgres_ffi",
  "pprof",
@@ -5720,11 +5692,11 @@ dependencies = [
  "storage_broker",
  "strum",
  "strum_macros",
- "thiserror 1.0.69",
+ "thiserror",
  "tikv-jemallocator",
  "tokio",
  "tokio-io-timeout",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-stream",
  "tokio-tar",
  "tokio-util",
@@ -5759,7 +5731,7 @@ dependencies = [
  "reqwest",
  "safekeeper_api",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
  "utils",
  "workspace_hack",
 ]
@@ -5783,12 +5755,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "scoped-futures"
-version = "0.1.4"
+name = "scheduled-thread-pool"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b24aae2d0636530f359e9d5ef0c04669d11c5e756699b27a6a6d845d8329091"
+checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
 dependencies = [
- "pin-project-lite",
+ "parking_lot 0.12.1",
 ]
 
 [[package]]
@@ -5968,7 +5940,7 @@ dependencies = [
  "rand 0.8.5",
  "serde",
  "serde_json",
- "thiserror 1.0.69",
+ "thiserror",
  "time",
  "url",
  "uuid",
@@ -6040,7 +6012,7 @@ checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6"
 dependencies = [
  "percent-encoding",
  "serde",
- "thiserror 1.0.69",
+ "thiserror",
 ]
 
 [[package]]
@@ -6202,7 +6174,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085"
 dependencies = [
  "num-bigint",
  "num-traits",
- "thiserror 1.0.69",
+ "thiserror",
  "time",
 ]
 
@@ -6323,7 +6295,6 @@ dependencies = [
  "clap",
  "control_plane",
  "diesel",
- "diesel-async",
  "diesel_migrations",
  "fail",
  "futures",
@@ -6338,16 +6309,16 @@ dependencies = [
  "pageserver_api",
  "pageserver_client",
  "postgres_connection",
+ "r2d2",
  "rand 0.8.5",
  "reqwest",
  "routerify",
- "scoped-futures",
  "scopeguard",
  "serde",
  "serde_json",
  "strum",
  "strum_macros",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "tokio-util",
  "tracing",
@@ -6394,7 +6365,7 @@ dependencies = [
  "serde_json",
  "storage_controller_client",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-postgres-rustls",
  "tokio-stream",
  "tokio-util",
@@ -6639,16 +6610,7 @@ version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
- "thiserror-impl 1.0.69",
-]
-
-[[package]]
-name = "thiserror"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
-dependencies = [
- "thiserror-impl 2.0.11",
+ "thiserror-impl",
 ]
 
 [[package]]
@@ -6662,17 +6624,6 @@ dependencies = [
  "syn 2.0.90",
 ]
 
-[[package]]
-name = "thiserror-impl"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.90",
-]
-
 [[package]]
 name = "thread_local"
 version = "1.1.7"
@@ -6829,7 +6780,7 @@ dependencies = [
  "nix 0.26.4",
  "once_cell",
  "scopeguard",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "tokio-util",
  "tracing",
@@ -6859,8 +6810,8 @@ dependencies = [
 
 [[package]]
 name = "tokio-postgres"
-version = "0.7.9"
-source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#8b44892f7851e705810b2cb54504325699966070"
+version = "0.7.7"
+source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#511f998c00148ab7c847bd7e6cfd3a906d0e7473"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -6873,39 +6824,11 @@ dependencies = [
  "percent-encoding",
  "phf",
  "pin-project-lite",
- "postgres-protocol 0.6.6",
- "postgres-types 0.2.6",
- "rand 0.8.5",
+ "postgres-protocol",
+ "postgres-types",
  "socket2",
  "tokio",
  "tokio-util",
- "whoami",
-]
-
-[[package]]
-name = "tokio-postgres"
-version = "0.7.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb"
-dependencies = [
- "async-trait",
- "byteorder",
- "bytes",
- "fallible-iterator",
- "futures-channel",
- "futures-util",
- "log",
- "parking_lot 0.12.1",
- "percent-encoding",
- "phf",
- "pin-project-lite",
- "postgres-protocol 0.6.7",
- "postgres-types 0.2.8",
- "rand 0.8.5",
- "socket2",
- "tokio",
- "tokio-util",
- "whoami",
 ]
 
 [[package]]
@@ -6917,7 +6840,7 @@ dependencies = [
  "ring",
  "rustls 0.23.18",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-rustls 0.26.0",
  "x509-certificate",
 ]
@@ -6938,7 +6861,6 @@ dependencies = [
  "pin-project-lite",
  "postgres-protocol2",
  "postgres-types2",
- "serde",
  "tokio",
  "tokio-util",
 ]
@@ -7015,14 +6937,14 @@ dependencies = [
 
 [[package]]
 name = "tokio-tungstenite"
-version = "0.26.1"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4bf6fecd69fcdede0ec680aaf474cdab988f9de6bc73d3758f0160e3b7025a"
+checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
 dependencies = [
  "futures-util",
  "log",
  "tokio",
- "tungstenite 0.26.1",
+ "tungstenite 0.24.0",
 ]
 
 [[package]]
@@ -7332,16 +7254,16 @@ dependencies = [
  "log",
  "rand 0.8.5",
  "sha1",
- "thiserror 1.0.69",
+ "thiserror",
  "url",
  "utf-8",
 ]
 
 [[package]]
 name = "tungstenite"
-version = "0.26.1"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413083a99c579593656008130e29255e54dcaae495be556cc26888f211648c24"
+checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
 dependencies = [
  "byteorder",
  "bytes",
@@ -7351,7 +7273,7 @@ dependencies = [
  "log",
  "rand 0.8.5",
  "sha1",
- "thiserror 2.0.11",
+ "thiserror",
  "utf-8",
 ]
 
@@ -7546,7 +7468,7 @@ dependencies = [
  "signal-hook",
  "strum",
  "strum_macros",
- "thiserror 1.0.69",
+ "thiserror",
  "tokio",
  "tokio-stream",
  "tokio-tar",
@@ -7576,6 +7498,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -7595,7 +7523,7 @@ dependencies = [
  "serde_json",
  "sysinfo",
  "tokio",
- "tokio-postgres 0.7.9",
+ "tokio-postgres",
  "tokio-util",
  "tracing",
  "tracing-subscriber",
@@ -7646,7 +7574,7 @@ dependencies = [
  "remote_storage",
  "serde",
  "serde_json",
- "thiserror 1.0.69",
+ "thiserror",
  "tikv-jemallocator",
  "tokio",
  "tokio-util",
@@ -8175,7 +8103,7 @@ dependencies = [
  "ring",
  "signature 2.2.0",
  "spki 0.7.3",
- "thiserror 1.0.69",
+ "thiserror",
  "zeroize",
 ]
 
@@ -8192,7 +8120,7 @@ dependencies = [
  "nom",
  "oid-registry",
  "rusticata-macros",
- "thiserror 1.0.69",
+ "thiserror",
  "time",
 ]
 

Cargo.toml

@@ -65,7 +65,7 @@ aws-smithy-types = "1.2"
 aws-credential-types = "1.2.0"
 aws-sigv4 = { version = "1.2", features = ["sign-http"] }
 aws-types = "1.3"
-axum = { version = "0.8.1", features = ["ws"] }
+axum = { version = "0.7.9", features = ["ws"] }
 base64 = "0.13.0"
 bincode = "1.3"
 bindgen = "0.70"

Dockerfile

@@ -45,7 +45,7 @@ COPY --chown=nonroot . .
 
 ARG ADDITIONAL_RUSTFLAGS
 RUN set -e \
-    && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo build \
+    && PQ_LIB_DIR=$(pwd)/pg_install/v${STABLE_PG_VERSION}/lib RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo build \
       --bin pg_sni_router  \
       --bin pageserver  \
       --bin pagectl  \
@@ -64,7 +64,6 @@ ARG DEFAULT_PG_VERSION
 WORKDIR /data
 
 RUN set -e \
-    && echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries \
     && apt update \
     && apt install -y \
         libreadline-dev \
@@ -73,7 +72,6 @@ RUN set -e \
 	# System postgres for use with client libraries (e.g. in storage controller)
         postgresql-15 \
         openssl \
-    && rm -f /etc/apt/apt.conf.d/80-retries \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
     && useradd -d /data neon \
     && chown -R neon:neon /data

Makefile

@@ -64,6 +64,8 @@ CARGO_BUILD_FLAGS += $(filter -j1,$(MAKEFLAGS))
 CARGO_CMD_PREFIX += $(if $(filter n,$(MAKEFLAGS)),,+)
 # Force cargo not to print progress bar
 CARGO_CMD_PREFIX += CARGO_TERM_PROGRESS_WHEN=never CI=1
+# Set PQ_LIB_DIR to make sure `storage_controller` get linked with bundled libpq (through diesel)
+CARGO_CMD_PREFIX += PQ_LIB_DIR=$(POSTGRES_INSTALL_DIR)/v16/lib
 
 CACHEDIR_TAG_CONTENTS := "Signature: 8a477f597d28d172789f06886806bc55"
 

README.md

@@ -21,10 +21,8 @@ The Neon storage engine consists of two major components:
 
 See developer documentation in [SUMMARY.md](/docs/SUMMARY.md) for more information.
 
-## Running a local development environment
+## Running local installation
 
-Neon can be run on a workstation for small experiments and to test code changes, by
-following these instructions.
 
 #### Installing dependencies on Linux
 1. Install build dependencies and other applicable packages
@@ -240,7 +238,7 @@ postgres=# select * from t;
 > cargo neon stop
 ```
 
-More advanced usages can be found at [Local Development Control Plane (`neon_local`))](./control_plane/README.md).
+More advanced usages can be found at [Control Plane and Neon Local](./control_plane/README.md).
 
 #### Handling build failures
 

build-tools.Dockerfile

@@ -3,10 +3,6 @@ ARG DEBIAN_VERSION=bookworm
 FROM debian:bookworm-slim AS pgcopydb_builder
 ARG DEBIAN_VERSION
 
-RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc \
-    echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc
-
 RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
         set -e && \
         apt update && \
@@ -65,10 +61,6 @@ RUN mkdir -p /pgcopydb/bin && \
 COPY --from=pgcopydb_builder /usr/lib/postgresql/16/bin/pgcopydb /pgcopydb/bin/pgcopydb
 COPY --from=pgcopydb_builder /pgcopydb/lib/libpq.so.5 /pgcopydb/lib/libpq.so.5
 
-RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc \
-    echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc
-
 # System deps
 #
 # 'gdb' is included so that we get backtraces of core dumps produced in
@@ -226,8 +218,6 @@ RUN wget -O /tmp/libicu-${ICU_VERSION}.tgz https://github.com/unicode-org/icu/re
 USER nonroot:nonroot
 WORKDIR /home/nonroot
 
-RUN echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /home/nonroot/.curlrc
-
 # Python
 ENV PYTHON_VERSION=3.11.10 \
     PYENV_ROOT=/home/nonroot/.pyenv \

compute/compute-node.Dockerfile

@@ -5,7 +5,6 @@ ARG TAG=pinned
 ARG BUILD_TAG
 ARG DEBIAN_VERSION=bookworm
 ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim
-ARG ALPINE_CURL_VERSION=8.11.1
 
 #########################################################################################
 #
@@ -18,10 +17,6 @@ ARG DEBIAN_VERSION
 # Use strict mode for bash to catch errors early
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
-RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc \
-    echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc
-
 RUN case $DEBIAN_VERSION in \
       # Version-specific installs for Bullseye (PG14-PG16):
       # The h3_pg extension needs a cmake 3.20+, but Debian bullseye has 3.18.
@@ -842,8 +837,6 @@ ENV PATH="/home/nonroot/.cargo/bin:$PATH"
 USER nonroot
 WORKDIR /home/nonroot
 
-RUN echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /home/nonroot/.curlrc
-
 RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux-gnu/rustup-init && \
     chmod +x rustup-init && \
     ./rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \
@@ -880,8 +873,6 @@ ENV PATH="/home/nonroot/.cargo/bin:$PATH"
 USER nonroot
 WORKDIR /home/nonroot
 
-RUN echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /home/nonroot/.curlrc
-
 RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux-gnu/rustup-init && \
     chmod +x rustup-init && \
     ./rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \
@@ -1251,7 +1242,6 @@ RUN mold -run cargo build --locked --profile release-line-debug-size-lto --bin c
 
 FROM debian:$DEBIAN_FLAVOR AS pgbouncer
 RUN set -e \
-    && echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries \
     && apt update \
     && apt install --no-install-suggests --no-install-recommends -y \
         build-essential \
@@ -1276,31 +1266,15 @@ RUN set -e \
 
 #########################################################################################
 #
-# Layer "exporters"
+# Layers "postgres-exporter" and "sql-exporter"
 #
 #########################################################################################
-FROM alpine/curl:${ALPINE_CURL_VERSION} AS exporters
-ARG TARGETARCH
-# Keep sql_exporter version same as in build-tools.Dockerfile and
-# test_runner/regress/test_compute_metrics.py
-RUN if [ "$TARGETARCH" = "amd64" ]; then\
-        postgres_exporter_sha256='027e75dda7af621237ff8f5ac66b78a40b0093595f06768612b92b1374bd3105';\
-        pgbouncer_exporter_sha256='c9f7cf8dcff44f0472057e9bf52613d93f3ffbc381ad7547a959daa63c5e84ac';\
-        sql_exporter_sha256='38e439732bbf6e28ca4a94d7bc3686d3fa1abdb0050773d5617a9efdb9e64d08';\
-    else\
-        postgres_exporter_sha256='131a376d25778ff9701a4c81f703f179e0b58db5c2c496e66fa43f8179484786';\
-        pgbouncer_exporter_sha256='217c4afd7e6492ae904055bc14fe603552cf9bac458c063407e991d68c519da3';\
-        sql_exporter_sha256='11918b00be6e2c3a67564adfdb2414fdcbb15a5db76ea17d1d1a944237a893c6';\
-    fi\
-    && curl -sL https://github.com/prometheus-community/postgres_exporter/releases/download/v0.16.0/postgres_exporter-0.16.0.linux-${TARGETARCH}.tar.gz\
-     | tar xzf - --strip-components=1 -C.\
-    && curl -sL https://github.com/prometheus-community/pgbouncer_exporter/releases/download/v0.10.2/pgbouncer_exporter-0.10.2.linux-${TARGETARCH}.tar.gz\
-     | tar xzf - --strip-components=1 -C.\
-    && curl -sL https://github.com/burningalchemist/sql_exporter/releases/download/0.17.0/sql_exporter-0.17.0.linux-${TARGETARCH}.tar.gz\
-     | tar xzf - --strip-components=1 -C.\
-    && echo "${postgres_exporter_sha256} postgres_exporter" | sha256sum -c -\
-    && echo "${pgbouncer_exporter_sha256} pgbouncer_exporter" | sha256sum -c -\
-    && echo "${sql_exporter_sha256} sql_exporter" | sha256sum -c -
+
+FROM quay.io/prometheuscommunity/postgres-exporter:v0.16.0 AS postgres-exporter
+
+# Keep the version the same as in build-tools.Dockerfile and
+# test_runner/regress/test_compute_metrics.py.
+FROM burningalchemist/sql_exporter:0.17.0 AS sql-exporter
 
 #########################################################################################
 #
@@ -1355,8 +1329,7 @@ COPY --from=vector-pg-build /pgvector.patch /ext-src/
 COPY --from=pgjwt-pg-build /pgjwt.tar.gz /ext-src
 #COPY --from=pgrag-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 #COPY --from=pg-jsonschema-pg-build /home/nonroot/pg_jsonschema.tar.gz /ext-src
-COPY --from=pg-graphql-pg-build /home/nonroot/pg_graphql.tar.gz /ext-src
-COPY compute/patches/pg_graphql.patch /ext-src
+#COPY --from=pg-graphql-pg-build /home/nonroot/pg_graphql.tar.gz /ext-src
 #COPY --from=pg-tiktoken-pg-build /home/nonroot/pg_tiktoken.tar.gz /ext-src
 COPY --from=hypopg-pg-build /hypopg.tar.gz /ext-src
 COPY --from=pg-hashids-pg-build /pg_hashids.tar.gz /ext-src
@@ -1390,7 +1363,6 @@ RUN cd /ext-src/pgvector-src && patch -p1 <../pgvector.patch
 RUN cd /ext-src/pg_hint_plan-src && patch -p1 < /ext-src/pg_hint_plan_${PG_VERSION}.patch
 COPY --chmod=755 docker-compose/run-tests.sh /run-tests.sh
 RUN patch -p1 </ext-src/pg_cron.patch
-RUN cd /ext-src/pg_graphql-src && patch -p1 </ext-src/pg_graphql.patch
 ENV PATH=/usr/local/pgsql/bin:$PATH
 ENV PGHOST=compute
 ENV PGPORT=55433
@@ -1428,10 +1400,9 @@ COPY --chmod=0666 --chown=postgres compute/etc/pgbouncer.ini /etc/pgbouncer.ini
 COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/local_proxy /usr/local/bin/local_proxy
 RUN mkdir -p /etc/local_proxy && chown postgres:postgres /etc/local_proxy
 
-# Metrics exporter binaries and configuration files
-COPY --from=exporters ./postgres_exporter /bin/postgres_exporter
-COPY --from=exporters ./pgbouncer_exporter /bin/pgbouncer_exporter
-COPY --from=exporters ./sql_exporter /bin/sql_exporter
+# Metrics exporter binaries and  configuration files
+COPY --from=postgres-exporter /bin/postgres_exporter /bin/postgres_exporter
+COPY --from=sql-exporter      /bin/sql_exporter      /bin/sql_exporter
 
 COPY --chown=postgres compute/etc/postgres_exporter.yml /etc/postgres_exporter.yml
 
@@ -1453,8 +1424,6 @@ RUN mkdir /usr/local/download_extensions && chown -R postgres:postgres /usr/loca
 # libboost* for rdkit
 # ca-certificates for communicating with s3 by compute_ctl
 
-RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc
 
 RUN apt update && \
     case $DEBIAN_VERSION in \
@@ -1511,7 +1480,7 @@ RUN set -ex; \
     else \
         echo "Unsupported architecture: ${TARGETARCH}"; exit 1; \
     fi; \
-    curl --retry 5 -L "https://awscli.amazonaws.com/awscli-exe-linux-${TARGETARCH_ALT}-2.17.5.zip" -o /tmp/awscliv2.zip; \
+    curl -L "https://awscli.amazonaws.com/awscli-exe-linux-${TARGETARCH_ALT}-2.17.5.zip" -o /tmp/awscliv2.zip; \
     echo "${CHECKSUM}  /tmp/awscliv2.zip" | sha256sum -c -; \
     unzip /tmp/awscliv2.zip -d /tmp/awscliv2; \
     /tmp/awscliv2/aws/install; \

compute/etc/pgbouncer.ini

@@ -19,8 +19,6 @@ max_prepared_statements=0
 admin_users=postgres
 unix_socket_dir=/tmp/
 unix_socket_mode=0777
-; required for pgbouncer_exporter
-ignore_startup_parameters=extra_float_digits
 
 ;; Disable connection logging. It produces a lot of logs that no one looks at,
 ;; and we can get similar log entries from the proxy too. We had incidents in

compute/patches/pg_graphql.patch

@@ -1,19 +0,0 @@
-commit ec6a491d126882966a696f9ad5d3698935361d55
-Author: Alexey Masterov <alexeymasterov@neon.tech>
-Date:   Tue Dec 17 10:25:00 2024 +0100
-
-    Changes required to run tests on Neon
-
-diff --git a/test/expected/permissions_functions.out b/test/expected/permissions_functions.out
-index 1e9fbc2..94cbe25 100644
---- a/test/expected/permissions_functions.out
-+++ b/test/expected/permissions_functions.out
-@@ -64,7 +64,7 @@ begin;
-     select current_user;
-  current_user 
- --------------
-- postgres
-+ cloud_admin
- (1 row)
- 
-     -- revoke default access from the public role for new functions

compute/vm-image-spec-bookworm.yaml

@@ -27,10 +27,6 @@ commands:
     user: nobody
     sysvInitAction: respawn
     shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter" /bin/postgres_exporter --config.file=/etc/postgres_exporter.yml'
-  - name: pgbouncer-exporter
-    user: postgres
-    sysvInitAction: respawn
-    shell: '/bin/pgbouncer_exporter --pgBouncer.connectionString="postgres:///pgbouncer?host=/tmp&port=6432&dbname=pgbouncer&user=pgbouncer"'
   - name: sql-exporter
     user: nobody
     sysvInitAction: respawn

compute/vm-image-spec-bullseye.yaml

@@ -27,10 +27,6 @@ commands:
     user: nobody
     sysvInitAction: respawn
     shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres application_name=postgres-exporter" /bin/postgres_exporter --config.file=/etc/postgres_exporter.yml'
-  - name: pgbouncer-exporter
-    user: postgres
-    sysvInitAction: respawn
-    shell: '/bin/pgbouncer_exporter --pgBouncer.connectionString="postgres:///pgbouncer?host=/tmp&port=6432&dbname=pgbouncer&user=pgbouncer"'
   - name: sql-exporter
     user: nobody
     sysvInitAction: respawn

compute_tools/src/extension_server.rs

@@ -85,8 +85,6 @@ use tracing::info;
 use tracing::log::warn;
 use zstd::stream::read::Decoder;
 
-use crate::metrics::{REMOTE_EXT_REQUESTS_TOTAL, UNKNOWN_HTTP_STATUS};
-
 fn get_pg_config(argument: &str, pgbin: &str) -> String {
     // gives the result of `pg_config [argument]`
     // where argument is a flag like `--version` or `--sharedir`
@@ -260,58 +258,21 @@ async fn download_extension_tar(ext_remote_storage: &str, ext_path: &str) -> Res
 
     info!("Download extension {:?} from uri {:?}", ext_path, uri);
 
-    match do_extension_server_request(&uri).await {
-        Ok(resp) => {
-            info!(
-                "Successfully downloaded remote extension data {:?}",
-                ext_path
-            );
-            REMOTE_EXT_REQUESTS_TOTAL
-                .with_label_values(&[&StatusCode::OK.to_string()])
-                .inc();
-            Ok(resp)
-        }
-        Err((msg, status)) => {
-            REMOTE_EXT_REQUESTS_TOTAL
-                .with_label_values(&[&status])
-                .inc();
-            bail!(msg);
-        }
-    }
-}
+    let resp = reqwest::get(uri).await?;
 
-// Do a single remote extensions server request.
-// Return result or (error message + stringified status code) in case of any failures.
-async fn do_extension_server_request(uri: &str) -> Result<Bytes, (String, String)> {
-    let resp = reqwest::get(uri).await.map_err(|e| {
-        (
-            format!("could not perform remote extensions server request: {}", e),
-            UNKNOWN_HTTP_STATUS.to_string(),
-        )
-    })?;
-    let status = resp.status();
-
-    match status {
+    match resp.status() {
         StatusCode::OK => match resp.bytes().await {
-            Ok(resp) => Ok(resp),
-            Err(e) => Err((
-                format!("could not read remote extensions server response: {}", e),
-                // It's fine to return and report error with status as 200 OK,
-                // because we still failed to read the response.
-                status.to_string(),
-            )),
+            Ok(resp) => {
+                info!("Download extension {:?} completed successfully", ext_path);
+                Ok(resp)
+            }
+            Err(e) => bail!("could not deserialize remote extension response: {}", e),
         },
-        StatusCode::SERVICE_UNAVAILABLE => Err((
-            "remote extensions server is temporarily unavailable".to_string(),
-            status.to_string(),
-        )),
-        _ => Err((
-            format!(
-                "unexpected remote extensions server response status code: {}",
-                status
-            ),
-            status.to_string(),
-        )),
+        StatusCode::SERVICE_UNAVAILABLE => bail!("remote extension is temporarily unavailable"),
+        _ => bail!(
+            "unexpected remote extension response status code: {}",
+            resp.status()
+        ),
     }
 }
 

compute_tools/src/http/extract/json.rs

@@ -1,6 +1,9 @@
 use std::ops::{Deref, DerefMut};
 
-use axum::extract::{rejection::JsonRejection, FromRequest, Request};
+use axum::{
+    async_trait,
+    extract::{rejection::JsonRejection, FromRequest, Request},
+};
 use compute_api::responses::GenericAPIError;
 use http::StatusCode;
 
@@ -9,6 +12,7 @@ use http::StatusCode;
 #[derive(Debug, Clone, Copy, Default)]
 pub(crate) struct Json<T>(pub T);
 
+#[async_trait]
 impl<S, T> FromRequest<S> for Json<T>
 where
     axum::Json<T>: FromRequest<S, Rejection = JsonRejection>,

compute_tools/src/http/extract/path.rs

@@ -1,6 +1,9 @@
 use std::ops::{Deref, DerefMut};
 
-use axum::extract::{rejection::PathRejection, FromRequestParts};
+use axum::{
+    async_trait,
+    extract::{rejection::PathRejection, FromRequestParts},
+};
 use compute_api::responses::GenericAPIError;
 use http::{request::Parts, StatusCode};
 
@@ -9,6 +12,7 @@ use http::{request::Parts, StatusCode};
 #[derive(Debug, Clone, Copy, Default)]
 pub(crate) struct Path<T>(pub T);
 
+#[async_trait]
 impl<S, T> FromRequestParts<S> for Path<T>
 where
     axum::extract::Path<T>: FromRequestParts<S, Rejection = PathRejection>,

compute_tools/src/http/extract/query.rs

@@ -1,6 +1,9 @@
 use std::ops::{Deref, DerefMut};
 
-use axum::extract::{rejection::QueryRejection, FromRequestParts};
+use axum::{
+    async_trait,
+    extract::{rejection::QueryRejection, FromRequestParts},
+};
 use compute_api::responses::GenericAPIError;
 use http::{request::Parts, StatusCode};
 
@@ -9,6 +12,7 @@ use http::{request::Parts, StatusCode};
 #[derive(Debug, Clone, Copy, Default)]
 pub(crate) struct Query<T>(pub T);
 
+#[async_trait]
 impl<S, T> FromRequestParts<S> for Query<T>
 where
     axum::extract::Query<T>: FromRequestParts<S, Rejection = QueryRejection>,

compute_tools/src/http/openapi_spec.yaml

@@ -68,6 +68,35 @@ paths:
               schema:
                 $ref: "#/components/schemas/ComputeInsights"
 
+  /installed_extensions:
+    get:
+      tags:
+      - Info
+      summary: Get installed extensions.
+      description: ""
+      operationId: getInstalledExtensions
+      responses:
+        200:
+          description: List of installed extensions
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/InstalledExtensions"
+  /info:
+    get:
+      tags:
+      - Info
+      summary: Get info about the compute pod / VM.
+      description: ""
+      operationId: getInfo
+      responses:
+        200:
+          description: Info
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Info"
+
   /dbs_and_roles:
     get:
       tags:

compute_tools/src/http/routes/info.rs

@@ -0,0 +1,11 @@
+use axum::response::Response;
+use compute_api::responses::InfoResponse;
+use http::StatusCode;
+
+use crate::http::JsonResponse;
+
+/// Get information about the physical characteristics about the compute.
+pub(in crate::http) async fn get_info() -> Response {
+    let num_cpus = num_cpus::get_physical();
+    JsonResponse::success(StatusCode::OK, &InfoResponse { num_cpus })
+}

compute_tools/src/http/routes/installed_extensions.rs

@@ -0,0 +1,33 @@
+use std::sync::Arc;
+
+use axum::{extract::State, response::Response};
+use compute_api::responses::ComputeStatus;
+use http::StatusCode;
+use tokio::task;
+
+use crate::{compute::ComputeNode, http::JsonResponse, installed_extensions};
+
+/// Get a list of installed extensions.
+pub(in crate::http) async fn get_installed_extensions(
+    State(compute): State<Arc<ComputeNode>>,
+) -> Response {
+    let status = compute.get_status();
+    if status != ComputeStatus::Running {
+        return JsonResponse::invalid_status(status);
+    }
+
+    let conf = compute.get_conn_conf(None);
+    let res = task::spawn_blocking(move || installed_extensions::get_installed_extensions(conf))
+        .await
+        .unwrap();
+
+    match res {
+        Ok(installed_extensions) => {
+            JsonResponse::success(StatusCode::OK, Some(installed_extensions))
+        }
+        Err(e) => JsonResponse::error(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            format!("failed to get list of installed extensions: {e}"),
+        ),
+    }
+}

compute_tools/src/http/routes/metrics.rs

@@ -2,16 +2,17 @@ use axum::{body::Body, response::Response};
 use http::header::CONTENT_TYPE;
 use http::StatusCode;
 use metrics::proto::MetricFamily;
-use metrics::{Encoder, TextEncoder};
+use metrics::Encoder;
+use metrics::TextEncoder;
 
-use crate::{http::JsonResponse, metrics::collect};
+use crate::{http::JsonResponse, installed_extensions};
 
 /// Expose Prometheus metrics.
 pub(in crate::http) async fn get_metrics() -> Response {
     // When we call TextEncoder::encode() below, it will immediately return an
     // error if a metric family has no metrics, so we need to preemptively
     // filter out metric families with no metrics.
-    let metrics = collect()
+    let metrics = installed_extensions::collect()
         .into_iter()
         .filter(|m| !m.get_metric().is_empty())
         .collect::<Vec<MetricFamily>>();

compute_tools/src/http/routes/mod.rs

@@ -10,7 +10,9 @@ pub(in crate::http) mod extension_server;
 pub(in crate::http) mod extensions;
 pub(in crate::http) mod failpoints;
 pub(in crate::http) mod grants;
+pub(in crate::http) mod info;
 pub(in crate::http) mod insights;
+pub(in crate::http) mod installed_extensions;
 pub(in crate::http) mod metrics;
 pub(in crate::http) mod metrics_json;
 pub(in crate::http) mod status;

compute_tools/src/http/server.rs

@@ -22,7 +22,8 @@ use uuid::Uuid;
 
 use super::routes::{
     check_writability, configure, database_schema, dbs_and_roles, extension_server, extensions,
-    grants, insights, metrics, metrics_json, status, terminate,
+    grants, info as info_route, insights, installed_extensions, metrics, metrics_json, status,
+    terminate,
 };
 use crate::compute::ComputeNode;
 
@@ -54,12 +55,17 @@ async fn serve(port: u16, compute: Arc<ComputeNode>) {
         .route("/database_schema", get(database_schema::get_schema_dump))
         .route("/dbs_and_roles", get(dbs_and_roles::get_catalog_objects))
         .route(
-            "/extension_server/{*filename}",
+            "/extension_server/*filename",
             post(extension_server::download_extension),
         )
         .route("/extensions", post(extensions::install_extension))
         .route("/grants", post(grants::add_grant))
+        .route("/info", get(info_route::get_info))
         .route("/insights", get(insights::get_insights))
+        .route(
+            "/installed_extensions",
+            get(installed_extensions::get_installed_extensions),
+        )
         .route("/metrics", get(metrics::get_metrics))
         .route("/metrics.json", get(metrics_json::get_metrics))
         .route("/status", get(status::get_status))

compute_tools/src/installed_extensions.rs

@@ -1,10 +1,13 @@
 use compute_api::responses::{InstalledExtension, InstalledExtensions};
+use metrics::proto::MetricFamily;
 use std::collections::HashMap;
 
 use anyhow::Result;
 use postgres::{Client, NoTls};
 
-use crate::metrics::INSTALLED_EXTENSIONS;
+use metrics::core::Collector;
+use metrics::{register_uint_gauge_vec, UIntGaugeVec};
+use once_cell::sync::Lazy;
 
 /// We don't reuse get_existing_dbs() just for code clarity
 /// and to make database listing query here more explicit.
@@ -99,3 +102,16 @@ pub fn get_installed_extensions(mut conf: postgres::config::Config) -> Result<In
         extensions: extensions_map.into_values().collect(),
     })
 }
+
+static INSTALLED_EXTENSIONS: Lazy<UIntGaugeVec> = Lazy::new(|| {
+    register_uint_gauge_vec!(
+        "compute_installed_extensions",
+        "Number of databases where the version of extension is installed",
+        &["extension_name", "version", "owned_by_superuser"]
+    )
+    .expect("failed to define a metric")
+});
+
+pub fn collect() -> Vec<MetricFamily> {
+    INSTALLED_EXTENSIONS.collect()
+}

compute_tools/src/lib.rs

@@ -16,7 +16,6 @@ pub mod extension_server;
 pub mod installed_extensions;
 pub mod local_proxy;
 pub mod lsn_lease;
-pub mod metrics;
 mod migration;
 pub mod monitor;
 pub mod params;

compute_tools/src/metrics.rs

@@ -1,70 +0,0 @@
-use metrics::core::Collector;
-use metrics::proto::MetricFamily;
-use metrics::{register_int_counter_vec, register_uint_gauge_vec, IntCounterVec, UIntGaugeVec};
-use once_cell::sync::Lazy;
-
-pub(crate) static INSTALLED_EXTENSIONS: Lazy<UIntGaugeVec> = Lazy::new(|| {
-    register_uint_gauge_vec!(
-        "compute_installed_extensions",
-        "Number of databases where the version of extension is installed",
-        &["extension_name", "version", "owned_by_superuser"]
-    )
-    .expect("failed to define a metric")
-});
-
-// Normally, any HTTP API request is described by METHOD (e.g. GET, POST, etc.) + PATH,
-// but for all our APIs we defined a 'slug'/method/operationId in the OpenAPI spec.
-// And it's fair to call it a 'RPC' (Remote Procedure Call).
-pub enum CPlaneRequestRPC {
-    GetSpec,
-}
-
-impl CPlaneRequestRPC {
-    pub fn as_str(&self) -> &str {
-        match self {
-            CPlaneRequestRPC::GetSpec => "GetSpec",
-        }
-    }
-}
-
-pub const UNKNOWN_HTTP_STATUS: &str = "unknown";
-
-pub(crate) static CPLANE_REQUESTS_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "compute_ctl_cplane_requests_total",
-        "Total number of control plane requests made by compute_ctl by status",
-        &["rpc", "http_status"]
-    )
-    .expect("failed to define a metric")
-});
-
-/// Total number of failed database migrations. Per-compute, this is actually a boolean metric,
-/// either empty or with a single value (1, migration_id) because we stop at the first failure.
-/// Yet, the sum over the fleet will provide the total number of failures.
-pub(crate) static DB_MIGRATION_FAILED: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "compute_ctl_db_migration_failed_total",
-        "Total number of failed database migrations",
-        &["migration_id"]
-    )
-    .expect("failed to define a metric")
-});
-
-pub(crate) static REMOTE_EXT_REQUESTS_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
-    register_int_counter_vec!(
-        "compute_ctl_remote_ext_requests_total",
-        "Total number of requests made by compute_ctl to download extensions from S3 proxy by status",
-        // Do not use any labels like extension name yet.
-        // We can add them later if needed.
-        &["http_status"]
-    )
-    .expect("failed to define a metric")
-});
-
-pub fn collect() -> Vec<MetricFamily> {
-    let mut metrics = INSTALLED_EXTENSIONS.collect();
-    metrics.extend(CPLANE_REQUESTS_TOTAL.collect());
-    metrics.extend(REMOTE_EXT_REQUESTS_TOTAL.collect());
-    metrics.extend(DB_MIGRATION_FAILED.collect());
-    metrics
-}

compute_tools/src/migration.rs

@@ -1,9 +1,7 @@
 use anyhow::{Context, Result};
 use fail::fail_point;
 use postgres::{Client, Transaction};
-use tracing::{error, info};
-
-use crate::metrics::DB_MIGRATION_FAILED;
+use tracing::info;
 
 /// Runs a series of migrations on a target database
 pub(crate) struct MigrationRunner<'m> {
@@ -80,31 +78,24 @@ impl<'m> MigrationRunner<'m> {
         Ok(())
     }
 
-    /// Run an individual migration in a separate transaction block.
-    fn run_migration(client: &mut Client, migration_id: i64, migration: &str) -> Result<()> {
-        let mut txn = client
-            .transaction()
-            .with_context(|| format!("begin transaction for migration {migration_id}"))?;
-
+    /// Run an individual migration
+    fn run_migration(txn: &mut Transaction, migration_id: i64, migration: &str) -> Result<()> {
         if migration.starts_with("-- SKIP") {
             info!("Skipping migration id={}", migration_id);
 
             // Even though we are skipping the migration, updating the
             // migration ID should help keep logic easy to understand when
             // trying to understand the state of a cluster.
-            Self::update_migration_id(&mut txn, migration_id)?;
+            Self::update_migration_id(txn, migration_id)?;
         } else {
             info!("Running migration id={}:\n{}\n", migration_id, migration);
 
             txn.simple_query(migration)
                 .with_context(|| format!("apply migration {migration_id}"))?;
 
-            Self::update_migration_id(&mut txn, migration_id)?;
+            Self::update_migration_id(txn, migration_id)?;
         }
 
-        txn.commit()
-            .with_context(|| format!("commit transaction for migration {migration_id}"))?;
-
         Ok(())
     }
 
@@ -118,20 +109,19 @@ impl<'m> MigrationRunner<'m> {
             // The index lags the migration ID by 1, so the current migration
             // ID is also the next index
             let migration_id = (current_migration + 1) as i64;
-            let migration = self.migrations[current_migration];
 
-            match Self::run_migration(self.client, migration_id, migration) {
-                Ok(_) => {
-                    info!("Finished migration id={}", migration_id);
-                }
-                Err(e) => {
-                    error!("Failed to run migration id={}: {}", migration_id, e);
-                    DB_MIGRATION_FAILED
-                        .with_label_values(&[migration_id.to_string().as_str()])
-                        .inc();
-                    return Err(e);
-                }
-            }
+            let mut txn = self
+                .client
+                .transaction()
+                .with_context(|| format!("begin transaction for migration {migration_id}"))?;
+
+            Self::run_migration(&mut txn, migration_id, self.migrations[current_migration])
+                .with_context(|| format!("running migration {migration_id}"))?;
+
+            txn.commit()
+                .with_context(|| format!("commit transaction for migration {migration_id}"))?;
+
+            info!("Finished migration id={}", migration_id);
 
             current_migration += 1;
         }

compute_tools/src/spec.rs

@@ -6,7 +6,6 @@ use std::path::Path;
 use tracing::{error, info, instrument, warn};
 
 use crate::config;
-use crate::metrics::{CPlaneRequestRPC, CPLANE_REQUESTS_TOTAL, UNKNOWN_HTTP_STATUS};
 use crate::migration::MigrationRunner;
 use crate::params::PG_HBA_ALL_MD5;
 use crate::pg_helpers::*;
@@ -20,7 +19,7 @@ use compute_api::spec::ComputeSpec;
 fn do_control_plane_request(
     uri: &str,
     jwt: &str,
-) -> Result<ControlPlaneSpecResponse, (bool, String, String)> {
+) -> Result<ControlPlaneSpecResponse, (bool, String)> {
     let resp = reqwest::blocking::Client::new()
         .get(uri)
         .header("Authorization", format!("Bearer {}", jwt))
@@ -29,41 +28,34 @@ fn do_control_plane_request(
             (
                 true,
                 format!("could not perform spec request to control plane: {}", e),
-                UNKNOWN_HTTP_STATUS.to_string(),
             )
         })?;
 
-    let status = resp.status();
-    match status {
+    match resp.status() {
         StatusCode::OK => match resp.json::<ControlPlaneSpecResponse>() {
             Ok(spec_resp) => Ok(spec_resp),
             Err(e) => Err((
                 true,
                 format!("could not deserialize control plane response: {}", e),
-                status.to_string(),
             )),
         },
-        StatusCode::SERVICE_UNAVAILABLE => Err((
-            true,
-            "control plane is temporarily unavailable".to_string(),
-            status.to_string(),
-        )),
+        StatusCode::SERVICE_UNAVAILABLE => {
+            Err((true, "control plane is temporarily unavailable".to_string()))
+        }
         StatusCode::BAD_GATEWAY => {
             // We have a problem with intermittent 502 errors now
             // https://github.com/neondatabase/cloud/issues/2353
             // It's fine to retry GET request in this case.
-            Err((
-                true,
-                "control plane request failed with 502".to_string(),
-                status.to_string(),
-            ))
+            Err((true, "control plane request failed with 502".to_string()))
         }
         // Another code, likely 500 or 404, means that compute is unknown to the control plane
         // or some internal failure happened. Doesn't make much sense to retry in this case.
         _ => Err((
             false,
-            format!("unexpected control plane response status code: {}", status),
-            status.to_string(),
+            format!(
+                "unexpected control plane response status code: {}",
+                resp.status()
+            ),
         )),
     }
 }
@@ -91,28 +83,17 @@ pub fn get_spec_from_control_plane(
     // - got spec -> return Ok(Some(spec))
     while attempt < 4 {
         spec = match do_control_plane_request(&cp_uri, &jwt) {
-            Ok(spec_resp) => {
-                CPLANE_REQUESTS_TOTAL
-                    .with_label_values(&[
-                        CPlaneRequestRPC::GetSpec.as_str(),
-                        &StatusCode::OK.to_string(),
-                    ])
-                    .inc();
-                match spec_resp.status {
-                    ControlPlaneComputeStatus::Empty => Ok(None),
-                    ControlPlaneComputeStatus::Attached => {
-                        if let Some(spec) = spec_resp.spec {
-                            Ok(Some(spec))
-                        } else {
-                            bail!("compute is attached, but spec is empty")
-                        }
+            Ok(spec_resp) => match spec_resp.status {
+                ControlPlaneComputeStatus::Empty => Ok(None),
+                ControlPlaneComputeStatus::Attached => {
+                    if let Some(spec) = spec_resp.spec {
+                        Ok(Some(spec))
+                    } else {
+                        bail!("compute is attached, but spec is empty")
                     }
                 }
-            }
-            Err((retry, msg, status)) => {
-                CPLANE_REQUESTS_TOTAL
-                    .with_label_values(&[CPlaneRequestRPC::GetSpec.as_str(), &status])
-                    .inc();
+            },
+            Err((retry, msg)) => {
                 if retry {
                     Err(anyhow!(msg))
                 } else {

control_plane/README.md

@@ -1,10 +1,6 @@
-# Local Development Control Plane (`neon_local`)
+# Control Plane and Neon Local
 
-This crate contains tools to start a Neon development environment locally. This utility can be used with the `cargo neon` command.  This is a convenience to invoke
-the `neon_local` binary.
-
-**Note**: this is a dev/test tool -- a minimal control plane suitable for testing
-code changes locally, but not suitable for running production systems.
+This crate contains tools to start a Neon development environment locally. This utility can be used with the `cargo neon` command.
 
 ## Example: Start with Postgres 16
 

control_plane/src/pageserver.rs

@@ -347,11 +347,6 @@ impl PageServerNode {
                 .map(|x| x.parse::<usize>())
                 .transpose()
                 .context("Failed to parse 'compaction_threshold' as an integer")?,
-            compaction_upper_limit: settings
-                .remove("compaction_upper_limit")
-                .map(|x| x.parse::<usize>())
-                .transpose()
-                .context("Failed to parse 'compaction_upper_limit' as an integer")?,
             compaction_algorithm: settings
                 .remove("compaction_algorithm")
                 .map(serde_json::from_str)
@@ -362,11 +357,6 @@ impl PageServerNode {
                 .map(|x| x.parse::<usize>())
                 .transpose()
                 .context("Failed to parse 'l0_flush_delay_threshold' as an integer")?,
-            l0_flush_wait_upload: settings
-                .remove("l0_flush_wait_upload")
-                .map(|x| x.parse::<bool>())
-                .transpose()
-                .context("Failed to parse 'l0_flush_wait_upload' as a boolean")?,
             l0_flush_stall_threshold: settings
                 .remove("l0_flush_stall_threshold")
                 .map(|x| x.parse::<usize>())

deny.toml

@@ -41,8 +41,8 @@ allow = [
     "MIT",
     "MPL-2.0",
     "OpenSSL",
+    "Unicode-DFS-2016",
     "Unicode-3.0",
-    "Zlib",
 ]
 confidence-threshold = 0.8
 exceptions = [

docker-compose/compute_wrapper/Dockerfile

@@ -7,12 +7,11 @@ FROM $REPOSITORY/${COMPUTE_IMAGE}:$TAG
 ARG COMPUTE_IMAGE
 
 USER root
-RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    apt-get update &&       \
+RUN apt-get update &&       \
     apt-get install -y curl \
                        jq   \
                        netcat-openbsd
 #This is required for the pg_hintplan test
-RUN mkdir -p /ext-src/pg_hint_plan-src && chown postgres /ext-src/pg_hint_plan-src
+RUN mkdir -p /ext-src/pg_hint_plan-src && chown postgres /ext-src/pg_hint_plan-src 
 
-USER postgres
+USER postgres

docker-compose/compute_wrapper/shell/compute.sh

@@ -20,55 +20,30 @@ while ! nc -z pageserver 6400; do
 done
 echo "Page server is ready."
 
-cp ${SPEC_FILE_ORG} ${SPEC_FILE}
+echo "Create a tenant and timeline"
+generate_id tenant_id
+PARAMS=(
+     -X PUT
+     -H "Content-Type: application/json"
+     -d "{\"mode\": \"AttachedSingle\", \"generation\": 1, \"tenant_conf\": {}}"
+     "http://pageserver:9898/v1/tenant/${tenant_id}/location_config"
+)
+result=$(curl "${PARAMS[@]}")
+echo $result | jq .
 
- if [ -n "${TENANT_ID:-}" ] && [ -n "${TIMELINE_ID:-}" ]; then
-   tenant_id=${TENANT_ID}
-   timeline_id=${TIMELINE_ID}
-else
-  echo "Check if a tenant present"
-  PARAMS=(
-       -X GET
-       -H "Content-Type: application/json"
-       "http://pageserver:9898/v1/tenant"
-  )
-  tenant_id=$(curl "${PARAMS[@]}" | jq -r .[0].id)
-  if [ -z "${tenant_id}" ] || [ "${tenant_id}" = null ]; then
-    echo "Create a tenant"
-    generate_id tenant_id
-    PARAMS=(
-         -X PUT
-         -H "Content-Type: application/json"
-         -d "{\"mode\": \"AttachedSingle\", \"generation\": 1, \"tenant_conf\": {}}"
-        "http://pageserver:9898/v1/tenant/${tenant_id}/location_config"
-    )
-    result=$(curl "${PARAMS[@]}")
-    echo $result | jq .
-  fi
-
-  echo "Check if a timeline present"
-  PARAMS=(
-       -X GET
-       -H "Content-Type: application/json"
-       "http://pageserver:9898/v1/tenant/${tenant_id}/timeline"
-  )
-  timeline_id=$(curl "${PARAMS[@]}" | jq -r .[0].timeline_id)
-  if [ -z "${timeline_id}" ] || [ "${timeline_id}" = null ]; then
-    generate_id timeline_id
-    PARAMS=(
-        -sbf
-        -X POST
-        -H "Content-Type: application/json"
-        -d "{\"new_timeline_id\": \"${timeline_id}\", \"pg_version\": ${PG_VERSION}}"
-        "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/"
-    )
-    result=$(curl "${PARAMS[@]}")
-    echo $result | jq .
-  fi
-fi
+generate_id timeline_id
+PARAMS=(
+     -sbf
+     -X POST
+     -H "Content-Type: application/json"
+     -d "{\"new_timeline_id\": \"${timeline_id}\", \"pg_version\": ${PG_VERSION}}"
+     "http://pageserver:9898/v1/tenant/${tenant_id}/timeline/"
+)
+result=$(curl "${PARAMS[@]}")
+echo $result | jq .
 
 echo "Overwrite tenant id and timeline id in spec file"
-sed -i "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE}
+sed "s/TENANT_ID/${tenant_id}/" ${SPEC_FILE_ORG} > ${SPEC_FILE}
 sed -i "s/TIMELINE_ID/${timeline_id}/" ${SPEC_FILE}
 
 cat ${SPEC_FILE}

docker-compose/docker-compose.yml

@@ -149,13 +149,11 @@ services:
       args:
         - REPOSITORY=${REPOSITORY:-neondatabase}
         - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-16}
-        - TAG=${COMPUTE_TAG:-${TAG:-latest}}
+        - TAG=${TAG:-latest}
         - http_proxy=${http_proxy:-}
         - https_proxy=${https_proxy:-}
     environment:
       - PG_VERSION=${PG_VERSION:-16}
-      - TENANT_ID=${TENANT_ID:-}
-      - TIMELINE_ID=${TIMELINE_ID:-}
       #- RUST_BACKTRACE=1
     # Mount the test files directly, for faster editing cycle.
     volumes:

docker-compose/docker_compose_test.sh

@@ -31,7 +31,7 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do
     echo "clean up containers if exists"
     cleanup
     PG_TEST_VERSION=$((pg_version < 16 ? 16 : pg_version))
-    PG_VERSION=$pg_version PG_TEST_VERSION=$PG_TEST_VERSION docker compose --profile test-extensions -f $COMPOSE_FILE up --quiet-pull --build -d
+    PG_VERSION=$pg_version PG_TEST_VERSION=$PG_TEST_VERSION docker compose --profile test-extensions -f $COMPOSE_FILE up --build -d
 
     echo "wait until the compute is ready. timeout after 60s. "
     cnt=0
@@ -51,7 +51,6 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do
     done
 
     if [ $pg_version -ge 16 ]; then
-        docker cp ext-src $TEST_CONTAINER_NAME:/
         # This is required for the pg_hint_plan test, to prevent flaky log message causing the test to fail
         # It cannot be moved to Dockerfile now because the database directory is created after the start of the container
         echo Adding dummy config
@@ -62,7 +61,7 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do
         docker cp $TMPDIR/data $COMPUTE_CONTAINER_NAME:/ext-src/pg_hint_plan-src/
         rm -rf $TMPDIR
         # We are running tests now
-        if ! docker exec -e SKIP=timescaledb-src,rdkit-src,postgis-src,pgx_ulid-src,pgtap-src,pg_tiktoken-src,pg_jsonschema-src,kq_imcx-src,wal2json_2_5-src \
+        if ! docker exec -e SKIP=timescaledb-src,rdkit-src,postgis-src,pgx_ulid-src,pgtap-src,pg_tiktoken-src,pg_jsonschema-src,pg_graphql-src,kq_imcx-src,wal2json_2_5-src \
             $TEST_CONTAINER_NAME /run-tests.sh | tee testout.txt
         then
             FAILED=$(tail -1 testout.txt)

docker-compose/ext-src/hll-src/test-upgrade.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin'    --dbname=contrib_regression add_agg agg_oob auto_sparse card_op cast_shape copy_binary cumulative_add_cardinality_correction cumulative_add_comprehensive_promotion cumulative_add_sparse_edge cumulative_add_sparse_random cumulative_add_sparse_step cumulative_union_comprehensive cumulative_union_explicit_explicit cumulative_union_explicit_promotion cumulative_union_probabilistic_probabilistic cumulative_union_sparse_full_representation cumulative_union_sparse_promotion cumulative_union_sparse_sparse disable_hashagg equal explicit_thresh hash hash_any meta_func murmur_bigint murmur_bytea nosparse notequal scalar_oob storedproc transaction typmod typmod_insert union_op

docker-compose/ext-src/hypopg-src/test-upgrade.patch

@@ -1,27 +0,0 @@
-diff --git a/expected/hypopg.out b/expected/hypopg.out
-index 90121d0..859260b 100644
---- a/expected/hypopg.out
-+++ b/expected/hypopg.out
-@@ -11,7 +11,8 @@ BEGIN
- END;
- $_$
- LANGUAGE plpgsql;
--CREATE EXTENSION hypopg;
-+CREATE EXTENSION IF NOT EXISTS hypopg;
-+NOTICE:  extension "hypopg" already exists, skipping
- CREATE TABLE hypo (id integer, val text, "Id2" bigint);
- INSERT INTO hypo SELECT i, 'line ' || i
- FROM generate_series(1,100000) f(i);
-diff --git a/test/sql/hypopg.sql b/test/sql/hypopg.sql
-index 99722b0..8d6bacb 100644
---- a/test/sql/hypopg.sql
-+++ b/test/sql/hypopg.sql
-@@ -12,7 +12,7 @@ END;
- $_$
- LANGUAGE plpgsql;
-
--CREATE EXTENSION hypopg;
-+CREATE EXTENSION IF NOT EXISTS hypopg;
-
- CREATE TABLE hypo (id integer, val text, "Id2" bigint);
-

docker-compose/ext-src/hypopg-src/test-upgrade.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --inputdir=test --dbname=contrib_regression hypopg hypo_brin hypo_index_part hypo_include hypo_hash hypo_hide_index

docker-compose/ext-src/ip4r-src/test-upgrade.patch

@@ -1,23 +0,0 @@
-diff --git a/expected/ip4r.out b/expected/ip4r.out
-index 7527af3..b38ed29 100644
---- a/expected/ip4r.out
-+++ b/expected/ip4r.out
-@@ -1,6 +1,5 @@
- --
- /*CUT-HERE*/
--CREATE EXTENSION ip4r;
- -- Check whether any of our opclasses fail amvalidate
- DO $d$
-   DECLARE
-diff --git a/sql/ip4r.sql b/sql/ip4r.sql
-index 65c49ec..24ade09 100644
---- a/sql/ip4r.sql
-+++ b/sql/ip4r.sql
-@@ -1,7 +1,6 @@
- --
-
- /*CUT-HERE*/
--CREATE EXTENSION ip4r;
-
- -- Check whether any of our opclasses fail amvalidate
-

docker-compose/ext-src/ip4r-src/test-upgrade.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin'    --dbname=contrib_regression ip4r ip4r-softerr ip4r-v11

docker-compose/ext-src/pg_cron-src/test-upgrade.patch

@@ -1,75 +0,0 @@
-diff --git a/expected/pg_cron-test.out b/expected/pg_cron-test.out
-index d79d542..1663886 100644
---- a/expected/pg_cron-test.out
-+++ b/expected/pg_cron-test.out
-@@ -1,30 +1,3 @@
--CREATE EXTENSION pg_cron VERSION '1.0';
--SELECT extversion FROM pg_extension WHERE extname='pg_cron';
-- extversion 
--------------
-- 1.0
--(1 row)
--
---- Test binary compatibility with v1.4 function signature.
--ALTER EXTENSION pg_cron UPDATE TO '1.4';
--SELECT cron.unschedule(job_name := 'no_such_job');
--ERROR:  could not find valid entry for job 'no_such_job'
--SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1');
-- schedule 
------------
--        1
--(1 row)
--
--SELECT cron.unschedule('testjob');
-- unschedule 
--------------
-- t
--(1 row)
--
---- Test cache invalidation
--DROP EXTENSION pg_cron;
--CREATE EXTENSION pg_cron VERSION '1.4';
--ALTER EXTENSION pg_cron UPDATE;
- -- Vacuum every day at 10:00am (GMT)
- SELECT cron.schedule('0 10 * * *', 'VACUUM');
-  schedule 
-@@ -300,8 +273,3 @@ SELECT jobid, jobname, schedule, command FROM cron.job ORDER BY jobid;
- SELECT cron.schedule('bad-last-dom-job1', '0 11 $foo * *', 'VACUUM FULL');
- ERROR:  invalid schedule: 0 11 $foo * *
- HINT:  Use cron format (e.g. 5 4 * * *), or interval format '[1-59] seconds'
---- cleaning
--DROP EXTENSION pg_cron;
--drop user pgcron_cront;
--drop database pgcron_dbno;
--drop database pgcron_dbyes;
-diff --git a/sql/pg_cron-test.sql b/sql/pg_cron-test.sql
-index 45f94d9..241cf73 100644
---- a/sql/pg_cron-test.sql
-+++ b/sql/pg_cron-test.sql
-@@ -1,17 +1,3 @@
--CREATE EXTENSION pg_cron VERSION '1.0';
--SELECT extversion FROM pg_extension WHERE extname='pg_cron';
---- Test binary compatibility with v1.4 function signature.
--ALTER EXTENSION pg_cron UPDATE TO '1.4';
--SELECT cron.unschedule(job_name := 'no_such_job');
--SELECT cron.schedule('testjob', '* * * * *', 'SELECT 1');
--SELECT cron.unschedule('testjob');
--
---- Test cache invalidation
--DROP EXTENSION pg_cron;
--CREATE EXTENSION pg_cron VERSION '1.4';
--
--ALTER EXTENSION pg_cron UPDATE;
--
- -- Vacuum every day at 10:00am (GMT)
- SELECT cron.schedule('0 10 * * *', 'VACUUM');
- 
-@@ -156,8 +142,3 @@ SELECT jobid, jobname, schedule, command FROM cron.job ORDER BY jobid;
- -- invalid last of day job
- SELECT cron.schedule('bad-last-dom-job1', '0 11 $foo * *', 'VACUUM FULL');
- 
---- cleaning
--DROP EXTENSION pg_cron;
--drop user pgcron_cront;
--drop database pgcron_dbno;
--drop database pgcron_dbyes;

docker-compose/ext-src/pg_cron-src/test-upgrade.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin'    --dbname=contrib_regression pg_cron-test

docker-compose/ext-src/pg_graphql-src/neon-test.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-set -ex
-cd "$(dirname "${0}")"
-dropdb --if-exists contrib_regression
-createdb contrib_regression
-PGXS="$(dirname "$(pg_config --pgxs)" )"
-REGRESS="${PGXS}/../test/regress/pg_regress"
-TESTDIR="test"
-TESTS=$(ls "${TESTDIR}/sql" | sort )
-TESTS=${TESTS//\.sql/}
-psql -v ON_ERROR_STOP=1 -f test/fixtures.sql -d contrib_regression
-${REGRESS} --use-existing --dbname=contrib_regression --inputdir=${TESTDIR} ${TESTS}
-

docker-compose/ext-src/pg_ivm-src/test-upgrade.patch

@@ -1,18 +0,0 @@
-diff --git a/expected/pg_ivm.out b/expected/pg_ivm.out
-index e8798ee..cca58d0 100644
---- a/expected/pg_ivm.out
-+++ b/expected/pg_ivm.out
-@@ -1,4 +1,3 @@
--CREATE EXTENSION pg_ivm;
- GRANT ALL ON SCHEMA public TO public;
- -- create a table to use as a basis for views and materialized views in various combinations
- CREATE TABLE mv_base_a (i int, j int);
-diff --git a/sql/pg_ivm.sql b/sql/pg_ivm.sql
-index d3c1a01..9382d7f 100644
---- a/sql/pg_ivm.sql
-+++ b/sql/pg_ivm.sql
-@@ -1,4 +1,3 @@
--CREATE EXTENSION pg_ivm;
- GRANT ALL ON SCHEMA public TO public;
- 
- -- create a table to use as a basis for views and materialized views in various combinations

docker-compose/ext-src/pg_ivm-src/test-upgrade.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin' --dbname=contrib_regression pg_ivm create_immv refresh_immv

docker-compose/ext-src/pg_roaringbitmap-src/test-upgrade.patch

@@ -1,25 +0,0 @@
-diff --git a/expected/roaringbitmap.out b/expected/roaringbitmap.out
-index de70531..a5f7c15 100644
---- a/expected/roaringbitmap.out
-+++ b/expected/roaringbitmap.out
-@@ -1,7 +1,6 @@
- --
- --  Test roaringbitmap extension
- --
--CREATE EXTENSION if not exists roaringbitmap;
- -- Test input and output
- set roaringbitmap.output_format='array';
- set extra_float_digits = 0;
-diff --git a/sql/roaringbitmap.sql b/sql/roaringbitmap.sql
-index a0e9c74..84bc966 100644
---- a/sql/roaringbitmap.sql
-+++ b/sql/roaringbitmap.sql
-@@ -2,8 +2,6 @@
- --  Test roaringbitmap extension
- --
- 
--CREATE EXTENSION if not exists roaringbitmap;
--
- -- Test input and output
- 
- set roaringbitmap.output_format='array';

docker-compose/ext-src/pg_roaringbitmap-src/test-upgrade.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin'    --dbname=contrib_regression roaringbitmap

docker-compose/ext-src/pg_semver-src/test-upgrade.patch

@@ -1,24 +0,0 @@
-diff --git a/test/sql/base.sql b/test/sql/base.sql
-index af599d8..2eed91b 100644
---- a/test/sql/base.sql
-+++ b/test/sql/base.sql
-@@ -2,7 +2,6 @@
- BEGIN;
- 
- \i test/pgtap-core.sql
--\i sql/semver.sql
- 
- SELECT plan(334);
- --SELECT * FROM no_plan();
-diff --git a/test/sql/corpus.sql b/test/sql/corpus.sql
-index 1f5f637..a519905 100644
---- a/test/sql/corpus.sql
-+++ b/test/sql/corpus.sql
-@@ -4,7 +4,6 @@ BEGIN;
- -- Test the SemVer corpus from https://regex101.com/r/Ly7O1x/3/.
- 
- \i test/pgtap-core.sql
--\i sql/semver.sql
- 
- SELECT plan(71);
- --SELECT * FROM no_plan();

docker-compose/ext-src/pg_semver-src/test-upgrade.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin'    --inputdir=test --dbname=contrib_regression base corpus

docker-compose/ext-src/pg_uuidv7-src/test-upgrade.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin'    --inputdir=test --dbname=contrib_regression  002_uuid_generate_v7 003_uuid_v7_to_timestamptz 004_uuid_timestamptz_to_v7 005_uuid_v7_to_timestamp 006_uuid_timestamp_to_v7

docker-compose/ext-src/pgvector-src/test-upgrade.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin' --inputdir=test --use-existing --dbname=contrib_regression bit btree cast copy halfvec hnsw_bit hnsw_halfvec hnsw_sparsevec hnsw_vector ivfflat_bit ivfflat_halfvec ivfflat_vector sparsevec vector_type

docker-compose/ext-src/plv8-src/test-upgrade.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin'  --use-existing --dbname=contrib_regression plv8 plv8-errors scalar_args inline json startup_pre startup varparam json_conv jsonb_conv window guc es6 arraybuffer composites currentresource startup_perms bytea find_function_perms memory_limits reset show array_spread regression dialect bigint procedure

docker-compose/ext-src/postgresql-unit-src/test-upgrade.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression extension tables unit binary unicode prefix units time temperature functions language_functions round derived compare aggregate iec custom crosstab convert

docker-compose/ext-src/prefix-src/test-upgrade.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --use-existing --inputdir=./ --bindir='/usr/local/pgsql/bin'    --dbname=contrib_regression prefix falcon explain queries

docker-compose/ext-src/rum-src/test-upgrade.patch

@@ -1,19 +0,0 @@
-diff --git a/expected/rum.out b/expected/rum.out
-index 5966d19..8860b79 100644
---- a/expected/rum.out
-+++ b/expected/rum.out
-@@ -1,4 +1,3 @@
--CREATE EXTENSION rum;
- CREATE TABLE test_rum( t text, a tsvector );
- CREATE TRIGGER tsvectorupdate
- BEFORE UPDATE OR INSERT ON test_rum
-diff --git a/sql/rum.sql b/sql/rum.sql
-index 8414bb9..898e6ab 100644
---- a/sql/rum.sql
-+++ b/sql/rum.sql
-@@ -1,5 +1,3 @@
--CREATE EXTENSION rum;
--
- CREATE TABLE test_rum( t text, a tsvector );
-
- CREATE TRIGGER tsvectorupdate

docker-compose/ext-src/rum-src/test-upgrade.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression rum rum_validate rum_hash ruminv timestamp orderby orderby_hash altorder altorder_hash limits int2 int4 int8 float4 float8 money oid time timetz date interval macaddr inet cidr text varchar char bytea bit varbit numeric rum_weight expr array

docker-compose/run-tests.sh

@@ -4,17 +4,14 @@ set -x
 cd /ext-src || exit 2
 FAILED=
 LIST=$( (echo -e "${SKIP//","/"\n"}"; ls -d -- *-src) | sort | uniq -u)
-for d in ${LIST}; do
-    [ -d "${d}" ] || continue
-    if ! psql -w -c "select 1" >/dev/null; then
-      FAILED="${d} ${FAILED}"
-      break
-    fi
-    if [ -f "${d}/neon-test.sh" ]; then
-       "${d}/neon-test.sh" || FAILED="${d} ${FAILED}"
-    else
+for d in ${LIST}
+do
+       [ -d "${d}" ] || continue
+       if ! psql -w -c "select 1" >/dev/null; then
+          FAILED="${d} ${FAILED}"
+          break
+       fi
        USE_PGXS=1 make -C "${d}" installcheck || FAILED="${d} ${FAILED}"
-    fi
 done
 [ -z "${FAILED}" ] && exit 0
 echo "${FAILED}"

docker-compose/test_extensions_upgrade.sh

@@ -1,93 +0,0 @@
-#!/bin/bash
-set -eux -o pipefail
-cd "$(dirname "${0}")"
-# Takes a variable name as argument. The result is stored in that variable.
-generate_id() {
-    local -n resvar=$1
-    printf -v resvar '%08x%08x%08x%08x' $SRANDOM $SRANDOM $SRANDOM $SRANDOM
-}
-if [ -z ${OLDTAG+x} ] || [ -z ${NEWTAG+x} ] || [ -z "${OLDTAG}" ] || [ -z "${NEWTAG}" ]; then
-  echo OLDTAG and NEWTAG must be defined
-  exit 1
-fi
-export PG_VERSION=${PG_VERSION:-16}
-function wait_for_ready {
-  TIME=0
-  while ! docker compose logs compute_is_ready | grep -q "accepting connections" && [ ${TIME} -le 300 ] ; do
-    ((TIME += 1 ))
-    sleep 1
-  done
-  if [ ${TIME} -gt 300 ]; then
-    echo Time is out.
-    exit 2
-  fi
-}
-function create_extensions() {
-  for ext in ${1}; do
-    docker compose exec neon-test-extensions psql -X -v ON_ERROR_STOP=1 -d contrib_regression -c "CREATE EXTENSION IF NOT EXISTS ${ext}"
-  done
-}
-EXTENSIONS='[
-{"extname": "plv8", "extdir": "plv8-src"},
-{"extname": "vector", "extdir": "pgvector-src"},
-{"extname": "unit", "extdir": "postgresql-unit-src"},
-{"extname": "hypopg", "extdir": "hypopg-src"},
-{"extname": "rum", "extdir": "rum-src"},
-{"extname": "ip4r", "extdir": "ip4r-src"},
-{"extname": "prefix", "extdir": "prefix-src"},
-{"extname": "hll", "extdir": "hll-src"},
-{"extname": "pg_cron", "extdir": "pg_cron-src"},
-{"extname": "pg_uuidv7", "extdir": "pg_uuidv7-src"},
-{"extname": "roaringbitmap", "extdir": "pg_roaringbitmap-src"},
-{"extname": "semver", "extdir": "pg_semver-src"},
-{"extname": "pg_ivm", "extdir": "pg_ivm-src"}
-]'
-EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -)
-TAG=${NEWTAG} docker compose --profile test-extensions up --quiet-pull --build -d
-wait_for_ready
-docker compose exec neon-test-extensions psql -c "DROP DATABASE IF EXISTS contrib_regression"
-docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression"
-create_extensions "${EXTNAMES}"
-query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /\',\'}')"
-new_vers=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query")
-docker compose --profile test-extensions down
-TAG=${OLDTAG} docker compose --profile test-extensions up --quiet-pull --build -d --force-recreate
-wait_for_ready
-docker compose cp  ext-src neon-test-extensions:/
-docker compose exec neon-test-extensions psql -c "DROP DATABASE IF EXISTS contrib_regression"
-docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression"
-create_extensions "${EXTNAMES}"
-query="select pge.extname from pg_extension pge join (select key as extname, value as extversion from json_each_text('${new_vers}')) x on pge.extname=x.extname and pge.extversion <> x.extversion"
-exts=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query")
-if [ -z "${exts}" ]; then
-  echo "No extensions were upgraded"
-else
-  tenant_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.tenant_id")
-  timeline_id=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id")
-  for ext in ${exts}; do
-    echo Testing ${ext}...
-    EXTDIR=$(echo ${EXTENSIONS} | jq -r '.[] | select(.extname=="'${ext}'") | .extdir')
-    generate_id new_timeline_id
-    PARAMS=(
-        -sbf
-        -X POST
-        -H "Content-Type: application/json"
-        -d "{\"new_timeline_id\": \"${new_timeline_id}\", \"pg_version\": ${PG_VERSION}, \"ancestor_timeline_id\": \"${timeline_id}\"}"
-        "http://127.0.0.1:9898/v1/tenant/${tenant_id}/timeline/"
-    )
-    result=$(curl "${PARAMS[@]}")
-    echo $result | jq .
-    TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} TAG=${OLDTAG} docker compose down compute compute_is_ready
-    COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} docker compose up --quiet-pull -d --build compute compute_is_ready
-    wait_for_ready
-    TID=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id")
-    if [ ${TID} != ${new_timeline_id} ]; then
-      echo Timeline mismatch
-      exit 1
-    fi
-    docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}"
-    docker compose exec neon-test-extensions sh -c /ext-src/${EXTDIR}/test-upgrade.sh
-    docker compose exec neon-test-extensions psql -d contrib_regression -c "alter extension ${ext} update"
-    docker compose exec neon-test-extensions psql -d contrib_regression -c "\dx ${ext}"
-  done
-fi

libs/compute_api/src/responses.rs

@@ -15,6 +15,11 @@ pub struct GenericAPIError {
     pub error: String,
 }
 
+#[derive(Debug, Clone, Serialize)]
+pub struct InfoResponse {
+    pub num_cpus: usize,
+}
+
 #[derive(Debug, Clone, Serialize)]
 pub struct ExtensionInstallResponse {
     pub extension: PgIdent,

libs/pageserver_api/src/config.rs

@@ -256,24 +256,16 @@ pub struct TenantConfigToml {
     pub compaction_period: Duration,
     /// Level0 delta layer threshold for compaction.
     pub compaction_threshold: usize,
-    /// Controls the amount of L0 included in a single compaction iteration.
-    /// The unit is `checkpoint_distance`, i.e., a size.
-    /// We add L0s to the set of layers to compact until their cumulative
-    /// size exceeds `compaction_upper_limit * checkpoint_distance`.
-    pub compaction_upper_limit: usize,
     pub compaction_algorithm: crate::models::CompactionAlgorithmSettings,
     /// Level0 delta layer threshold at which to delay layer flushes for compaction backpressure,
     /// such that they take 2x as long, and start waiting for layer flushes during ephemeral layer
     /// rolls. This helps compaction keep up with WAL ingestion, and avoids read amplification
-    /// blowing up. Should be >compaction_threshold. 0 to disable. Disabled by default.
+    /// blowing up. Should be >compaction_threshold. If None, defaults to 2 * compaction_threshold.
+    /// 0 to disable.
     pub l0_flush_delay_threshold: Option<usize>,
-    /// Level0 delta layer threshold at which to stall layer flushes. Must be >compaction_threshold
-    /// to avoid deadlock. 0 to disable. Disabled by default.
+    /// Level0 delta layer threshold at which to stall layer flushes. 0 to disable. If None,
+    /// defaults to 4 * compaction_threshold. Must be >compaction_threshold to avoid deadlock.
     pub l0_flush_stall_threshold: Option<usize>,
-    /// If true, Level0 delta layer flushes will wait for S3 upload before flushing the next
-    /// layer. This is a temporary backpressure mechanism which should be removed once
-    /// l0_flush_{delay,stall}_threshold is fully enabled.
-    pub l0_flush_wait_upload: bool,
     // Determines how much history is retained, to allow
     // branching and read replicas at an older point in time.
     // The unit is #of bytes of WAL.
@@ -528,17 +520,9 @@ pub mod tenant_conf_defaults {
 
     pub const DEFAULT_COMPACTION_PERIOD: &str = "20 s";
     pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10;
-
-    // This value needs to be tuned to avoid OOM. We have 3/4 of the total CPU threads to do background works, that's 16*3/4=9 on
-    // most of our pageservers. Compaction ~50 layers requires about 2GB memory (could be reduced later by optimizing L0 hole
-    // calculation to avoid loading all keys into the memory). So with this config, we can get a maximum peak compaction usage of 18GB.
-    pub const DEFAULT_COMPACTION_UPPER_LIMIT: usize = 50;
-
     pub const DEFAULT_COMPACTION_ALGORITHM: crate::models::CompactionAlgorithm =
         crate::models::CompactionAlgorithm::Legacy;
 
-    pub const DEFAULT_L0_FLUSH_WAIT_UPLOAD: bool = true;
-
     pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
 
     // Large DEFAULT_GC_PERIOD is fine as long as PITR_INTERVAL is larger.
@@ -574,13 +558,11 @@ impl Default for TenantConfigToml {
             compaction_period: humantime::parse_duration(DEFAULT_COMPACTION_PERIOD)
                 .expect("cannot parse default compaction period"),
             compaction_threshold: DEFAULT_COMPACTION_THRESHOLD,
-            compaction_upper_limit: DEFAULT_COMPACTION_UPPER_LIMIT,
             compaction_algorithm: crate::models::CompactionAlgorithmSettings {
                 kind: DEFAULT_COMPACTION_ALGORITHM,
             },
             l0_flush_delay_threshold: None,
             l0_flush_stall_threshold: None,
-            l0_flush_wait_upload: DEFAULT_L0_FLUSH_WAIT_UPLOAD,
             gc_horizon: DEFAULT_GC_HORIZON,
             gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)
                 .expect("cannot parse default gc period"),

libs/pageserver_api/src/models.rs

@@ -458,8 +458,6 @@ pub struct TenantConfigPatch {
     pub compaction_period: FieldPatch<String>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub compaction_threshold: FieldPatch<usize>,
-    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
-    pub compaction_upper_limit: FieldPatch<usize>,
     // defer parsing compaction_algorithm, like eviction_policy
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub compaction_algorithm: FieldPatch<CompactionAlgorithmSettings>,
@@ -468,8 +466,6 @@ pub struct TenantConfigPatch {
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub l0_flush_stall_threshold: FieldPatch<usize>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
-    pub l0_flush_wait_upload: FieldPatch<bool>,
-    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub gc_horizon: FieldPatch<u64>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub gc_period: FieldPatch<String>,
@@ -524,12 +520,10 @@ pub struct TenantConfig {
     pub compaction_target_size: Option<u64>,
     pub compaction_period: Option<String>,
     pub compaction_threshold: Option<usize>,
-    pub compaction_upper_limit: Option<usize>,
     // defer parsing compaction_algorithm, like eviction_policy
     pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
     pub l0_flush_delay_threshold: Option<usize>,
     pub l0_flush_stall_threshold: Option<usize>,
-    pub l0_flush_wait_upload: Option<bool>,
     pub gc_horizon: Option<u64>,
     pub gc_period: Option<String>,
     pub image_creation_threshold: Option<usize>,
@@ -562,11 +556,9 @@ impl TenantConfig {
             mut compaction_target_size,
             mut compaction_period,
             mut compaction_threshold,
-            mut compaction_upper_limit,
             mut compaction_algorithm,
             mut l0_flush_delay_threshold,
             mut l0_flush_stall_threshold,
-            mut l0_flush_wait_upload,
             mut gc_horizon,
             mut gc_period,
             mut image_creation_threshold,
@@ -598,9 +590,6 @@ impl TenantConfig {
             .apply(&mut compaction_target_size);
         patch.compaction_period.apply(&mut compaction_period);
         patch.compaction_threshold.apply(&mut compaction_threshold);
-        patch
-            .compaction_upper_limit
-            .apply(&mut compaction_upper_limit);
         patch.compaction_algorithm.apply(&mut compaction_algorithm);
         patch
             .l0_flush_delay_threshold
@@ -608,7 +597,6 @@ impl TenantConfig {
         patch
             .l0_flush_stall_threshold
             .apply(&mut l0_flush_stall_threshold);
-        patch.l0_flush_wait_upload.apply(&mut l0_flush_wait_upload);
         patch.gc_horizon.apply(&mut gc_horizon);
         patch.gc_period.apply(&mut gc_period);
         patch
@@ -660,11 +648,9 @@ impl TenantConfig {
             compaction_target_size,
             compaction_period,
             compaction_threshold,
-            compaction_upper_limit,
             compaction_algorithm,
             l0_flush_delay_threshold,
             l0_flush_stall_threshold,
-            l0_flush_wait_upload,
             gc_horizon,
             gc_period,
             image_creation_threshold,
@@ -1029,13 +1015,6 @@ pub struct TenantConfigPatchRequest {
     pub config: TenantConfigPatch, // as we have a flattened field, we should reject all unknown fields in it
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct TenantWaitLsnRequest {
-    #[serde(flatten)]
-    pub timelines: HashMap<TimelineId, Lsn>,
-    pub timeout: Duration,
-}
-
 /// See [`TenantState::attachment_status`] and the OpenAPI docs for context.
 #[derive(Serialize, Deserialize, Clone)]
 #[serde(tag = "slug", content = "data", rename_all = "snake_case")]

libs/pq_proto/src/lib.rs

@@ -182,13 +182,6 @@ pub struct CancelKeyData {
     pub cancel_key: i32,
 }
 
-pub fn id_to_cancel_key(id: u64) -> CancelKeyData {
-    CancelKeyData {
-        backend_pid: (id >> 32) as i32,
-        cancel_key: (id & 0xffffffff) as i32,
-    }
-}
-
 impl fmt::Display for CancelKeyData {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let hi = (self.backend_pid as u64) << 32;

libs/proxy/tokio-postgres2/Cargo.toml

@@ -19,4 +19,3 @@ postgres-protocol2 = { path = "../postgres-protocol2" }
 postgres-types2 = { path = "../postgres-types2" }
 tokio = { workspace = true, features = ["io-util", "time", "net"] }
 tokio-util = { workspace = true, features = ["codec"] }
-serde = { workspace = true, features = ["derive"] }

libs/proxy/tokio-postgres2/src/cancel_token.rs

@@ -3,13 +3,12 @@ use crate::tls::TlsConnect;
 
 use crate::{cancel_query, client::SocketConfig, tls::MakeTlsConnect};
 use crate::{cancel_query_raw, Error};
-use serde::{Deserialize, Serialize};
 use tokio::io::{AsyncRead, AsyncWrite};
 use tokio::net::TcpStream;
 
 /// The capability to request cancellation of in-progress queries on a
 /// connection.
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone)]
 pub struct CancelToken {
     pub socket_config: Option<SocketConfig>,
     pub ssl_mode: SslMode,

libs/proxy/tokio-postgres2/src/client.rs

@@ -18,7 +18,6 @@ use fallible_iterator::FallibleIterator;
 use futures_util::{future, ready, TryStreamExt};
 use parking_lot::Mutex;
 use postgres_protocol2::message::{backend::Message, frontend};
-use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::fmt;
 use std::sync::Arc;
@@ -138,7 +137,7 @@ impl InnerClient {
     }
 }
 
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone)]
 pub struct SocketConfig {
     pub host: Host,
     pub port: u16,

libs/proxy/tokio-postgres2/src/config.rs

@@ -7,7 +7,6 @@ use crate::tls::MakeTlsConnect;
 use crate::tls::TlsConnect;
 use crate::{Client, Connection, Error};
 use postgres_protocol2::message::frontend::StartupMessageParams;
-use serde::{Deserialize, Serialize};
 use std::fmt;
 use std::str;
 use std::time::Duration;
@@ -17,7 +16,7 @@ pub use postgres_protocol2::authentication::sasl::ScramKeys;
 use tokio::net::TcpStream;
 
 /// TLS configuration.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 #[non_exhaustive]
 pub enum SslMode {
     /// Do not use TLS.
@@ -51,7 +50,7 @@ pub enum ReplicationMode {
 }
 
 /// A host specification.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Host {
     /// A TCP hostname.
     Tcp(String),

libs/remote_storage/src/azure_blob.rs

@@ -377,8 +377,7 @@ impl RemoteStorage for AzureBlobStorage {
 
                 let next_item = next_item?;
 
-                // Log a warning if we saw two timeouts in a row before a successful request
-                if timeout_try_cnt > 2 {
+                if timeout_try_cnt >= 2 {
                     tracing::warn!("Azure Blob Storage list timed out and succeeded after {} tries", timeout_try_cnt);
                 }
                 timeout_try_cnt = 1;

libs/vm_monitor/src/dispatcher.rs

@@ -7,7 +7,7 @@
 //! (notifying it of upscale).
 
 use anyhow::{bail, Context};
-use axum::extract::ws::{Message, Utf8Bytes, WebSocket};
+use axum::extract::ws::{Message, WebSocket};
 use futures::{
     stream::{SplitSink, SplitStream},
     SinkExt, StreamExt,
@@ -82,21 +82,21 @@ impl Dispatcher {
 
         let highest_shared_version = match monitor_range.highest_shared_version(&agent_range) {
             Ok(version) => {
-                sink.send(Message::Text(Utf8Bytes::from(
+                sink.send(Message::Text(
                     serde_json::to_string(&ProtocolResponse::Version(version)).unwrap(),
-                )))
+                ))
                 .await
                 .context("failed to notify agent of negotiated protocol version")?;
                 version
             }
             Err(e) => {
-                sink.send(Message::Text(Utf8Bytes::from(
+                sink.send(Message::Text(
                     serde_json::to_string(&ProtocolResponse::Error(format!(
                         "Received protocol version range {} which does not overlap with {}",
                         agent_range, monitor_range
                     )))
                     .unwrap(),
-                )))
+                ))
                 .await
                 .context("failed to notify agent of no overlap between protocol version ranges")?;
                 Err(e).context("error determining suitable protocol version range")?
@@ -126,7 +126,7 @@ impl Dispatcher {
 
         let json = serde_json::to_string(&message).context("failed to serialize message")?;
         self.sink
-            .send(Message::Text(Utf8Bytes::from(json)))
+            .send(Message::Text(json))
             .await
             .context("stream error sending message")
     }

pageserver/Cargo.toml

@@ -36,7 +36,7 @@ itertools.workspace = true
 md5.workspace = true
 nix.workspace = true
 # hack to get the number of worker threads tokio uses
-num_cpus.workspace = true
+num_cpus = { version = "1.15" }
 num-traits.workspace = true
 once_cell.workspace = true
 pin-project-lite.workspace = true

pageserver/client/src/mgmt_api.rs

@@ -763,19 +763,4 @@ impl Client {
             .await
             .map_err(Error::ReceiveBody)
     }
-
-    pub async fn wait_lsn(
-        &self,
-        tenant_shard_id: TenantShardId,
-        request: TenantWaitLsnRequest,
-    ) -> Result<StatusCode> {
-        let uri = format!(
-            "{}/v1/tenant/{tenant_shard_id}/wait_lsn",
-            self.mgmt_api_endpoint,
-        );
-
-        self.request_noerror(Method::POST, uri, request)
-            .await
-            .map(|resp| resp.status())
-    }
 }

pageserver/compaction/src/simulator/draw.rs

@@ -160,12 +160,9 @@ pub fn draw_history<W: std::io::Write>(history: &[LayerTraceEvent], mut output:
 
         // Fill in and thicken rectangle if it's an
         // image layer so that we can see it.
-        let mut style = Style {
-            fill: Fill::Color(rgb(0x80, 0x80, 0x80)),
-            stroke: Stroke::Color(rgb(0, 0, 0), 0.5),
-            opacity: 1.0,
-            stroke_opacity: 1.0,
-        };
+        let mut style = Style::default();
+        style.fill = Fill::Color(rgb(0x80, 0x80, 0x80));
+        style.stroke = Stroke::Color(rgb(0, 0, 0), 0.5);
 
         let y_start = lsn_max - lsn_start;
         let y_end = lsn_max - lsn_end;
@@ -217,6 +214,10 @@ pub fn draw_history<W: std::io::Write>(history: &[LayerTraceEvent], mut output:
         files_seen.insert(f);
     }
 
+    let mut record_style = Style::default();
+    record_style.fill = Fill::Color(rgb(0x80, 0x80, 0x80));
+    record_style.stroke = Stroke::None;
+
     writeln!(svg, "{}", EndSvg)?;
 
     let mut layer_events_str = String::new();

pageserver/src/http/openapi_spec.yml

@@ -984,8 +984,6 @@ components:
           type: string
         compaction_threshold:
           type: string
-        compaction_upper_limit:
-          type: string
         image_creation_threshold:
           type: integer
         walreceiver_connect_timeout:

pageserver/src/http/routes.rs

@@ -10,7 +10,6 @@ use std::time::Duration;
 
 use anyhow::{anyhow, Context, Result};
 use enumset::EnumSet;
-use futures::future::join_all;
 use futures::StreamExt;
 use futures::TryFutureExt;
 use humantime::format_rfc3339;
@@ -41,7 +40,6 @@ use pageserver_api::models::TenantShardSplitRequest;
 use pageserver_api::models::TenantShardSplitResponse;
 use pageserver_api::models::TenantSorting;
 use pageserver_api::models::TenantState;
-use pageserver_api::models::TenantWaitLsnRequest;
 use pageserver_api::models::TimelineArchivalConfigRequest;
 use pageserver_api::models::TimelineCreateRequestMode;
 use pageserver_api::models::TimelineCreateRequestModeImportPgdata;
@@ -97,8 +95,6 @@ use crate::tenant::timeline::CompactOptions;
 use crate::tenant::timeline::CompactRequest;
 use crate::tenant::timeline::CompactionError;
 use crate::tenant::timeline::Timeline;
-use crate::tenant::timeline::WaitLsnTimeout;
-use crate::tenant::timeline::WaitLsnWaiter;
 use crate::tenant::GetTimelineError;
 use crate::tenant::OffloadedTimeline;
 use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError};
@@ -2794,63 +2790,6 @@ async fn secondary_download_handler(
     json_response(status, progress)
 }
 
-async fn wait_lsn_handler(
-    mut request: Request<Body>,
-    cancel: CancellationToken,
-) -> Result<Response<Body>, ApiError> {
-    let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
-    let wait_lsn_request: TenantWaitLsnRequest = json_request(&mut request).await?;
-
-    let state = get_state(&request);
-    let tenant = state
-        .tenant_manager
-        .get_attached_tenant_shard(tenant_shard_id)?;
-
-    let mut wait_futures = Vec::default();
-    for timeline in tenant.list_timelines() {
-        let Some(lsn) = wait_lsn_request.timelines.get(&timeline.timeline_id) else {
-            continue;
-        };
-
-        let fut = {
-            let timeline = timeline.clone();
-            let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Error);
-            async move {
-                timeline
-                    .wait_lsn(
-                        *lsn,
-                        WaitLsnWaiter::HttpEndpoint,
-                        WaitLsnTimeout::Custom(wait_lsn_request.timeout),
-                        &ctx,
-                    )
-                    .await
-            }
-        };
-        wait_futures.push(fut);
-    }
-
-    if wait_futures.is_empty() {
-        return json_response(StatusCode::NOT_FOUND, ());
-    }
-
-    let all_done = tokio::select! {
-        results = join_all(wait_futures) => {
-            results.iter().all(|res| res.is_ok())
-        },
-        _ = cancel.cancelled() => {
-            return Err(ApiError::Cancelled);
-        }
-    };
-
-    let status = if all_done {
-        StatusCode::OK
-    } else {
-        StatusCode::ACCEPTED
-    };
-
-    json_response(status, ())
-}
-
 async fn secondary_status_handler(
     request: Request<Body>,
     _cancel: CancellationToken,
@@ -3638,9 +3577,6 @@ pub fn make_router(
         .post("/v1/tenant/:tenant_shard_id/secondary/download", |r| {
             api_handler(r, secondary_download_handler)
         })
-        .post("/v1/tenant/:tenant_shard_id/wait_lsn", |r| {
-            api_handler(r, wait_lsn_handler)
-        })
         .put("/v1/tenant/:tenant_shard_id/break", |r| {
             testing_api_handler("set tenant state to broken", r, handle_tenant_break)
         })

pageserver/src/lib.rs

@@ -375,6 +375,48 @@ async fn timed_after_cancellation<Fut: std::future::Future>(
     }
 }
 
+async fn log_if_slow<Fut: std::future::Future>(
+    name: &str,
+    warn_after: std::time::Duration,
+    fut: Fut,
+) -> <Fut as std::future::Future>::Output {
+    let started = std::time::Instant::now();
+
+    let mut fut = std::pin::pin!(fut);
+
+    match tokio::time::timeout(warn_after, &mut fut).await {
+        Ok(ret) => ret,
+        Err(_) => {
+            tracing::info!(
+                what = name,
+                elapsed_ms = started.elapsed().as_millis(),
+                "slow future"
+            );
+
+            let res = fut.await;
+
+            tracing::info!(
+                what = name,
+                elapsed_ms = started.elapsed().as_millis(),
+                "slow future completed"
+            );
+
+            res
+        }
+    }
+}
+
+pub(crate) trait LogIfSlowFutureExt: std::future::Future {
+    async fn log_if_slow(self, name: &'static str, warn_after: std::time::Duration) -> Self::Output
+    where
+        Self: Sized,
+    {
+        log_if_slow(name, warn_after, self).await
+    }
+}
+
+impl<Fut> LogIfSlowFutureExt for Fut where Fut: std::future::Future {}
+
 #[cfg(test)]
 mod timed_tests {
     use super::timed;

pageserver/src/metrics.rs

@@ -1,18 +1,9 @@
-use std::collections::HashMap;
-use std::num::NonZeroUsize;
-use std::pin::Pin;
-use std::sync::atomic::AtomicU64;
-use std::sync::{Arc, Mutex};
-use std::task::{Context, Poll};
-use std::time::{Duration, Instant};
-
 use enum_map::EnumMap;
-use futures::Future;
 use metrics::{
     register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
     register_int_counter, register_int_counter_pair_vec, register_int_counter_vec,
     register_int_gauge, register_int_gauge_vec, register_uint_gauge, register_uint_gauge_vec,
-    Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
+    Counter, CounterVec, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterPair,
     IntCounterPairVec, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec,
 };
 use once_cell::sync::Lazy;
@@ -20,26 +11,13 @@ use pageserver_api::config::{
     PageServicePipeliningConfig, PageServicePipeliningConfigPipelined,
     PageServiceProtocolPipelinedExecutionStrategy,
 };
-use pageserver_api::models::InMemoryLayerInfo;
 use pageserver_api::shard::TenantShardId;
-use pin_project_lite::pin_project;
 use postgres_backend::{is_expected_io_error, QueryError};
 use pq_proto::framed::ConnectionError;
-
-use strum::{EnumCount, IntoEnumIterator as _, VariantNames};
+use strum::{EnumCount, VariantNames};
 use strum_macros::{IntoStaticStr, VariantNames};
 use utils::id::TimelineId;
 
-use crate::config::PageServerConf;
-use crate::context::{PageContentKind, RequestContext};
-use crate::task_mgr::TaskKind;
-use crate::tenant::layer_map::LayerMap;
-use crate::tenant::mgr::TenantSlot;
-use crate::tenant::storage_layer::{InMemoryLayer, PersistentLayerDesc};
-use crate::tenant::tasks::BackgroundLoopKind;
-use crate::tenant::throttle::ThrottleResult;
-use crate::tenant::Timeline;
-
 /// Prometheus histogram buckets (in seconds) for operations in the critical
 /// path. In other words, operations that directly affect that latency of user
 /// queries.
@@ -420,15 +398,6 @@ pub(crate) static WAIT_LSN_TIME: Lazy<Histogram> = Lazy::new(|| {
     .expect("failed to define a metric")
 });
 
-static FLUSH_WAIT_UPLOAD_TIME: Lazy<GaugeVec> = Lazy::new(|| {
-    register_gauge_vec!(
-        "pageserver_flush_wait_upload_seconds",
-        "Time spent waiting for preceding uploads during layer flush",
-        &["tenant_id", "shard_id", "timeline_id"]
-    )
-    .expect("failed to define a metric")
-});
-
 static LAST_RECORD_LSN: Lazy<IntGaugeVec> = Lazy::new(|| {
     register_int_gauge_vec!(
         "pageserver_last_record_lsn",
@@ -465,38 +434,18 @@ static PITR_HISTORY_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
     .expect("failed to define a metric")
 });
 
-#[derive(
-    strum_macros::EnumIter,
-    strum_macros::EnumString,
-    strum_macros::Display,
-    strum_macros::IntoStaticStr,
-)]
+#[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)]
 #[strum(serialize_all = "kebab_case")]
-pub(crate) enum LayerKind {
+pub(crate) enum MetricLayerKind {
     Delta,
     Image,
 }
 
-#[derive(
-    strum_macros::EnumIter,
-    strum_macros::EnumString,
-    strum_macros::Display,
-    strum_macros::IntoStaticStr,
-)]
-#[strum(serialize_all = "kebab_case")]
-pub(crate) enum LayerLevel {
-    // We don't track the currently open ephemeral layer, since there's always exactly 1 and its
-    // size changes. See `TIMELINE_EPHEMERAL_BYTES`.
-    Frozen,
-    L0,
-    L1,
-}
-
 static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
     register_uint_gauge_vec!(
         "pageserver_layer_bytes",
-        "Sum of frozen, L0, and L1 layer physical sizes in bytes (excluding the open ephemeral layer)",
-        &["tenant_id", "shard_id", "timeline_id", "level", "kind"]
+        "Sum of layer physical sizes in bytes",
+        &["tenant_id", "shard_id", "timeline_id", "kind"]
     )
     .expect("failed to define a metric")
 });
@@ -504,8 +453,8 @@ static TIMELINE_LAYER_SIZE: Lazy<UIntGaugeVec> = Lazy::new(|| {
 static TIMELINE_LAYER_COUNT: Lazy<UIntGaugeVec> = Lazy::new(|| {
     register_uint_gauge_vec!(
         "pageserver_layer_count",
-        "Number of frozen, L0, and L1 layers (excluding the open ephemeral layer)",
-        &["tenant_id", "shard_id", "timeline_id", "level", "kind"]
+        "Number of layers that exist",
+        &["tenant_id", "shard_id", "timeline_id", "kind"]
     )
     .expect("failed to define a metric")
 });
@@ -2620,7 +2569,6 @@ pub(crate) struct TimelineMetrics {
     timeline_id: String,
     pub flush_time_histo: StorageTimeMetrics,
     pub flush_delay_histo: StorageTimeMetrics,
-    pub flush_wait_upload_time_gauge: Gauge,
     pub compact_time_histo: StorageTimeMetrics,
     pub create_images_time_histo: StorageTimeMetrics,
     pub logical_size_histo: StorageTimeMetrics,
@@ -2632,6 +2580,10 @@ pub(crate) struct TimelineMetrics {
     pub disk_consistent_lsn_gauge: IntGauge,
     pub pitr_history_size: UIntGauge,
     pub archival_size: UIntGauge,
+    pub(crate) layer_size_image: UIntGauge,
+    pub(crate) layer_count_image: UIntGauge,
+    pub(crate) layer_size_delta: UIntGauge,
+    pub(crate) layer_count_delta: UIntGauge,
     pub standby_horizon_gauge: IntGauge,
     pub resident_physical_size_gauge: UIntGauge,
     pub visible_physical_size_gauge: UIntGauge,
@@ -2668,9 +2620,6 @@ impl TimelineMetrics {
             &shard_id,
             &timeline_id,
         );
-        let flush_wait_upload_time_gauge = FLUSH_WAIT_UPLOAD_TIME
-            .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
-            .unwrap();
         let compact_time_histo = StorageTimeMetrics::new(
             StorageTimeOperation::Compact,
             &tenant_id,
@@ -2729,6 +2678,42 @@ impl TimelineMetrics {
             .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
             .unwrap();
 
+        let layer_size_image = TIMELINE_LAYER_SIZE
+            .get_metric_with_label_values(&[
+                &tenant_id,
+                &shard_id,
+                &timeline_id,
+                MetricLayerKind::Image.into(),
+            ])
+            .unwrap();
+
+        let layer_count_image = TIMELINE_LAYER_COUNT
+            .get_metric_with_label_values(&[
+                &tenant_id,
+                &shard_id,
+                &timeline_id,
+                MetricLayerKind::Image.into(),
+            ])
+            .unwrap();
+
+        let layer_size_delta = TIMELINE_LAYER_SIZE
+            .get_metric_with_label_values(&[
+                &tenant_id,
+                &shard_id,
+                &timeline_id,
+                MetricLayerKind::Delta.into(),
+            ])
+            .unwrap();
+
+        let layer_count_delta = TIMELINE_LAYER_COUNT
+            .get_metric_with_label_values(&[
+                &tenant_id,
+                &shard_id,
+                &timeline_id,
+                MetricLayerKind::Delta.into(),
+            ])
+            .unwrap();
+
         let standby_horizon_gauge = STANDBY_HORIZON
             .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id])
             .unwrap();
@@ -2781,7 +2766,6 @@ impl TimelineMetrics {
             timeline_id,
             flush_time_histo,
             flush_delay_histo,
-            flush_wait_upload_time_gauge,
             compact_time_histo,
             create_images_time_histo,
             logical_size_histo,
@@ -2793,6 +2777,10 @@ impl TimelineMetrics {
             disk_consistent_lsn_gauge,
             pitr_history_size,
             archival_size,
+            layer_size_image,
+            layer_count_image,
+            layer_size_delta,
+            layer_count_delta,
             standby_horizon_gauge,
             resident_physical_size_gauge,
             visible_physical_size_gauge,
@@ -2827,100 +2815,6 @@ impl TimelineMetrics {
         self.resident_physical_size_gauge.get()
     }
 
-    pub(crate) fn flush_wait_upload_time_gauge_add(&self, duration: f64) {
-        self.flush_wait_upload_time_gauge.add(duration);
-        crate::metrics::FLUSH_WAIT_UPLOAD_TIME
-            .get_metric_with_label_values(&[&self.tenant_id, &self.shard_id, &self.timeline_id])
-            .unwrap()
-            .add(duration);
-    }
-
-    /// Generates TIMELINE_LAYER labels for a persistent layer.
-    fn make_layer_labels(&self, layer_desc: &PersistentLayerDesc) -> [&str; 5] {
-        let level = match LayerMap::is_l0(&layer_desc.key_range, layer_desc.is_delta()) {
-            true => LayerLevel::L0,
-            false => LayerLevel::L1,
-        };
-        let kind = match layer_desc.is_delta() {
-            true => LayerKind::Delta,
-            false => LayerKind::Image,
-        };
-        [
-            &self.tenant_id,
-            &self.shard_id,
-            &self.timeline_id,
-            level.into(),
-            kind.into(),
-        ]
-    }
-
-    /// Generates TIMELINE_LAYER labels for a frozen ephemeral layer.
-    fn make_frozen_layer_labels(&self, _layer: &InMemoryLayer) -> [&str; 5] {
-        [
-            &self.tenant_id,
-            &self.shard_id,
-            &self.timeline_id,
-            LayerLevel::Frozen.into(),
-            LayerKind::Delta.into(), // by definition
-        ]
-    }
-
-    /// Removes a frozen ephemeral layer to TIMELINE_LAYER metrics.
-    pub fn dec_frozen_layer(&self, layer: &InMemoryLayer) {
-        assert!(matches!(layer.info(), InMemoryLayerInfo::Frozen { .. }));
-        let labels = self.make_frozen_layer_labels(layer);
-        let size = layer.try_len().expect("frozen layer should have no writer");
-        TIMELINE_LAYER_COUNT
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .dec();
-        TIMELINE_LAYER_SIZE
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .sub(size);
-    }
-
-    /// Adds a frozen ephemeral layer to TIMELINE_LAYER metrics.
-    pub fn inc_frozen_layer(&self, layer: &InMemoryLayer) {
-        assert!(matches!(layer.info(), InMemoryLayerInfo::Frozen { .. }));
-        let labels = self.make_frozen_layer_labels(layer);
-        let size = layer.try_len().expect("frozen layer should have no writer");
-        TIMELINE_LAYER_COUNT
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .inc();
-        TIMELINE_LAYER_SIZE
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .add(size);
-    }
-
-    /// Removes a persistent layer from TIMELINE_LAYER metrics.
-    pub fn dec_layer(&self, layer_desc: &PersistentLayerDesc) {
-        let labels = self.make_layer_labels(layer_desc);
-        TIMELINE_LAYER_COUNT
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .dec();
-        TIMELINE_LAYER_SIZE
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .sub(layer_desc.file_size);
-    }
-
-    /// Adds a persistent layer to TIMELINE_LAYER metrics.
-    pub fn inc_layer(&self, layer_desc: &PersistentLayerDesc) {
-        let labels = self.make_layer_labels(layer_desc);
-        TIMELINE_LAYER_COUNT
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .inc();
-        TIMELINE_LAYER_SIZE
-            .get_metric_with_label_values(&labels)
-            .unwrap()
-            .add(layer_desc.file_size);
-    }
-
     pub(crate) fn shutdown(&self) {
         let was_shutdown = self
             .shutdown
@@ -2938,7 +2832,6 @@ impl TimelineMetrics {
         let shard_id = &self.shard_id;
         let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
         let _ = DISK_CONSISTENT_LSN.remove_label_values(&[tenant_id, shard_id, timeline_id]);
-        let _ = FLUSH_WAIT_UPLOAD_TIME.remove_label_values(&[tenant_id, shard_id, timeline_id]);
         let _ = STANDBY_HORIZON.remove_label_values(&[tenant_id, shard_id, timeline_id]);
         {
             RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(self.resident_physical_size_get());
@@ -2953,14 +2846,30 @@ impl TimelineMetrics {
         let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
         let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
 
-        for ref level in LayerLevel::iter() {
-            for ref kind in LayerKind::iter() {
-                let labels: [&str; 5] =
-                    [tenant_id, shard_id, timeline_id, level.into(), kind.into()];
-                let _ = TIMELINE_LAYER_SIZE.remove_label_values(&labels);
-                let _ = TIMELINE_LAYER_COUNT.remove_label_values(&labels);
-            }
-        }
+        let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
+            tenant_id,
+            shard_id,
+            timeline_id,
+            MetricLayerKind::Image.into(),
+        ]);
+        let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
+            tenant_id,
+            shard_id,
+            timeline_id,
+            MetricLayerKind::Image.into(),
+        ]);
+        let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[
+            tenant_id,
+            shard_id,
+            timeline_id,
+            MetricLayerKind::Delta.into(),
+        ]);
+        let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[
+            tenant_id,
+            shard_id,
+            timeline_id,
+            MetricLayerKind::Delta.into(),
+        ]);
 
         let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]);
         let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]);
@@ -3042,6 +2951,24 @@ pub(crate) fn remove_tenant_metrics(tenant_shard_id: &TenantShardId) {
     // we leave the BROKEN_TENANTS_SET entry if any
 }
 
+use futures::Future;
+use pin_project_lite::pin_project;
+use std::collections::HashMap;
+use std::num::NonZeroUsize;
+use std::pin::Pin;
+use std::sync::atomic::AtomicU64;
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+use std::time::{Duration, Instant};
+
+use crate::config::PageServerConf;
+use crate::context::{PageContentKind, RequestContext};
+use crate::task_mgr::TaskKind;
+use crate::tenant::mgr::TenantSlot;
+use crate::tenant::tasks::BackgroundLoopKind;
+use crate::tenant::throttle::ThrottleResult;
+use crate::tenant::Timeline;
+
 /// Maintain a per timeline gauge in addition to the global gauge.
 pub(crate) struct PerTimelineRemotePhysicalSizeGauge {
     last_set: AtomicU64,

pageserver/src/page_service.rs

@@ -1708,7 +1708,6 @@ impl PageServerHandler {
                 .wait_lsn(
                     not_modified_since,
                     crate::tenant::timeline::WaitLsnWaiter::PageService,
-                    timeline::WaitLsnTimeout::Default,
                     ctx,
                 )
                 .await?;
@@ -2045,7 +2044,6 @@ impl PageServerHandler {
                 .wait_lsn(
                     lsn,
                     crate::tenant::timeline::WaitLsnWaiter::PageService,
-                    crate::tenant::timeline::WaitLsnTimeout::Default,
                     ctx,
                 )
                 .await?;

pageserver/src/tenant.rs

@@ -37,8 +37,6 @@ use remote_timeline_client::manifest::{
     OffloadedTimelineManifest, TenantManifest, LATEST_TENANT_MANIFEST_VERSION,
 };
 use remote_timeline_client::UploadQueueNotReadyError;
-use remote_timeline_client::FAILED_REMOTE_OP_RETRIES;
-use remote_timeline_client::FAILED_UPLOAD_WARN_THRESHOLD;
 use std::collections::BTreeMap;
 use std::fmt;
 use std::future::Future;
@@ -2560,12 +2558,7 @@ impl Tenant {
                     // sizes etc. and that would get confused if the previous page versions
                     // are not in the repository yet.
                     ancestor_timeline
-                        .wait_lsn(
-                            *lsn,
-                            timeline::WaitLsnWaiter::Tenant,
-                            timeline::WaitLsnTimeout::Default,
-                            ctx,
-                        )
+                        .wait_lsn(*lsn, timeline::WaitLsnWaiter::Tenant, ctx)
                         .await
                         .map_err(|e| match e {
                             e @ (WaitLsnError::Timeout(_) | WaitLsnError::BadState { .. }) => {
@@ -3816,13 +3809,6 @@ impl Tenant {
             .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
     }
 
-    pub fn get_compaction_upper_limit(&self) -> usize {
-        let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
-        tenant_conf
-            .compaction_upper_limit
-            .unwrap_or(self.conf.default_tenant_conf.compaction_upper_limit)
-    }
-
     pub fn get_gc_horizon(&self) -> u64 {
         let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
         tenant_conf
@@ -5322,37 +5308,27 @@ impl Tenant {
             return Ok(());
         }
 
-        // Remote storage does no retries internally, so wrap it
-        match backoff::retry(
-            || async {
-                upload_tenant_manifest(
-                    &self.remote_storage,
-                    &self.tenant_shard_id,
-                    self.generation,
-                    &manifest,
-                    &self.cancel,
-                )
-                .await
-            },
-            |_e| self.cancel.is_cancelled(),
-            FAILED_UPLOAD_WARN_THRESHOLD,
-            FAILED_REMOTE_OP_RETRIES,
-            "uploading tenant manifest",
+        upload_tenant_manifest(
+            &self.remote_storage,
+            &self.tenant_shard_id,
+            self.generation,
+            &manifest,
             &self.cancel,
         )
         .await
-        {
-            None => Err(TenantManifestError::Cancelled),
-            Some(Err(_)) if self.cancel.is_cancelled() => Err(TenantManifestError::Cancelled),
-            Some(Err(e)) => Err(TenantManifestError::RemoteStorage(e)),
-            Some(Ok(_)) => {
-                // Store the successfully uploaded manifest, so that future callers can avoid
-                // re-uploading the same thing.
-                *guard = Some(manifest);
-
-                Ok(())
+        .map_err(|e| {
+            if self.cancel.is_cancelled() {
+                TenantManifestError::Cancelled
+            } else {
+                TenantManifestError::RemoteStorage(e)
             }
-        }
+        })?;
+
+        // Store the successfully uploaded manifest, so that future callers can avoid
+        // re-uploading the same thing.
+        *guard = Some(manifest);
+
+        Ok(())
     }
 }
 
@@ -5476,11 +5452,9 @@ pub(crate) mod harness {
                 compaction_target_size: Some(tenant_conf.compaction_target_size),
                 compaction_period: Some(tenant_conf.compaction_period),
                 compaction_threshold: Some(tenant_conf.compaction_threshold),
-                compaction_upper_limit: Some(tenant_conf.compaction_upper_limit),
                 compaction_algorithm: Some(tenant_conf.compaction_algorithm),
                 l0_flush_delay_threshold: tenant_conf.l0_flush_delay_threshold,
                 l0_flush_stall_threshold: tenant_conf.l0_flush_stall_threshold,
-                l0_flush_wait_upload: Some(tenant_conf.l0_flush_wait_upload),
                 gc_horizon: Some(tenant_conf.gc_horizon),
                 gc_period: Some(tenant_conf.gc_period),
                 image_creation_threshold: Some(tenant_conf.image_creation_threshold),

pageserver/src/tenant/config.rs

@@ -277,10 +277,6 @@ pub struct TenantConfOpt {
     #[serde(default)]
     pub compaction_threshold: Option<usize>,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
-    pub compaction_upper_limit: Option<usize>,
-
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(default)]
     pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
@@ -293,10 +289,6 @@ pub struct TenantConfOpt {
     #[serde(default)]
     pub l0_flush_stall_threshold: Option<usize>,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
-    pub l0_flush_wait_upload: Option<bool>,
-
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(default)]
     pub gc_horizon: Option<u64>,
@@ -405,9 +397,6 @@ impl TenantConfOpt {
             compaction_threshold: self
                 .compaction_threshold
                 .unwrap_or(global_conf.compaction_threshold),
-            compaction_upper_limit: self
-                .compaction_upper_limit
-                .unwrap_or(global_conf.compaction_upper_limit),
             compaction_algorithm: self
                 .compaction_algorithm
                 .as_ref()
@@ -419,9 +408,6 @@ impl TenantConfOpt {
             l0_flush_stall_threshold: self
                 .l0_flush_stall_threshold
                 .or(global_conf.l0_flush_stall_threshold),
-            l0_flush_wait_upload: self
-                .l0_flush_wait_upload
-                .unwrap_or(global_conf.l0_flush_wait_upload),
             gc_horizon: self.gc_horizon.unwrap_or(global_conf.gc_horizon),
             gc_period: self.gc_period.unwrap_or(global_conf.gc_period),
             image_creation_threshold: self
@@ -485,11 +471,9 @@ impl TenantConfOpt {
             mut compaction_target_size,
             mut compaction_period,
             mut compaction_threshold,
-            mut compaction_upper_limit,
             mut compaction_algorithm,
             mut l0_flush_delay_threshold,
             mut l0_flush_stall_threshold,
-            mut l0_flush_wait_upload,
             mut gc_horizon,
             mut gc_period,
             mut image_creation_threshold,
@@ -527,9 +511,6 @@ impl TenantConfOpt {
             .map(|v| humantime::parse_duration(&v))?
             .apply(&mut compaction_period);
         patch.compaction_threshold.apply(&mut compaction_threshold);
-        patch
-            .compaction_upper_limit
-            .apply(&mut compaction_upper_limit);
         patch.compaction_algorithm.apply(&mut compaction_algorithm);
         patch
             .l0_flush_delay_threshold
@@ -537,7 +518,6 @@ impl TenantConfOpt {
         patch
             .l0_flush_stall_threshold
             .apply(&mut l0_flush_stall_threshold);
-        patch.l0_flush_wait_upload.apply(&mut l0_flush_wait_upload);
         patch.gc_horizon.apply(&mut gc_horizon);
         patch
             .gc_period
@@ -607,11 +587,9 @@ impl TenantConfOpt {
             compaction_target_size,
             compaction_period,
             compaction_threshold,
-            compaction_upper_limit,
             compaction_algorithm,
             l0_flush_delay_threshold,
             l0_flush_stall_threshold,
-            l0_flush_wait_upload,
             gc_horizon,
             gc_period,
             image_creation_threshold,
@@ -669,10 +647,8 @@ impl From<TenantConfOpt> for models::TenantConfig {
             compaction_target_size: value.compaction_target_size,
             compaction_period: value.compaction_period.map(humantime),
             compaction_threshold: value.compaction_threshold,
-            compaction_upper_limit: value.compaction_upper_limit,
             l0_flush_delay_threshold: value.l0_flush_delay_threshold,
             l0_flush_stall_threshold: value.l0_flush_stall_threshold,
-            l0_flush_wait_upload: value.l0_flush_wait_upload,
             gc_horizon: value.gc_horizon,
             gc_period: value.gc_period.map(humantime),
             image_creation_threshold: value.image_creation_threshold,

pageserver/src/tenant/mgr.rs

@@ -1643,7 +1643,6 @@ impl TenantManager {
                         .wait_lsn(
                             *target_lsn,
                             crate::tenant::timeline::WaitLsnWaiter::Tenant,
-                            crate::tenant::timeline::WaitLsnTimeout::Default,
                             ctx,
                         )
                         .await

pageserver/src/tenant/remote_timeline_client/index.rs

@@ -222,10 +222,6 @@ impl LayerFileMetadata {
             shard,
         }
     }
-    /// Helper to get both generation and file size in a tuple
-    pub fn generation_file_size(&self) -> (Generation, u64) {
-        (self.generation, self.file_size)
-    }
 }
 
 /// Limited history of earlier ancestors.

pageserver/src/tenant/secondary/downloader.rs

@@ -559,13 +559,6 @@ impl JobGenerator<PendingDownload, RunningDownload, CompleteDownload, DownloadCo
     }
 }
 
-enum LayerAction {
-    Download,
-    NoAction,
-    Skip,
-    Touch,
-}
-
 /// This type is a convenience to group together the various functions involved in
 /// freshening a secondary tenant.
 struct TenantDownloader<'a> {
@@ -1015,15 +1008,67 @@ impl<'a> TenantDownloader<'a> {
                 return (Err(UpdateError::Restart), touched);
             }
 
-            match self.layer_action(&timeline_state, &layer).await {
-                LayerAction::Download => (),
-                LayerAction::NoAction => continue,
-                LayerAction::Skip => {
-                    self.skip_layer(layer);
-                    continue;
+            // Existing on-disk layers: just update their access time.
+            if let Some(on_disk) = timeline_state.on_disk_layers.get(&layer.name) {
+                tracing::debug!("Layer {} is already on disk", layer.name);
+
+                if cfg!(debug_assertions) {
+                    // Debug for https://github.com/neondatabase/neon/issues/6966: check that the files we think
+                    // are already present on disk are really there.
+                    match tokio::fs::metadata(&on_disk.local_path).await {
+                        Ok(meta) => {
+                            tracing::debug!(
+                                "Layer {} present at {}, size {}",
+                                layer.name,
+                                on_disk.local_path,
+                                meta.len(),
+                            );
+                        }
+                        Err(e) => {
+                            tracing::warn!(
+                                "Layer {} not found at {} ({})",
+                                layer.name,
+                                on_disk.local_path,
+                                e
+                            );
+                            debug_assert!(false);
+                        }
+                    }
                 }
-                LayerAction::Touch => {
+
+                if on_disk.metadata != layer.metadata || on_disk.access_time != layer.access_time {
+                    // We already have this layer on disk.  Update its access time.
+                    tracing::debug!(
+                        "Access time updated for layer {}: {} -> {}",
+                        layer.name,
+                        strftime(&on_disk.access_time),
+                        strftime(&layer.access_time)
+                    );
                     touched.push(layer);
+                }
+                continue;
+            } else {
+                tracing::debug!("Layer {} not present on disk yet", layer.name);
+            }
+
+            // Eviction: if we evicted a layer, then do not re-download it unless it was accessed more
+            // recently than it was evicted.
+            if let Some(evicted_at) = timeline_state.evicted_at.get(&layer.name) {
+                if &layer.access_time > evicted_at {
+                    tracing::info!(
+                        "Re-downloading evicted layer {}, accessed at {}, evicted at {}",
+                        layer.name,
+                        strftime(&layer.access_time),
+                        strftime(evicted_at)
+                    );
+                } else {
+                    tracing::trace!(
+                        "Not re-downloading evicted layer {}, accessed at {}, evicted at {}",
+                        layer.name,
+                        strftime(&layer.access_time),
+                        strftime(evicted_at)
+                    );
+                    self.skip_layer(layer);
                     continue;
                 }
             }
@@ -1046,86 +1091,6 @@ impl<'a> TenantDownloader<'a> {
         (Ok(()), touched)
     }
 
-    async fn layer_action(
-        &self,
-        timeline_state: &SecondaryDetailTimeline,
-        layer: &HeatMapLayer,
-    ) -> LayerAction {
-        // Existing on-disk layers: just update their access time.
-        if let Some(on_disk) = timeline_state.on_disk_layers.get(&layer.name) {
-            tracing::debug!("Layer {} is already on disk", layer.name);
-
-            if cfg!(debug_assertions) {
-                // Debug for https://github.com/neondatabase/neon/issues/6966: check that the files we think
-                // are already present on disk are really there.
-                match tokio::fs::metadata(&on_disk.local_path).await {
-                    Ok(meta) => {
-                        tracing::debug!(
-                            "Layer {} present at {}, size {}",
-                            layer.name,
-                            on_disk.local_path,
-                            meta.len(),
-                        );
-                    }
-                    Err(e) => {
-                        tracing::warn!(
-                            "Layer {} not found at {} ({})",
-                            layer.name,
-                            on_disk.local_path,
-                            e
-                        );
-                        debug_assert!(false);
-                    }
-                }
-            }
-
-            if on_disk.metadata.generation_file_size() != on_disk.metadata.generation_file_size() {
-                tracing::info!(
-                    "Re-downloading layer {} with changed size or generation: {:?}->{:?}",
-                    layer.name,
-                    on_disk.metadata.generation_file_size(),
-                    on_disk.metadata.generation_file_size()
-                );
-                return LayerAction::Download;
-            }
-            if on_disk.metadata != layer.metadata || on_disk.access_time != layer.access_time {
-                // We already have this layer on disk.  Update its access time.
-                tracing::debug!(
-                    "Access time updated for layer {}: {} -> {}",
-                    layer.name,
-                    strftime(&on_disk.access_time),
-                    strftime(&layer.access_time)
-                );
-                return LayerAction::Touch;
-            }
-            return LayerAction::NoAction;
-        } else {
-            tracing::debug!("Layer {} not present on disk yet", layer.name);
-        }
-
-        // Eviction: if we evicted a layer, then do not re-download it unless it was accessed more
-        // recently than it was evicted.
-        if let Some(evicted_at) = timeline_state.evicted_at.get(&layer.name) {
-            if &layer.access_time > evicted_at {
-                tracing::info!(
-                    "Re-downloading evicted layer {}, accessed at {}, evicted at {}",
-                    layer.name,
-                    strftime(&layer.access_time),
-                    strftime(evicted_at)
-                );
-            } else {
-                tracing::trace!(
-                    "Not re-downloading evicted layer {}, accessed at {}, evicted at {}",
-                    layer.name,
-                    strftime(&layer.access_time),
-                    strftime(evicted_at)
-                );
-                return LayerAction::Skip;
-            }
-        }
-        LayerAction::Download
-    }
-
     async fn download_timeline(
         &self,
         timeline: HeatMapTimeline,

pageserver/src/tenant/storage_layer.rs

@@ -33,7 +33,6 @@ use utils::sync::gate::GateGuard;
 
 use utils::lsn::Lsn;
 
-pub use batch_split_writer::{BatchLayerWriter, SplitDeltaLayerWriter, SplitImageLayerWriter};
 pub use delta_layer::{DeltaLayer, DeltaLayerWriter, ValueRef};
 pub use image_layer::{ImageLayer, ImageLayerWriter};
 pub use inmemory_layer::InMemoryLayer;

pageserver/src/tenant/storage_layer/batch_split_writer.rs

@@ -87,23 +87,6 @@ impl BatchLayerWriter {
         ));
     }
 
-    pub(crate) async fn finish(
-        self,
-        tline: &Arc<Timeline>,
-        ctx: &RequestContext,
-    ) -> anyhow::Result<Vec<ResidentLayer>> {
-        let res = self
-            .finish_with_discard_fn(tline, ctx, |_| async { false })
-            .await?;
-        let mut output = Vec::new();
-        for r in res {
-            if let BatchWriterResult::Produced(layer) = r {
-                output.push(layer);
-            }
-        }
-        Ok(output)
-    }
-
     pub(crate) async fn finish_with_discard_fn<D, F>(
         self,
         tline: &Arc<Timeline>,

pageserver/src/tenant/storage_layer/layer.rs

@@ -701,7 +701,13 @@ impl Drop for LayerInner {
         if let Some(timeline) = timeline.as_ref() {
             // Only need to decrement metrics if the timeline still exists: otherwise
             // it will have already de-registered these metrics via TimelineMetrics::shutdown
-            timeline.metrics.dec_layer(&self.desc);
+            if self.desc.is_delta() {
+                timeline.metrics.layer_count_delta.dec();
+                timeline.metrics.layer_size_delta.sub(self.desc.file_size);
+            } else {
+                timeline.metrics.layer_count_image.dec();
+                timeline.metrics.layer_size_image.sub(self.desc.file_size);
+            }
 
             if matches!(self.access_stats.visibility(), LayerVisibilityHint::Visible) {
                 debug_assert!(
@@ -811,7 +817,13 @@ impl LayerInner {
         };
 
         // This object acts as a RAII guard on these metrics: increment on construction
-        timeline.metrics.inc_layer(&desc);
+        if desc.is_delta() {
+            timeline.metrics.layer_count_delta.inc();
+            timeline.metrics.layer_size_delta.add(desc.file_size);
+        } else {
+            timeline.metrics.layer_count_image.inc();
+            timeline.metrics.layer_size_image.add(desc.file_size);
+        }
 
         // New layers are visible by default. This metric is later updated on drop or in set_visibility
         timeline

pageserver/src/tenant/timeline.rs

@@ -70,7 +70,6 @@ use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
 use std::time::{Duration, Instant, SystemTime};
 
 use crate::l0_flush::{self, L0FlushGlobalState};
-use crate::tenant::storage_layer::ImageLayerName;
 use crate::{
     aux_file::AuxFileSizeEstimator,
     page_service::TenantManagerTypes,
@@ -79,7 +78,7 @@ use crate::{
         layer_map::{LayerMap, SearchResult},
         metadata::TimelineMetadata,
         storage_layer::{
-            inmemory_layer::IndexEntry, BatchLayerWriter, IoConcurrency, PersistentLayerDesc,
+            inmemory_layer::IndexEntry, IoConcurrency, PersistentLayerDesc,
             ValueReconstructSituation,
         },
     },
@@ -145,19 +144,15 @@ use self::layer_manager::LayerManager;
 use self::logical_size::LogicalSize;
 use self::walreceiver::{WalReceiver, WalReceiverConf};
 
+use super::config::TenantConf;
+use super::remote_timeline_client::index::IndexPart;
+use super::remote_timeline_client::RemoteTimelineClient;
+use super::secondary::heatmap::{HeatMapLayer, HeatMapTimeline};
+use super::storage_layer::{LayerFringe, LayerVisibilityHint, ReadableLayer};
+use super::upload_queue::NotInitialized;
+use super::GcError;
 use super::{
-    config::TenantConf, storage_layer::LayerVisibilityHint, upload_queue::NotInitialized,
-    MaybeOffloaded,
-};
-use super::{debug_assert_current_span_has_tenant_and_timeline_id, AttachedTenantConf};
-use super::{remote_timeline_client::index::IndexPart, storage_layer::LayerFringe};
-use super::{
-    remote_timeline_client::RemoteTimelineClient, remote_timeline_client::WaitCompletionError,
-    storage_layer::ReadableLayer,
-};
-use super::{
-    secondary::heatmap::{HeatMapLayer, HeatMapTimeline},
-    GcError,
+    debug_assert_current_span_has_tenant_and_timeline_id, AttachedTenantConf, MaybeOffloaded,
 };
 
 #[cfg(test)]
@@ -902,17 +897,10 @@ impl From<GetReadyAncestorError> for PageReconstructError {
     }
 }
 
-pub(crate) enum WaitLsnTimeout {
-    Custom(Duration),
-    // Use the [`PageServerConf::wait_lsn_timeout`] default
-    Default,
-}
-
 pub(crate) enum WaitLsnWaiter<'a> {
     Timeline(&'a Timeline),
     Tenant,
     PageService,
-    HttpEndpoint,
 }
 
 /// Argument to [`Timeline::shutdown`].
@@ -934,7 +922,7 @@ pub(crate) enum ShutdownMode {
 }
 
 struct ImageLayerCreationOutcome {
-    unfinished_image_layer: Option<ImageLayerWriter>,
+    image: Option<ResidentLayer>,
     next_start_key: Key,
 }
 
@@ -1309,7 +1297,6 @@ impl Timeline {
         &self,
         lsn: Lsn,
         who_is_waiting: WaitLsnWaiter<'_>,
-        timeout: WaitLsnTimeout,
         ctx: &RequestContext, /* Prepare for use by cancellation */
     ) -> Result<(), WaitLsnError> {
         let state = self.current_state();
@@ -1326,7 +1313,7 @@ impl Timeline {
                 | TaskKind::WalReceiverConnectionPoller => {
                     let is_myself = match who_is_waiting {
                         WaitLsnWaiter::Timeline(waiter) => Weak::ptr_eq(&waiter.myself, &self.myself),
-                        WaitLsnWaiter::Tenant | WaitLsnWaiter::PageService | WaitLsnWaiter::HttpEndpoint => unreachable!("tenant or page_service context are not expected to have task kind {:?}", ctx.task_kind()),
+                        WaitLsnWaiter::Tenant | WaitLsnWaiter::PageService => unreachable!("tenant or page_service context are not expected to have task kind {:?}", ctx.task_kind()),
                     };
                     if is_myself {
                         if let Err(current) = self.last_record_lsn.would_wait_for(lsn) {
@@ -1342,14 +1329,13 @@ impl Timeline {
             }
         }
 
-        let timeout = match timeout {
-            WaitLsnTimeout::Custom(t) => t,
-            WaitLsnTimeout::Default => self.conf.wait_lsn_timeout,
-        };
-
         let _timer = crate::metrics::WAIT_LSN_TIME.start_timer();
 
-        match self.last_record_lsn.wait_for_timeout(lsn, timeout).await {
+        match self
+            .last_record_lsn
+            .wait_for_timeout(lsn, self.conf.wait_lsn_timeout)
+            .await
+        {
             Ok(()) => Ok(()),
             Err(e) => {
                 use utils::seqwait::SeqWaitError::*;
@@ -2181,17 +2167,9 @@ impl Timeline {
             .unwrap_or(self.conf.default_tenant_conf.compaction_threshold)
     }
 
-    fn get_compaction_upper_limit(&self) -> usize {
-        let tenant_conf = self.tenant_conf.load();
-        tenant_conf
-            .tenant_conf
-            .compaction_upper_limit
-            .unwrap_or(self.conf.default_tenant_conf.compaction_upper_limit)
-    }
-
     fn get_l0_flush_delay_threshold(&self) -> Option<usize> {
-        // Disable L0 flushes by default. This and compaction needs further tuning.
-        const DEFAULT_L0_FLUSH_DELAY_FACTOR: usize = 0; // TODO: default to e.g. 3
+        // Default to delay L0 flushes at 2x compaction threshold.
+        const DEFAULT_L0_FLUSH_DELAY_FACTOR: usize = 2;
 
         // If compaction is disabled, don't delay.
         if self.get_compaction_period() == Duration::ZERO {
@@ -2219,9 +2197,8 @@ impl Timeline {
     }
 
     fn get_l0_flush_stall_threshold(&self) -> Option<usize> {
-        // Disable L0 stalls by default. In ingest benchmarks, we see image compaction take >10
-        // minutes, blocking L0 compaction, and we can't stall L0 flushes for that long.
-        const DEFAULT_L0_FLUSH_STALL_FACTOR: usize = 0; // TODO: default to e.g. 5
+        // Default to stall L0 flushes at 4x compaction threshold.
+        const DEFAULT_L0_FLUSH_STALL_FACTOR: usize = 4;
 
         // If compaction is disabled, don't stall.
         if self.get_compaction_period() == Duration::ZERO {
@@ -2268,14 +2245,6 @@ impl Timeline {
         Some(max(l0_flush_stall_threshold, compaction_threshold))
     }
 
-    fn get_l0_flush_wait_upload(&self) -> bool {
-        let tenant_conf = self.tenant_conf.load();
-        tenant_conf
-            .tenant_conf
-            .l0_flush_wait_upload
-            .unwrap_or(self.conf.default_tenant_conf.l0_flush_wait_upload)
-    }
-
     fn get_image_creation_threshold(&self) -> usize {
         let tenant_conf = self.tenant_conf.load();
         tenant_conf
@@ -3468,13 +3437,6 @@ impl Timeline {
         let mut completed_keyspace = KeySpace::default();
         let mut image_covered_keyspace = KeySpaceRandomAccum::new();
 
-        // Prevent GC from progressing while visiting the current timeline.
-        // If we are GC-ing because a new image layer was added while traversing
-        // the timeline, then it will remove layers that are required for fulfilling
-        // the current get request (read-path cannot "look back" and notice the new
-        // image layer).
-        let _gc_cutoff_holder = timeline.get_latest_gc_cutoff_lsn();
-
         loop {
             if cancel.is_cancelled() {
                 return Err(GetVectoredError::Cancelled);
@@ -3615,12 +3577,7 @@ impl Timeline {
             }
         }
         ancestor
-            .wait_lsn(
-                self.ancestor_lsn,
-                WaitLsnWaiter::Timeline(self),
-                WaitLsnTimeout::Default,
-                ctx,
-            )
+            .wait_lsn(self.ancestor_lsn, WaitLsnWaiter::Timeline(self), ctx)
             .await
             .map_err(|e| match e {
                 e @ WaitLsnError::Timeout(_) => GetReadyAncestorError::AncestorLsnTimeout(e),
@@ -3703,7 +3660,7 @@ impl Timeline {
             let mut guard = self.layers.write().await;
             guard
                 .open_mut()?
-                .try_freeze_in_memory_layer(at, &self.last_freeze_at, write_lock, &self.metrics)
+                .try_freeze_in_memory_layer(at, &self.last_freeze_at, write_lock)
                 .await
         };
 
@@ -4070,27 +4027,6 @@ impl Timeline {
             // release lock on 'layers'
         };
 
-        // Backpressure mechanism: wait with continuation of the flush loop until we have uploaded all layer files.
-        // This makes us refuse ingest until the new layers have been persisted to the remote
-        // TODO: remove this, and rely on l0_flush_{delay,stall}_threshold instead.
-        if self.get_l0_flush_wait_upload() {
-            let start = Instant::now();
-            self.remote_client
-                .wait_completion()
-                .await
-                .map_err(|e| match e {
-                    WaitCompletionError::UploadQueueShutDownOrStopped
-                    | WaitCompletionError::NotInitialized(
-                        NotInitialized::ShuttingDown | NotInitialized::Stopped,
-                    ) => FlushLayerError::Cancelled,
-                    WaitCompletionError::NotInitialized(NotInitialized::Uninitialized) => {
-                        FlushLayerError::Other(anyhow!(e).into())
-                    }
-                })?;
-            let duration = start.elapsed().as_secs_f64();
-            self.metrics.flush_wait_upload_time_gauge_add(duration);
-        }
-
         // FIXME: between create_delta_layer and the scheduling of the upload in `update_metadata_file`,
         // a compaction can delete the file and then it won't be available for uploads any more.
         // We still schedule the upload, resulting in an error, but ideally we'd somehow avoid this
@@ -4421,15 +4357,11 @@ impl Timeline {
         if wrote_keys {
             // Normal path: we have written some data into the new image layer for this
             // partition, so flush it to disk.
-            info!(
-                "produced image layer for rel {}",
-                ImageLayerName {
-                    key_range: img_range.clone(),
-                    lsn
-                },
-            );
+            let (desc, path) = image_layer_writer.finish(ctx).await?;
+            let image_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
+            info!("created image layer for rel {}", image_layer.local_path());
             Ok(ImageLayerCreationOutcome {
-                unfinished_image_layer: Some(image_layer_writer),
+                image: Some(image_layer),
                 next_start_key: img_range.end,
             })
         } else {
@@ -4439,7 +4371,7 @@ impl Timeline {
             // layer we write will cover the key range that we just scanned.
             tracing::debug!("no data in range {}-{}", img_range.start, img_range.end);
             Ok(ImageLayerCreationOutcome {
-                unfinished_image_layer: None,
+                image: None,
                 next_start_key: start,
             })
         }
@@ -4488,7 +4420,7 @@ impl Timeline {
 
         if !trigger_generation && mode == ImageLayerCreationMode::Try {
             return Ok(ImageLayerCreationOutcome {
-                unfinished_image_layer: None,
+                image: None,
                 next_start_key: img_range.end,
             });
         }
@@ -4514,15 +4446,14 @@ impl Timeline {
         if wrote_any_image {
             // Normal path: we have written some data into the new image layer for this
             // partition, so flush it to disk.
+            let (desc, path) = image_layer_writer.finish(ctx).await?;
+            let image_layer = Layer::finish_creating(self.conf, self, desc, &path)?;
             info!(
                 "created image layer for metadata {}",
-                ImageLayerName {
-                    key_range: img_range.clone(),
-                    lsn
-                }
+                image_layer.local_path()
             );
             Ok(ImageLayerCreationOutcome {
-                unfinished_image_layer: Some(image_layer_writer),
+                image: Some(image_layer),
                 next_start_key: img_range.end,
             })
         } else {
@@ -4532,7 +4463,7 @@ impl Timeline {
             // layer we write will cover the key range that we just scanned.
             tracing::debug!("no data in range {}-{}", img_range.start, img_range.end);
             Ok(ImageLayerCreationOutcome {
-                unfinished_image_layer: None,
+                image: None,
                 next_start_key: start,
             })
         }
@@ -4599,6 +4530,7 @@ impl Timeline {
         ctx: &RequestContext,
     ) -> Result<Vec<ResidentLayer>, CreateImageLayersError> {
         let timer = self.metrics.create_images_time_histo.start_timer();
+        let mut image_layers = Vec::new();
 
         // We need to avoid holes between generated image layers.
         // Otherwise LayerMap::image_layer_exists will return false if key range of some layer is covered by more than one
@@ -4613,8 +4545,6 @@ impl Timeline {
 
         let check_for_image_layers = self.should_check_if_image_layers_required(lsn);
 
-        let mut batch_image_writer = BatchLayerWriter::new(self.conf).await?;
-
         for partition in partitioning.parts.iter() {
             if self.cancel.is_cancelled() {
                 return Err(CreateImageLayersError::Cancelled);
@@ -4687,45 +4617,45 @@ impl Timeline {
                     .map_err(|_| CreateImageLayersError::Cancelled)?,
             );
 
-            let ImageLayerCreationOutcome {
-                unfinished_image_layer,
-                next_start_key,
-            } = if !compact_metadata {
-                self.create_image_layer_for_rel_blocks(
-                    partition,
-                    image_layer_writer,
-                    lsn,
-                    ctx,
-                    img_range.clone(),
-                    start,
-                    io_concurrency,
-                )
-                .await?
+            if !compact_metadata {
+                let ImageLayerCreationOutcome {
+                    image,
+                    next_start_key,
+                } = self
+                    .create_image_layer_for_rel_blocks(
+                        partition,
+                        image_layer_writer,
+                        lsn,
+                        ctx,
+                        img_range,
+                        start,
+                        io_concurrency,
+                    )
+                    .await?;
+
+                start = next_start_key;
+                image_layers.extend(image);
             } else {
-                self.create_image_layer_for_metadata_keys(
-                    partition,
-                    image_layer_writer,
-                    lsn,
-                    ctx,
-                    img_range.clone(),
-                    mode,
-                    start,
-                    io_concurrency,
-                )
-                .await?
-            };
-            start = next_start_key;
-            if let Some(unfinished_image_layer) = unfinished_image_layer {
-                batch_image_writer.add_unfinished_image_writer(
-                    unfinished_image_layer,
-                    img_range,
-                    lsn,
-                );
+                let ImageLayerCreationOutcome {
+                    image,
+                    next_start_key,
+                } = self
+                    .create_image_layer_for_metadata_keys(
+                        partition,
+                        image_layer_writer,
+                        lsn,
+                        ctx,
+                        img_range,
+                        mode,
+                        start,
+                        io_concurrency,
+                    )
+                    .await?;
+                start = next_start_key;
+                image_layers.extend(image);
             }
         }
 
-        let image_layers = batch_image_writer.finish(self, ctx).await?;
-
         let mut guard = self.layers.write().await;
 
         // FIXME: we could add the images to be uploaded *before* returning from here, but right

pageserver/src/tenant/timeline/compaction.rs

@@ -7,6 +7,7 @@
 use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
 use std::ops::{Deref, Range};
 use std::sync::Arc;
+use std::time::Duration;
 
 use super::layer_manager::LayerManager;
 use super::{
@@ -47,7 +48,10 @@ use crate::tenant::timeline::{ImageLayerCreationOutcome, IoConcurrency};
 use crate::tenant::timeline::{Layer, ResidentLayer};
 use crate::tenant::{gc_block, DeltaLayer, MaybeOffloaded};
 use crate::virtual_file::{MaybeFatalIo, VirtualFile};
-use pageserver_api::config::tenant_conf_defaults::DEFAULT_CHECKPOINT_DISTANCE;
+use crate::LogIfSlowFutureExt;
+use pageserver_api::config::tenant_conf_defaults::{
+    DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD,
+};
 
 use pageserver_api::key::Key;
 use pageserver_api::keyspace::KeySpace;
@@ -624,13 +628,7 @@ impl Timeline {
 
         // High level strategy for compaction / image creation:
         //
-        // 1. First, do a L0 compaction to ensure we move the L0
-        // layers into the historic layer map get flat levels of
-        // layers. If we did not compact all L0 layers, we will
-        // prioritize compacting the timeline again and not do
-        // any of the compactions below.
-        //
-        // 2. Then, calculate the desired "partitioning" of the
+        // 1. First, calculate the desired "partitioning" of the
         // currently in-use key space. The goal is to partition the
         // key space into roughly fixed-size chunks, but also take into
         // account any existing image layers, and try to align the
@@ -644,7 +642,7 @@ impl Timeline {
         // identify a relation. This is just an optimization,
         // though.
         //
-        // 3. Once we know the partitioning, for each partition,
+        // 2. Once we know the partitioning, for each partition,
         // decide if it's time to create a new image layer. The
         // criteria is: there has been too much "churn" since the last
         // image layer? The "churn" is fuzzy concept, it's a
@@ -652,8 +650,15 @@ impl Timeline {
         // total in the delta file. Or perhaps: if creating an image
         // file would allow to delete some older files.
         //
-        // 4. In the end, if the tenant gets auto-sharded, we will run
-        // a shard-ancestor compaction.
+        // 3. After that, we compact all level0 delta files if there
+        // are too many of them.  While compacting, we also garbage
+        // collect any page versions that are no longer needed because
+        // of the new image layers we created in step 2.
+        //
+        // TODO: This high level strategy hasn't been implemented yet.
+        // Below are functions compact_level0() and create_image_layers()
+        // but they are a bit ad hoc and don't quite work like it's explained
+        // above. Rewrite it.
 
         // Is the timeline being deleted?
         if self.is_stopping() {
@@ -665,36 +670,15 @@ impl Timeline {
 
         // Define partitioning schema if needed
 
-        // 1. L0 Compact
-        let fully_compacted = {
-            let timer = self.metrics.compact_time_histo.start_timer();
-            let fully_compacted = self
-                .compact_level0(
-                    target_file_size,
-                    options.flags.contains(CompactFlags::ForceL0Compaction),
-                    ctx,
-                )
-                .await?;
-            timer.stop_and_record();
-            fully_compacted
-        };
-
-        if !fully_compacted {
-            // Yield and do not do any other kind of compaction. True means
-            // that we have pending L0 compaction tasks and the compaction scheduler
-            // will prioritize compacting this tenant/timeline again.
-            info!("skipping image layer generation and shard ancestor compaction due to L0 compaction did not include all layers.");
-            return Ok(true);
-        }
-
-        // 2. Repartition and create image layers if necessary
-        let partition_count = match self
+        // FIXME: the match should only cover repartitioning, not the next steps
+        let (partition_count, has_pending_tasks) = match self
             .repartition(
-                self.get_last_record_lsn(), // TODO: use L0-L1 boundary
+                self.get_last_record_lsn(),
                 self.get_compaction_target_size(),
                 options.flags,
                 ctx,
             )
+            .log_if_slow("compact_legacy_repartition", Duration::from_secs(30))
             .await
         {
             Ok(((dense_partitioning, sparse_partitioning), lsn)) => {
@@ -703,30 +687,50 @@ impl Timeline {
                     .access_stats_behavior(AccessStatsBehavior::Skip)
                     .build();
 
+                // 2. Compact
+                let timer = self.metrics.compact_time_histo.start_timer();
+                let fully_compacted = self
+                    .compact_level0(
+                        target_file_size,
+                        options.flags.contains(CompactFlags::ForceL0Compaction),
+                        ctx,
+                    )
+                    .await?;
+                timer.stop_and_record();
+
                 let mut partitioning = dense_partitioning;
                 partitioning
                     .parts
                     .extend(sparse_partitioning.into_dense().parts);
 
-                // 3. Create new image layers for partitions that have been modified "enough".
-                let image_layers = self
-                    .create_image_layers(
-                        &partitioning,
-                        lsn,
-                        if options
-                            .flags
-                            .contains(CompactFlags::ForceImageLayerCreation)
-                        {
-                            ImageLayerCreationMode::Force
-                        } else {
-                            ImageLayerCreationMode::Try
-                        },
-                        &image_ctx,
-                    )
-                    .await?;
+                // 3. Create new image layers for partitions that have been modified
+                // "enough". Skip image layer creation if L0 compaction cannot keep up.
+                if fully_compacted {
+                    let image_layers = self
+                        .create_image_layers(
+                            &partitioning,
+                            lsn,
+                            if options
+                                .flags
+                                .contains(CompactFlags::ForceImageLayerCreation)
+                            {
+                                ImageLayerCreationMode::Force
+                            } else {
+                                ImageLayerCreationMode::Try
+                            },
+                            &image_ctx,
+                        )
+                        .log_if_slow(
+                            "compact_legacy_create_image_layers",
+                            Duration::from_secs(120),
+                        )
+                        .await?;
 
-                self.upload_new_image_layers(image_layers)?;
-                partitioning.parts.len()
+                    self.upload_new_image_layers(image_layers)?;
+                } else {
+                    info!("skipping image layer generation due to L0 compaction did not include all layers.");
+                }
+                (partitioning.parts.len(), !fully_compacted)
             }
             Err(err) => {
                 // no partitioning? This is normal, if the timeline was just created
@@ -738,22 +742,22 @@ impl Timeline {
                 if !self.cancel.is_cancelled() && !err.is_cancelled() {
                     tracing::error!("could not compact, repartitioning keyspace failed: {err:?}");
                 }
-                1
+                (1, false)
             }
         };
 
-        // 4. Shard ancestor compaction
-
         if self.shard_identity.count >= ShardCount::new(2) {
             // Limit the number of layer rewrites to the number of partitions: this means its
             // runtime should be comparable to a full round of image layer creations, rather than
             // being potentially much longer.
             let rewrite_max = partition_count;
 
-            self.compact_shard_ancestors(rewrite_max, ctx).await?;
+            self.compact_shard_ancestors(rewrite_max, ctx)
+                .log_if_slow("compact_legacy_shard_ancestors", Duration::from_secs(300))
+                .await?;
         }
 
-        Ok(false)
+        Ok(has_pending_tasks)
     }
 
     /// Check for layers that are elegible to be rewritten:
@@ -1038,6 +1042,10 @@ impl Timeline {
                 &ctx,
             )
             .instrument(phase1_span)
+            .log_if_slow(
+                "compact_legacy_compact_level0_phase1",
+                Duration::from_secs(120),
+            )
             .await?
         };
 
@@ -1047,6 +1055,10 @@ impl Timeline {
         }
 
         self.finish_compact_batch(&new_layers, &Vec::new(), &deltas_to_compact)
+            .log_if_slow(
+                "compact_legacy_finish_compact_batch",
+                Duration::from_secs(20),
+            )
             .await?;
         Ok(fully_compacted)
     }
@@ -1119,10 +1131,17 @@ impl Timeline {
         // Accumulate the size of layers in `deltas_to_compact`
         let mut deltas_to_compact_bytes = 0;
 
-        // Under normal circumstances, we will accumulate up to compaction_upper_limit L0s of size
+        // Under normal circumstances, we will accumulate up to compaction_interval L0s of size
         // checkpoint_distance each.  To avoid edge cases using extra system resources, bound our
         // work in this function to only operate on this much delta data at once.
-        let delta_size_limit = self.get_compaction_upper_limit() as u64
+        //
+        // Take the max of the configured value & the default, so that tests that configure tiny values
+        // can still use a sensible amount of memory, but if a deployed system configures bigger values we
+        // still let them compact a full stack of L0s in one go.
+        let delta_size_limit = std::cmp::max(
+            self.get_compaction_threshold(),
+            DEFAULT_COMPACTION_THRESHOLD,
+        ) as u64
             * std::cmp::max(self.get_checkpoint_distance(), DEFAULT_CHECKPOINT_DISTANCE);
 
         let mut fully_compacted = true;
@@ -3195,7 +3214,7 @@ impl TimelineAdaptor {
         // TODO set proper (stateful) start. The create_image_layer_for_rel_blocks function mostly
         let start = Key::MIN;
         let ImageLayerCreationOutcome {
-            unfinished_image_layer,
+            image,
             next_start_key: _,
         } = self
             .timeline
@@ -3210,10 +3229,7 @@ impl TimelineAdaptor {
             )
             .await?;
 
-        if let Some(image_layer_writer) = unfinished_image_layer {
-            let (desc, path) = image_layer_writer.finish(ctx).await?;
-            let image_layer =
-                Layer::finish_creating(self.timeline.conf, &self.timeline, desc, &path)?;
+        if let Some(image_layer) = image {
             self.new_images.push(image_layer);
         }
 

pageserver/src/tenant/timeline/import_pgdata.rs

@@ -113,7 +113,7 @@ pub async fn doit(
             match res {
                 Ok(_) => break,
                 Err(err) => {
-                    info!(?err, "indefinitely waiting for pgdata to finish");
+                    info!(?err, "indefintely waiting for pgdata to finish");
                     if tokio::time::timeout(std::time::Duration::from_secs(10), cancel.cancelled())
                         .await
                         .is_ok()

pageserver/src/tenant/timeline/import_pgdata/importbucket_client.rs

@@ -308,7 +308,7 @@ impl ControlFile {
             202107181 => 14,
             202209061 => 15,
             202307071 => 16,
-            202406281 => 17,
+            /* XXX pg17 */
             catversion => {
                 anyhow::bail!("unrecognized catalog version {catversion}")
             }

pageserver/src/tenant/timeline/layer_manager.rs

@@ -91,7 +91,6 @@ impl LayerManager {
                 layer_map,
                 layer_fmgr: LayerFileManager(hashmap),
             }) => {
-                // NB: no need to decrement layer metrics; metrics are removed on timeline shutdown.
                 let open = layer_map.open_layer.take();
                 let frozen = layer_map.frozen_layers.len();
                 let taken_writer_state = writer_state.take();
@@ -235,7 +234,6 @@ impl OpenLayerManager {
         lsn: Lsn,
         last_freeze_at: &AtomicLsn,
         write_lock: &mut tokio::sync::MutexGuard<'_, Option<TimelineWriterState>>,
-        metrics: &TimelineMetrics,
     ) -> bool {
         let Lsn(last_record_lsn) = lsn;
         let end_lsn = Lsn(last_record_lsn + 1);
@@ -244,11 +242,6 @@ impl OpenLayerManager {
             let open_layer_rc = Arc::clone(open_layer);
             open_layer.freeze(end_lsn).await;
 
-            // Increment the frozen layer metrics. This is decremented in `finish_flush_l0_layer()`.
-            // TODO: It would be nicer to do this via `InMemoryLayer::drop()`, but it requires a
-            // reference to the timeline metrics. Other methods use a metrics borrow as well.
-            metrics.inc_frozen_layer(open_layer);
-
             // The layer is no longer open, update the layer map to reflect this.
             // We will replace it with on-disk historics below.
             self.layer_map.frozen_layers.push_back(open_layer_rc);
@@ -305,7 +298,6 @@ impl OpenLayerManager {
             .frozen_layers
             .pop_front()
             .expect("there must be a inmem layer to flush");
-        metrics.dec_frozen_layer(&inmem);
 
         // Only one task may call this function at a time (for this
         // timeline). If two tasks tried to flush the same frozen

pageserver/src/tenant/timeline/walreceiver/connection_manager.rs

@@ -164,10 +164,9 @@ pub(super) async fn connection_manager_loop_step(
                     Ok(Some(broker_update)) => connection_manager_state.register_timeline_update(broker_update),
                     Err(status) => {
                         match status.code() {
-                            Code::Unknown if status.message().contains("stream closed because of a broken pipe") || status.message().contains("connection reset") || status.message().contains("error reading a body from connection") => {
+                            Code::Unknown if status.message().contains("stream closed because of a broken pipe") || status.message().contains("connection reset") => {
                                 // tonic's error handling doesn't provide a clear code for disconnections: we get
                                 // "h2 protocol error: error reading a body from connection: stream closed because of a broken pipe"
-                                // => https://github.com/neondatabase/neon/issues/9562
                                 info!("broker disconnected: {status}");
                             },
                             _ => {
@@ -274,7 +273,7 @@ pub(super) async fn connection_manager_loop_step(
                     };
 
                 last_discovery_ts = Some(std::time::Instant::now());
-                info!("No active connection and no candidates, sending discovery request to the broker");
+                debug!("No active connection and no candidates, sending discovery request to the broker");
 
                 // Cancellation safety: we want to send a message to the broker, but publish_one()
                 // function can get cancelled by the other select! arm. This is absolutely fine, because

pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs

@@ -118,7 +118,7 @@ pub(super) async fn handle_walreceiver_connection(
     cancellation: CancellationToken,
     connect_timeout: Duration,
     ctx: RequestContext,
-    safekeeper_node: NodeId,
+    node: NodeId,
     ingest_batch_size: u64,
 ) -> Result<(), WalReceiverError> {
     debug_assert_current_span_has_tenant_and_timeline_id();
@@ -140,7 +140,7 @@ pub(super) async fn handle_walreceiver_connection(
 
     let (replication_client, connection) = {
         let mut config = wal_source_connconf.to_tokio_postgres_config();
-        config.application_name(format!("pageserver-{}", timeline.conf.id.0).as_str());
+        config.application_name(format!("pageserver-{}", node.0).as_str());
         config.replication_mode(tokio_postgres::config::ReplicationMode::Physical);
         match time::timeout(connect_timeout, config.connect(postgres::NoTls)).await {
             Ok(client_and_conn) => client_and_conn?,
@@ -162,7 +162,7 @@ pub(super) async fn handle_walreceiver_connection(
         latest_wal_update: Utc::now().naive_utc(),
         streaming_lsn: None,
         commit_lsn: None,
-        node: safekeeper_node,
+        node,
     };
     if let Err(e) = events_sender.send(TaskStateUpdate::Progress(connection_status)) {
         warn!("Wal connection event listener dropped right after connection init, aborting the connection: {e}");

pageserver/src/utilization.rs

@@ -49,7 +49,7 @@ pub(crate) fn regenerate(
     };
 
     // Express a static value for how many shards we may schedule on one node
-    const MAX_SHARDS: u32 = 5000;
+    const MAX_SHARDS: u32 = 20000;
 
     let mut doc = PageserverUtilization {
         disk_usage_bytes: used,

pgxn/neon/file_cache.c

@@ -480,7 +480,7 @@ lfc_cache_contains(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno)
 	if (LFC_ENABLED())
 	{
 		entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL);
-		found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & ((uint32)1 << (chunk_offs & 31))) != 0;
+		found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) != 0;
 	}
 	LWLockRelease(lfc_lock);
 	return found;
@@ -527,7 +527,7 @@ lfc_cache_containsv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 				for (; chunk_offs < BLOCKS_PER_CHUNK && i < nblocks; chunk_offs++, i++)
 				{
 					if ((entry->bitmap[chunk_offs >> 5] & 
-						((uint32)1 << (chunk_offs & 31))) != 0)
+						(1 << (chunk_offs & 31))) != 0)
 					{
 						BITMAP_SET(bitmap, i);
 						found++;
@@ -620,7 +620,7 @@ lfc_evict(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno)
 	}
 
 	/* remove the page from the cache */
-	entry->bitmap[chunk_offs >> 5] &= ~((uint32)1 << (chunk_offs & (32 - 1)));
+	entry->bitmap[chunk_offs >> 5] &= ~(1 << (chunk_offs & (32 - 1)));
 
 	if (entry->access_count == 0)
 	{
@@ -774,7 +774,7 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 			 * If the page is valid, we consider it "read".
 			 * All other pages will be fetched separately by the next cache
 			 */
-			if (entry->bitmap[(chunk_offs + i) / 32] & ((uint32)1 << ((chunk_offs + i) % 32)))
+			if (entry->bitmap[(chunk_offs + i) / 32] & (1 << ((chunk_offs + i) % 32)))
 			{
 				BITMAP_SET(mask, buf_offset + i);
 				iteration_hits++;
@@ -1034,7 +1034,7 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 				{
 					lfc_ctl->used_pages += 1 - ((entry->bitmap[(chunk_offs + i) >> 5] >> ((chunk_offs + i) & 31)) & 1);
 					entry->bitmap[(chunk_offs + i) >> 5] |=
-						((uint32)1 << ((chunk_offs + i) & 31));
+						(1 << ((chunk_offs + i) & 31));
 				}
 			}
 
@@ -1282,7 +1282,7 @@ local_cache_pages(PG_FUNCTION_ARGS)
 			{
 				for (int i = 0; i < BLOCKS_PER_CHUNK; i++)
 				{
-					if (entry->bitmap[i >> 5] & ((uint32)1 << (i & 31)))
+					if (entry->bitmap[i >> 5] & (1 << (i & 31)))
 					{
 						fctx->record[n].pageoffs = entry->offset * BLOCKS_PER_CHUNK + i;
 						fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key));