Fix concurrency

.github/workflows/_benchmarking_preparation.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ aws-rds-postgres, aws-aurora-serverless-v2-postgres, neon, neon_pg17, neon_pg18 ]
+        platform: [ aws-rds-postgres, aws-aurora-serverless-v2-postgres, neon, neon_pg17 ]
         database: [ clickbench, tpch, userexample ]
 
     env:
@@ -52,9 +52,6 @@ jobs:
           neon_pg17)
             CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR_PG17 }}
             ;;
-          neon_pg18)
-            CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR_PG18 }}
-            ;;
           aws-rds-postgres)
             CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CONNSTR }}
             ;;

.github/workflows/benchmarking.yml

@@ -142,10 +142,6 @@ jobs:
             PLATFORM: "neon-staging"
             region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
             RUNNER: [ self-hosted, us-east-2, x64 ]
-          - PG_VERSION: 18
-            PLATFORM: "neon-staging"
-            region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
-            RUNNER: [ self-hosted, us-east-2, x64 ]
           - PG_VERSION: 16
             PLATFORM: "azure-staging"
             region_id: 'azure-eastus2'
@@ -263,7 +259,7 @@ jobs:
       id-token: write # aws-actions/configure-aws-credentials
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      DEFAULT_PG_VERSION: 18
+      DEFAULT_PG_VERSION: 17
       TEST_OUTPUT: /tmp/test_output
       BUILD_TYPE: remote
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
@@ -546,11 +542,7 @@ jobs:
                       { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier",       "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'"                               },
                       { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
                       { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
-                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
-                      { "pg_version": 18, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier",       "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'"                               },
-                      { "pg_version": 18, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
-                      { "pg_version": 18, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               },
-                      { "pg_version": 18, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               }]
+                      { "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new",            "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'"                               }]
         }'
 
         if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
@@ -568,7 +560,7 @@ jobs:
             "neonvm-captest-reuse"
           ],
           "pg_version" : [
-            16,17,18
+            16,17
           ]
         }'
 
@@ -587,7 +579,7 @@ jobs:
             "neonvm-captest-reuse"
           ],
           "pg_version" : [
-            16,17,18
+            16,17
           ]
         }'
 
@@ -801,9 +793,6 @@ jobs:
           - PLATFORM: "neonvm-captest-pgvector-pg17"
             RUNNER: [ self-hosted, us-east-2, x64 ]
             postgres_version: 17
-          - PLATFORM: "neonvm-captest-pgvector-pg18"
-            RUNNER: [ self-hosted, us-east-2, x64 ]
-            postgres_version: 18
           - PLATFORM: "azure-captest-pgvector"
             RUNNER: [ self-hosted, eastus2, x64 ]
             postgres_version: 16
@@ -860,9 +849,6 @@ jobs:
           neonvm-captest-pgvector-pg17)
             CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_PG17 }}
             ;;
-          neonvm-captest-pgvector-pg18)
-            CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_PG18 }}
-            ;;
           azure-captest-pgvector)
             CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_AZURE }}
             ;;
@@ -999,9 +985,6 @@ jobs:
               17)
                 CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_CONNSTR_PG17 }}
                 ;;
-              18)
-                CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_CONNSTR_PG18 }}
-                ;;
               *)
                 echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
                 exit 1
@@ -1127,9 +1110,6 @@ jobs:
               17)
                 CONNSTR_SECRET_NAME="BENCHMARK_CAPTEST_TPCH_CONNSTR_PG17"
                 ;;
-              18)
-                CONNSTR_SECRET_NAME="BENCHMARK_CAPTEST_TPCH_CONNSTR_PG18"
-                ;;
               *)
                 echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
                 exit 1
@@ -1255,9 +1235,6 @@ jobs:
               17)
                 CONNSTR=${{ secrets.BENCHMARK_CAPTEST_USER_EXAMPLE_CONNSTR_PG17 }}
                 ;;
-              18)
-                CONNSTR=${{ secrets.BENCHMARK_CAPTEST_USER_EXAMPLE_CONNSTR_PG18 }}
-                ;;
               *)
                 echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
                 exit 1

.github/workflows/ingest_benchmark.yml

@@ -49,10 +49,6 @@ jobs:
             stripe_size: null # run with neon defaults which will shard split only when reaching the threshold
             disable_sharding: false
             postgres_version: 17
-          - target_project: new_empty_project
-            stripe_size: null # run with neon defaults which will shard split only when reaching the threshold
-            disable_sharding: false
-            postgres_version: 18
           - target_project: large_existing_project
             stripe_size: null # cannot re-shared or choose different stripe size for existing, already sharded project
             disable_sharding: false

compute/compute-node.Dockerfile

@@ -1143,6 +1143,23 @@ RUN cargo install --locked --version 0.14.1 cargo-pgrx && \
 
 USER root
 
+#########################################################################################
+#
+# Layer "rust extensions pgrx16"
+#
+# Version 16 is now required by a few
+# This layer should be used as a base for new pgrx extensions,
+# and eventually get merged with `rust-extensions-build`
+#
+#########################################################################################
+FROM pg-build-with-cargo AS rust-extensions-build-pgrx16
+ARG PG_VERSION
+
+RUN cargo install --locked --version 0.16.0 cargo-pgrx && \
+    /bin/bash -c 'cargo pgrx init --pg${PG_VERSION:1}=/usr/local/pgsql/bin/pg_config'
+
+USER root
+
 #########################################################################################
 #
 # Layers "pg-onnx-build" and "pgrag-build"
@@ -1491,19 +1508,15 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) && \
 FROM build-deps AS pg_mooncake-src
 ARG PG_VERSION
 WORKDIR /ext-src
-COPY compute/patches/duckdb_v113.patch .
-RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.2/pg_mooncake-0.1.2.tar.gz -O pg_mooncake.tar.gz && \
-    echo "4550473784fcdd2e1e18062bc01eb9c286abd27cdf5e11a4399be6c0a426ba90 pg_mooncake.tar.gz" | sha256sum --check && \
-    mkdir pg_mooncake-src && cd pg_mooncake-src && tar xzf ../pg_mooncake.tar.gz --strip-components=1 -C . && \
-    cd third_party/duckdb && patch -p1 < /ext-src/duckdb_v113.patch && cd ../.. && \
-    echo "make -f pg_mooncake-src/Makefile.build installcheck TEST_DIR=./test SQL_DIR=./sql SRC_DIR=./src" > neon-test.sh && \
-    chmod a+x neon-test.sh
+RUN git clone --depth 1 --branch poc-08.29 https://github.com/Mooncake-Labs/pg_mooncake.git pg_mooncake-src && \
+    cd pg_mooncake-src && \
+    git submodule update --init --recursive && \
+    sed -i 's/pgrx = "0.16.0"/pgrx = { version = "0.16.0", features = [ "unsafe-postgres" ] }/g' Cargo.toml
 
-FROM rust-extensions-build AS pg_mooncake-build
+FROM rust-extensions-build-pgrx16 AS pg_mooncake-build
 COPY --from=pg_mooncake-src /ext-src/ /ext-src/
 WORKDIR /ext-src/pg_mooncake-src
-RUN make release -j $(getconf _NPROCESSORS_ONLN) && \
-    make install -j $(getconf _NPROCESSORS_ONLN) && \
+RUN make install PG_VERSION=pg${PG_VERSION#v} && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_mooncake.control
 
 #########################################################################################
@@ -1512,27 +1525,27 @@ RUN make release -j $(getconf _NPROCESSORS_ONLN) && \
 # compile pg_duckdb extension
 #
 #########################################################################################
-FROM build-deps AS pg_duckdb-src
-WORKDIR /ext-src
-COPY compute/patches/pg_duckdb_v031.patch .
-COPY compute/patches/duckdb_v120.patch .
-# pg_duckdb build requires source dir to be a git repo to get submodules
-# allow {privileged_role_name} to execute some functions that in pg_duckdb are available to superuser only:
-# - extension management function duckdb.install_extension()
-# - access to duckdb.extensions table and its sequence
-RUN git clone --depth 1 --branch v0.3.1 https://github.com/duckdb/pg_duckdb.git pg_duckdb-src && \
-    cd pg_duckdb-src && \
-    git submodule update --init --recursive && \
-    patch -p1 < /ext-src/pg_duckdb_v031.patch && \
-    cd third_party/duckdb && \
-    patch -p1 < /ext-src/duckdb_v120.patch
+# FROM build-deps AS pg_duckdb-src
+# WORKDIR /ext-src
+# COPY compute/patches/pg_duckdb_v031.patch .
+# COPY compute/patches/duckdb_v120.patch .
+# # pg_duckdb build requires source dir to be a git repo to get submodules
+# # allow {privileged_role_name} to execute some functions that in pg_duckdb are available to superuser only:
+# # - extension management function duckdb.install_extension()
+# # - access to duckdb.extensions table and its sequence
+# RUN git clone --depth 1 --branch v0.3.1 https://github.com/duckdb/pg_duckdb.git pg_duckdb-src && \
+#     cd pg_duckdb-src && \
+#     git submodule update --init --recursive && \
+#     patch -p1 < /ext-src/pg_duckdb_v031.patch && \
+#     cd third_party/duckdb && \
+#     patch -p1 < /ext-src/duckdb_v120.patch
 
-FROM pg-build AS pg_duckdb-build
-ARG PG_VERSION
-COPY --from=pg_duckdb-src /ext-src/ /ext-src/
-WORKDIR /ext-src/pg_duckdb-src
-RUN make install -j $(getconf _NPROCESSORS_ONLN) && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control
+# FROM pg-build AS pg_duckdb-build
+# ARG PG_VERSION
+# COPY --from=pg_duckdb-src /ext-src/ /ext-src/
+# WORKDIR /ext-src/pg_duckdb-src
+# RUN make install -j $(getconf _NPROCESSORS_ONLN) && \
+#     echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control
 
 #########################################################################################
 #
@@ -1708,7 +1721,7 @@ COPY --from=pg-anon-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_ivm-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_partman-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_mooncake-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=pg_duckdb-build /usr/local/pgsql/ /usr/local/pgsql/
+# COPY --from=pg_duckdb-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_repack-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pgaudit-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pgauditlogtofile-build /usr/local/pgsql/ /usr/local/pgsql/