Rebased submodule references to match main

Plv8 consists of two parts:
1. the V8 engine, which is built from vendored sources, and
2. the PostgreSQL extension.

Split those into two separate steps in the Dockerfile. The first step
doesn't need any PostgreSQL sources or any other files from the neon
repository, just the build tools and the upstream plv8 sources. Use the
build-deps image as the base for that step, so that the layer can be
cached and doesn't need to be rebuilt every time. This is worthwhile
because the V8 build takes a very long time.

.github/workflows/build_and_test.yml

@@ -722,14 +722,14 @@ jobs:
           - pg: v17
             debian: bookworm
     env:
-      VM_BUILDER_VERSION: v0.42.1
+      VM_BUILDER_VERSION: v0.37.1
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Downloading vm-builder
         run: |
-          curl -fL https://github.com/neondatabase/autoscaling/releases/download/$VM_BUILDER_VERSION/vm-builder-amd64 -o vm-builder
+          curl -fL https://github.com/neondatabase/autoscaling/releases/download/$VM_BUILDER_VERSION/vm-builder -o vm-builder
           chmod +x vm-builder
 
       - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193
@@ -750,8 +750,7 @@ jobs:
             -size=2G \
             -spec=compute/vm-image-spec-${{ matrix.version.debian }}.yaml \
             -src=neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
-            -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
-            -target-arch linux/amd64
+            -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}
 
       - name: Pushing vm-compute-node image
         run: |

compute/compute-node.Dockerfile

@@ -1493,7 +1493,7 @@ FROM pg-build AS pg_duckdb-build
 ARG PG_VERSION
 COPY --from=pg_duckdb-src /ext-src/ /ext-src/
 WORKDIR /ext-src/pg_duckdb-src
-RUN make install -j $(getconf _NPROCESSORS_ONLN) && \
+RUN make DUCKDB_STATIC=1 install -j $(getconf _NPROCESSORS_ONLN) && \
     echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control 
         
 #########################################################################################
@@ -1676,11 +1676,7 @@ COPY --from=pg_anon-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_ivm-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_partman-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_mooncake-build /usr/local/pgsql/ /usr/local/pgsql/
-
-# Disabled temporarily, because it clashed with pg_mooncake. pg_mooncake
-# also depends on libduckdb, but a different version.
-#COPY --from=pg_duckdb-build /usr/local/pgsql/ /usr/local/pgsql/
-
+COPY --from=pg_duckdb-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_repack-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pgaudit-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pgauditlogtofile-build /usr/local/pgsql/ /usr/local/pgsql/

compute/patches/pg_duckdb_v031.patch

@@ -1,3 +1,47 @@
+diff --git a/Makefile b/Makefile
+index 3235cc8..e232052 100644
+--- a/Makefile
++++ b/Makefile
+@@ -32,7 +32,16 @@ else
+ 	DUCKDB_BUILD_TYPE = release
+ endif
+ 
+-DUCKDB_LIB = libduckdb$(DLSUFFIX)
++DUCKDB_STATIC ?= 0
++PG_DUCKDB_LINK_FLAGS = -Wl,-rpath,$(PG_LIB)/ -lpq -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/src -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/extension/json -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/extension/parquet -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/extension/core_functions -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/extension/jemalloc -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/extension/icu -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/extension/cached_httpfs -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/yyjson -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/miniz -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/zstd -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/skiplist -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/mbedtls -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/utf8proc -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/hyperloglog -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/fastpforlib -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/re2 -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/libpg_query -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/fsst -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/third_party/fmt -L$(PG_LIB) -lstdc++ -llz4
++ifeq ($(DUCKDB_STATIC), 1)
++	DUCKDB_LIB = libduckdb_static.a
++	PG_DUCKDB_LINK_FLAGS += -l:$(DUCKDB_LIB) -l:libjson_extension.a -l:libparquet_extension.a -l:libcore_functions_extension.a -l:libjemalloc_extension.a -l:libicu_extension.a -l:libcached_httpfs_extension.a -l:libduckdb_yyjson.a -l:libduckdb_miniz.a -l:libduckdb_zstd.a -l:libduckdb_skiplistlib.a -l:libduckdb_mbedtls.a -l:libduckdb_utf8proc.a -l:libduckdb_hyperloglog.a -l:libduckdb_fastpforlib.a -l:libduckdb_re2.a -l:libduckdb_pg_query.a -l:libduckdb_fsst.a -l:libduckdb_fmt.a
++else
++	DUCKDB_LIB = libduckdb$(DLSUFFIX)
++	PG_DUCKDB_LINK_FLAGS += -lduckdb
++endif
++
+ FULL_DUCKDB_LIB = third_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/src/$(DUCKDB_LIB)
+ 
+ ERROR_ON_WARNING ?=
+@@ -54,7 +63,7 @@ override PG_CXXFLAGS += -std=c++17 ${DUCKDB_BUILD_CXX_FLAGS} ${COMPILER_FLAGS} -
+ # changes to the vendored code in one place.
+ override PG_CFLAGS += -Wno-declaration-after-statement
+ 
+-SHLIB_LINK += -Wl,-rpath,$(PG_LIB)/ -lpq -Lthird_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/src -L$(PG_LIB) -lduckdb -lstdc++ -llz4
++SHLIB_LINK += $(PG_DUCKDB_LINK_FLAGS)
+ 
+ include Makefile.global
+ 
+diff --git a/docs/compilation.md b/docs/compilation.md
+index e25396f..b94a63a 100644
+--- a/docs/compilation.md
++++ b/docs/compilation.md
+@@ -14,6 +14,8 @@ To build and install, run:
+ make install
+ ```
+ 
++If you want to link `libduckdb` statically, set `DUCKDB_STATIC=1` when interacting with `make(1)`.
++
+ Add `pg_duckdb` to the `shared_preload_libraries` in your `postgresql.conf` file:
+ 
+ ```ini
 diff --git a/sql/pg_duckdb--0.2.0--0.3.0.sql b/sql/pg_duckdb--0.2.0--0.3.0.sql
 index d777d76..af60106 100644
 --- a/sql/pg_duckdb--0.2.0--0.3.0.sql

storage_controller/src/http.rs

@@ -598,10 +598,7 @@ async fn handle_tenant_timeline_passthrough(
 
     let _timer = latency.start_timer(labels.clone());
 
-    let client = mgmt_api::Client::new(
-        node.base_url(),
-        service.get_config().pageserver_jwt_token.as_deref(),
-    );
+    let client = mgmt_api::Client::new(node.base_url(), service.get_config().jwt_token.as_deref());
     let resp = client.get_raw(path).await.map_err(|e|
         // We return 503 here because if we can't successfully send a request to the pageserver,
         // either we aren't available or the pageserver is unavailable.
@@ -1357,7 +1354,10 @@ async fn handle_safekeeper_scheduling_policy(
         .set_safekeeper_scheduling_policy(id, body.scheduling_policy)
         .await?;
 
-    json_response(StatusCode::OK, ())
+    Ok(Response::builder()
+        .status(StatusCode::NO_CONTENT)
+        .body(Body::empty())
+        .unwrap())
 }
 
 /// Common wrapper for request handlers that call into Service and will operate on tenants: they must only

storage_controller/src/main.rs

@@ -53,10 +53,6 @@ struct Cli {
     #[arg(long)]
     jwt_token: Option<String>,
 
-    /// Token for authenticating this service with the safekeepers it controls
-    #[arg(long)]
-    safekeeper_jwt_token: Option<String>,
-
     /// Token for authenticating this service with the control plane, when calling
     /// the compute notification endpoint
     #[arg(long)]
@@ -157,8 +153,7 @@ impl Default for StrictMode {
 struct Secrets {
     database_url: String,
     public_key: Option<JwtAuth>,
-    pageserver_jwt_token: Option<String>,
-    safekeeper_jwt_token: Option<String>,
+    jwt_token: Option<String>,
     control_plane_jwt_token: Option<String>,
     peer_jwt_token: Option<String>,
 }
@@ -166,7 +161,6 @@ struct Secrets {
 impl Secrets {
     const DATABASE_URL_ENV: &'static str = "DATABASE_URL";
     const PAGESERVER_JWT_TOKEN_ENV: &'static str = "PAGESERVER_JWT_TOKEN";
-    const SAFEKEEPER_JWT_TOKEN_ENV: &'static str = "SAFEKEEPER_JWT_TOKEN";
     const CONTROL_PLANE_JWT_TOKEN_ENV: &'static str = "CONTROL_PLANE_JWT_TOKEN";
     const PEER_JWT_TOKEN_ENV: &'static str = "PEER_JWT_TOKEN";
     const PUBLIC_KEY_ENV: &'static str = "PUBLIC_KEY";
@@ -190,14 +184,7 @@ impl Secrets {
         let this = Self {
             database_url,
             public_key,
-            pageserver_jwt_token: Self::load_secret(
-                &args.jwt_token,
-                Self::PAGESERVER_JWT_TOKEN_ENV,
-            ),
-            safekeeper_jwt_token: Self::load_secret(
-                &args.safekeeper_jwt_token,
-                Self::SAFEKEEPER_JWT_TOKEN_ENV,
-            ),
+            jwt_token: Self::load_secret(&args.jwt_token, Self::PAGESERVER_JWT_TOKEN_ENV),
             control_plane_jwt_token: Self::load_secret(
                 &args.control_plane_jwt_token,
                 Self::CONTROL_PLANE_JWT_TOKEN_ENV,
@@ -277,17 +264,11 @@ async fn async_main() -> anyhow::Result<()> {
 
     let secrets = Secrets::load(&args).await?;
 
-    // TODO: once we've rolled out the safekeeper JWT token everywhere, put it into the validation code below
-    tracing::info!(
-        "safekeeper_jwt_token set: {:?}",
-        secrets.safekeeper_jwt_token.is_some()
-    );
-
     // Validate required secrets and arguments are provided in strict mode
     match strict_mode {
         StrictMode::Strict
             if (secrets.public_key.is_none()
-                || secrets.pageserver_jwt_token.is_none()
+                || secrets.jwt_token.is_none()
                 || secrets.control_plane_jwt_token.is_none()) =>
         {
             // Production systems should always have secrets configured: if public_key was not set
@@ -312,8 +293,7 @@ async fn async_main() -> anyhow::Result<()> {
     }
 
     let config = Config {
-        pageserver_jwt_token: secrets.pageserver_jwt_token,
-        safekeeper_jwt_token: secrets.safekeeper_jwt_token,
+        jwt_token: secrets.jwt_token,
         control_plane_jwt_token: secrets.control_plane_jwt_token,
         peer_jwt_token: secrets.peer_jwt_token,
         compute_hook_url: args.compute_hook_url,

storage_controller/src/reconciler.rs

@@ -296,7 +296,7 @@ impl Reconciler {
                         .location_config(tenant_shard_id, config.clone(), flush_ms, lazy)
                         .await
                 },
-                &self.service_config.pageserver_jwt_token,
+                &self.service_config.jwt_token,
                 1,
                 3,
                 timeout,
@@ -417,7 +417,7 @@ impl Reconciler {
         let client = PageserverClient::new(
             node.get_id(),
             node.base_url(),
-            self.service_config.pageserver_jwt_token.as_deref(),
+            self.service_config.jwt_token.as_deref(),
         );
 
         client
@@ -440,7 +440,7 @@ impl Reconciler {
         let client = PageserverClient::new(
             node.get_id(),
             node.base_url(),
-            self.service_config.pageserver_jwt_token.as_deref(),
+            self.service_config.jwt_token.as_deref(),
         );
 
         let timelines = client.timeline_list(&tenant_shard_id).await?;
@@ -478,7 +478,7 @@ impl Reconciler {
                             )
                             .await
                     },
-                    &self.service_config.pageserver_jwt_token,
+                    &self.service_config.jwt_token,
                     1,
                     3,
                     request_download_timeout * 2,
@@ -771,7 +771,7 @@ impl Reconciler {
             let observed_conf = match attached_node
                 .with_client_retries(
                     |client| async move { client.get_location_config(tenant_shard_id).await },
-                    &self.service_config.pageserver_jwt_token,
+                    &self.service_config.jwt_token,
                     1,
                     1,
                     Duration::from_secs(5),
@@ -1099,7 +1099,7 @@ impl Reconciler {
             match origin
                 .with_client_retries(
                     |client| async move { client.get_location_config(tenant_shard_id).await },
-                    &self.service_config.pageserver_jwt_token,
+                    &self.service_config.jwt_token,
                     1,
                     3,
                     Duration::from_secs(5),

storage_controller/src/service.rs

@@ -348,12 +348,7 @@ pub struct Config {
     // All pageservers managed by one instance of this service must have
     // the same public key.  This JWT token will be used to authenticate
     // this service to the pageservers it manages.
-    pub pageserver_jwt_token: Option<String>,
-
-    // All safekeepers managed by one instance of this service must have
-    // the same public key. This JWT token will be used to authenticate
-    // this service to the safekeepers it manages.
-    pub safekeeper_jwt_token: Option<String>,
+    pub jwt_token: Option<String>,
 
     // This JWT token will be used to authenticate this service to the control plane.
     pub control_plane_jwt_token: Option<String>,
@@ -887,7 +882,7 @@ impl Service {
                     let response = node
                         .with_client_retries(
                             |client| async move { client.list_location_config().await },
-                            &self.config.pageserver_jwt_token,
+                            &self.config.jwt_token,
                             1,
                             5,
                             timeout,
@@ -988,7 +983,7 @@ impl Service {
             let client = PageserverClient::new(
                 node.get_id(),
                 node.base_url(),
-                self.config.pageserver_jwt_token.as_deref(),
+                self.config.jwt_token.as_deref(),
             );
             match client
                 .location_config(
@@ -1558,14 +1553,14 @@ impl Service {
         let reconcilers_cancel = cancel.child_token();
 
         let heartbeater_ps = Heartbeater::new(
-            config.pageserver_jwt_token.clone(),
+            config.jwt_token.clone(),
             config.max_offline_interval,
             config.max_warming_up_interval,
             cancel.clone(),
         );
 
         let heartbeater_sk = Heartbeater::new(
-            config.safekeeper_jwt_token.clone(),
+            config.jwt_token.clone(),
             config.max_offline_interval,
             config.max_warming_up_interval,
             cancel.clone(),
@@ -1912,7 +1907,7 @@ impl Service {
         let configs = match node
             .with_client_retries(
                 |client| async move { client.list_location_config().await },
-                &self.config.pageserver_jwt_token,
+                &self.config.jwt_token,
                 1,
                 5,
                 SHORT_RECONCILE_TIMEOUT,
@@ -1970,7 +1965,7 @@ impl Service {
                             .location_config(tenant_shard_id, config, None, false)
                             .await
                     },
-                    &self.config.pageserver_jwt_token,
+                    &self.config.jwt_token,
                     1,
                     5,
                     SHORT_RECONCILE_TIMEOUT,
@@ -3105,7 +3100,7 @@ impl Service {
                 let client = PageserverClient::new(
                     node.get_id(),
                     node.base_url(),
-                    self.config.pageserver_jwt_token.as_deref(),
+                    self.config.jwt_token.as_deref(),
                 );
 
                 tracing::info!("Doing time travel recovery for shard {tenant_shard_id}",);
@@ -3166,7 +3161,7 @@ impl Service {
             let client = PageserverClient::new(
                 node.get_id(),
                 node.base_url(),
-                self.config.pageserver_jwt_token.as_deref(),
+                self.config.jwt_token.as_deref(),
             );
             futs.push(async move {
                 let result = client
@@ -3289,7 +3284,7 @@ impl Service {
                         .tenant_delete(TenantShardId::unsharded(tenant_id))
                         .await
                 },
-                &self.config.pageserver_jwt_token,
+                &self.config.jwt_token,
                 1,
                 3,
                 RECONCILE_TIMEOUT,
@@ -3508,7 +3503,7 @@ impl Service {
             let timeline_info = create_one(
                 shard_zero_tid,
                 shard_zero_locations,
-                self.config.pageserver_jwt_token.clone(),
+                self.config.jwt_token.clone(),
                 create_req.clone(),
             )
             .await?;
@@ -3524,7 +3519,7 @@ impl Service {
             // Create timeline on remaining shards with number >0
             if !targets.0.is_empty() {
                 // If we had multiple shards, issue requests for the remainder now.
-                let jwt = &self.config.pageserver_jwt_token;
+                let jwt = &self.config.jwt_token;
                 self.tenant_for_shards(
                     targets
                         .0
@@ -3607,7 +3602,7 @@ impl Service {
                         tenant_shard_id,
                         timeline_id,
                         node,
-                        self.config.pageserver_jwt_token.clone(),
+                        self.config.jwt_token.clone(),
                         req.clone(),
                     ))
                 })
@@ -3688,7 +3683,7 @@ impl Service {
                         tenant_shard_id,
                         timeline_id,
                         node,
-                        self.config.pageserver_jwt_token.clone(),
+                        self.config.jwt_token.clone(),
                     ))
                 })
                 .await?;
@@ -3762,7 +3757,7 @@ impl Service {
                     tenant_shard_id,
                     timeline_id,
                     node,
-                    self.config.pageserver_jwt_token.clone(),
+                    self.config.jwt_token.clone(),
                     dir,
                 ))
             })
@@ -3877,7 +3872,7 @@ impl Service {
             futs.push(async move {
                 node.with_client_retries(
                     |client| op(tenant_shard_id, client),
-                    &self.config.pageserver_jwt_token,
+                    &self.config.jwt_token,
                     warn_threshold,
                     max_retries,
                     timeout,
@@ -4126,7 +4121,7 @@ impl Service {
                         tenant_shard_id,
                         timeline_id,
                         node,
-                        self.config.pageserver_jwt_token.clone(),
+                        self.config.jwt_token.clone(),
                     ))
                 })
                 .await?;
@@ -4148,7 +4143,7 @@ impl Service {
                 shard_zero_tid,
                 timeline_id,
                 shard_zero_locations.latest.node,
-                self.config.pageserver_jwt_token.clone(),
+                self.config.jwt_token.clone(),
             )
             .await?;
             Ok(shard_zero_status)
@@ -4547,7 +4542,7 @@ impl Service {
 
                         client.location_config(child_id, config, None, false).await
                     },
-                    &self.config.pageserver_jwt_token,
+                    &self.config.jwt_token,
                     1,
                     10,
                     Duration::from_secs(5),
@@ -5147,7 +5142,7 @@ impl Service {
             let client = PageserverClient::new(
                 node.get_id(),
                 node.base_url(),
-                self.config.pageserver_jwt_token.as_deref(),
+                self.config.jwt_token.as_deref(),
             );
             let response = client
                 .tenant_shard_split(
@@ -5473,7 +5468,7 @@ impl Service {
         let client = PageserverClient::new(
             node.get_id(),
             node.base_url(),
-            self.config.pageserver_jwt_token.as_deref(),
+            self.config.jwt_token.as_deref(),
         );
 
         let scan_result = client
@@ -7099,7 +7094,7 @@ impl Service {
         match attached_node
             .with_client_retries(
                 |client| async move { client.tenant_heatmap_upload(tenant_shard_id).await },
-                &self.config.pageserver_jwt_token,
+                &self.config.jwt_token,
                 3,
                 10,
                 SHORT_RECONCILE_TIMEOUT,
@@ -7135,7 +7130,7 @@ impl Service {
                             )
                             .await
                     },
-                    &self.config.pageserver_jwt_token,
+                    &self.config.jwt_token,
                     3,
                     10,
                     SHORT_RECONCILE_TIMEOUT,
@@ -7190,7 +7185,7 @@ impl Service {
                         let request = request_ref.clone();
                         client.top_tenant_shards(request.clone()).await
                     },
-                    &self.config.pageserver_jwt_token,
+                    &self.config.jwt_token,
                     3,
                     3,
                     Duration::from_secs(5),
@@ -7363,7 +7358,7 @@ impl Service {
         match node
             .with_client_retries(
                 |client| async move { client.tenant_secondary_status(tenant_shard_id).await },
-                &self.config.pageserver_jwt_token,
+                &self.config.jwt_token,
                 1,
                 3,
                 Duration::from_millis(250),