rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-10 23:12:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixup: make it build

Browse Source
This commit is contained in:
Vlad Lazar
2025-07-22 15:49:55 +01:00
parent 2ddf8f64ce
commit b762de56ff
20 changed files with 315 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
Cargo.lock generated
View File

@@ -167,6 +167,45 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "asn1-rs"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048"
dependencies = [
"asn1-rs-derive",
"asn1-rs-impl",
"displaydoc",
"nom",
"num-traits",
"rusticata-macros",
"thiserror 1.0.69",
"time",
]
[[package]]
name = "asn1-rs-derive"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
"synstructure",
]
[[package]]
name = "asn1-rs-impl"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "assert-json-diff"
version = "2.0.2"
@@ -301,6 +340,30 @@ dependencies = [
"zeroize",
]
[[package]]
name = "aws-lc-rs"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08b5d4e069cbc868041a64bd68dc8cb39a0d79585cd6c5a24caa8c2d622121be"
dependencies = [
"aws-lc-sys",
"untrusted 0.7.1",
"zeroize",
]
[[package]]
name = "aws-lc-sys"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff"
dependencies = [
"bindgen 0.69.5",
"cc",
"cmake",
"dunce",
"fs_extra",
]
[[package]]
name = "aws-runtime"
version = "1.4.4"
@@ -962,6 +1025,29 @@ dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.69.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
dependencies = [
"bitflags 2.8.0",
"cexpr",
"clang-sys",
"itertools 0.12.1",
"lazy_static",
"lazycell",
"log",
"prettyplease",
"proc-macro2",
"quote",
"regex",
"rustc-hash 1.1.0",
"shlex",
"syn 2.0.100",
"which",
]
[[package]]
name = "bindgen"
version = "0.71.1"
@@ -1254,6 +1340,15 @@ dependencies = [
"replace_with",
]
[[package]]
name = "cmake"
version = "0.1.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
dependencies = [
"cc",
]
[[package]]
name = "colorchoice"
version = "1.0.0"
@@ -1819,6 +1914,20 @@ dependencies = [
"zeroize",
]
[[package]]
name = "der-parser"
version = "9.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"
dependencies = [
"asn1-rs",
"displaydoc",
"nom",
"num-bigint",
"num-traits",
"rusticata-macros",
]
[[package]]
name = "der_derive"
version = "0.7.3"
@@ -1975,6 +2084,12 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dyn-clone"
version = "1.0.14"
@@ -2374,6 +2489,12 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsevent-sys"
version = "4.1.0"
@@ -2803,6 +2924,15 @@ dependencies = [
"digest",
]
[[package]]
name = "home"
version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "hostname"
version = "0.4.0"
@@ -3577,6 +3707,12 @@ dependencies = [
"spin",
]
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.172"
@@ -4120,6 +4256,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "oid-registry"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9"
dependencies = [
"asn1-rs",
]
[[package]]
name = "once_cell"
version = "1.20.2"
@@ -4984,7 +5129,7 @@ name = "postgres_ffi"
version = "0.1.0"
dependencies = [
"anyhow",
"bindgen",
"bindgen 0.71.1",
"bytes",
"crc32c",
"criterion",
@@ -5644,6 +5789,7 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779"
dependencies = [
"aws-lc-rs",
"pem",
"ring",
"rustls-pki-types",
@@ -5959,7 +6105,7 @@ dependencies = [
"cfg-if",
"getrandom 0.2.11",
"libc",
"untrusted",
"untrusted 0.9.0",
"windows-sys 0.52.0",
]
@@ -6080,6 +6226,15 @@ dependencies = [
"semver",
]
[[package]]
name = "rusticata-macros"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
dependencies = [
"nom",
]
[[package]]
name = "rustix"
version = "0.38.41"
@@ -6204,7 +6359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
dependencies = [
"ring",
"untrusted",
"untrusted 0.9.0",
]
[[package]]
@@ -6215,7 +6370,7 @@ checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
"untrusted 0.9.0",
]
[[package]]
@@ -6226,7 +6381,7 @@ checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
"untrusted 0.9.0",
]
[[package]]
@@ -6382,7 +6537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
dependencies = [
"ring",
"untrusted",
"untrusted 0.9.0",
]
[[package]]
@@ -8146,6 +8301,12 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -8245,6 +8406,7 @@ dependencies = [
"jsonwebtoken",
"metrics",
"nix 0.30.1",
"oid-registry",
"once_cell",
"pem",
"pin-project-lite",
@@ -8252,7 +8414,10 @@ dependencies = [
"pprof",
"pq_proto",
"rand 0.8.5",
"rcgen",
"regex",
"rustls-pemfile 2.1.1",
"rustls-pki-types",
"scopeguard",
"sentry",
"serde",
@@ -8273,6 +8438,7 @@ dependencies = [
"tracing-utils",
"uuid",
"walkdir",
"x509-parser",
]
[[package]]
@@ -8387,7 +8553,7 @@ name = "walproposer"
version = "0.1.0"
dependencies = [
"anyhow",
"bindgen",
"bindgen 0.71.1",
"postgres_ffi",
"utils",
]
@@ -8552,6 +8718,18 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "which"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
dependencies = [
"either",
"home",
"once_cell",
"rustix",
]
[[package]]
name = "whoami"
version = "1.5.1"
@@ -8940,6 +9118,23 @@ dependencies = [
"zeroize",
]
[[package]]
name = "x509-parser"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69"
dependencies = [
"asn1-rs",
"data-encoding",
"der-parser",
"lazy_static",
"nom",
"oid-registry",
"rusticata-macros",
"thiserror 1.0.69",
"time",
]
[[package]]
name = "xattr"
version = "1.0.0"

3
Cargo.toml
View File

@@ -141,7 +141,7 @@ nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket"
notify = "6.0.0"
num_cpus = "1.15"
num-traits = "0.2.19"
oid-registry = "0.6.1"
oid-registry = "0.7.1"
once_cell = "1.13"
opentelemetry = "0.27"
opentelemetry_sdk = "0.27"
@@ -237,6 +237,7 @@ whoami = "1.5.1"
zerocopy = { version = "0.8", features = ["derive", "simd"] }
json-structural-diff = { version = "0.2.0" }
x509-cert = { version = "0.2.5" }
x509-parser = "0.16"
## TODO replace this with tracing
env_logger = "0.11"

15
control_plane/src/local_env.rs
View File

@@ -18,7 +18,7 @@ use postgres_backend::AuthType;
use reqwest::{Certificate, Url};
use safekeeper_api::PgMajorVersion;
use serde::{Deserialize, Serialize};
use utils::auth::{encode_from_key_file, Claims, encode_hadron_token};
use utils::auth::{encode_from_key_file, encode_hadron_token};
use utils::id::{NodeId, TenantId, TenantTimelineId, TimelineId};
use crate::broker::StorageBroker;
@@ -833,11 +833,16 @@ impl LocalEnv {
// this function is used only for testing purposes in CLI e g generate tokens during init
pub fn generate_auth_token<S: Serialize>(&self, claims: &S) -> anyhow::Result<String> {
let private_key_path = self.get_private_key_path();
let key_data = fs::read(private_key_path)?;
match self.token_auth_type {
AuthType::NeonJWT => encode_from_key_file(claims, &key_data),
AuthType::HadronJWT => encode_hadron_token(claims, &key_data),
AuthType::NeonJWT => {
let key_data = self.read_private_key()?;
encode_from_key_file(claims, &key_data)
}
AuthType::HadronJWT => {
let private_key_path = self.get_private_key_path();
let key_data = fs::read(private_key_path)?;
encode_hadron_token(claims, &key_data)
}
_ => panic!("unsupported token auth type {:?}", self.token_auth_type),
}
}

31
control_plane/src/safekeeper.rs
View File

@@ -20,8 +20,6 @@ use safekeeper_client::mgmt_api;
use thiserror::Error;
use utils::auth::{Claims, Scope};
use utils::id::NodeId;
use utils::ip_address::HADRON_NODE_IP_ADDRESS;
use utils::{http::error::HttpErrorBody, id::NodeId};
use crate::background_process;
use crate::local_env::{LocalEnv, SafekeeperConf};
@@ -113,7 +111,7 @@ impl SafekeeperNode {
}
// Generate a token file for authentication with other safekeepers
if self.conf.auth_enabled {
if self.conf.auth_type != AuthType::Trust {
let token = self
.env
.generate_auth_token(&Claims::new(None, Scope::SafekeeperData))?;
@@ -224,26 +222,14 @@ impl SafekeeperNode {
args.push(format!("--ssl-ca-file={}", ssl_ca_file.to_str().unwrap()));
}
if self.conf.auth_enabled {
let token_path = self.datadir_path().join("peer_jwt_token");
let token_path_str = token_path
.to_str()
.with_context(|| {
format!("Token path {token_path:?} cannot be represented as a unicode string")
})?
.to_owned();
args.extend(["--auth-token-path".to_owned(), token_path_str]);
}
args.extend_from_slice(extra_opts);
let env_variables = Vec::new();
background_process::start_process(
&format!("safekeeper-{id}"),
&datadir,
&self.env.safekeeper_bin(),
&args,
env_variables,
self.safekeeper_env_variables()?,
background_process::InitialPidFile::Expect(self.pid_file()),
retry_timeout,
|| async {
@@ -257,6 +243,19 @@ impl SafekeeperNode {
.await
}
fn safekeeper_env_variables(&self) -> anyhow::Result<Vec<(String, String)>> {
let mut env_vars = vec![];
// Generate a token to connect from safekeeper to peers
if self.conf.auth_type != AuthType::Trust {
let token = self
.env
.generate_auth_token(&Claims::new(None, Scope::SafekeeperData))?;
// TODO: Safekeepers don't read this env var
env_vars.push(("SAFEKEEPER_AUTH_TOKEN".to_owned(), token.clone()));
}
Ok(env_vars)
}
///
/// Stop the server.
///

28
control_plane/src/storage_controller.rs
View File

@@ -30,7 +30,7 @@ use serde::{Deserialize, Serialize};
use tokio::process::Command;
use tracing::instrument;
use url::Url;
use utils::auth::{Claims, Scope, encode_from_key_file};
use utils::auth::{Claims, Scope, encode_from_key_file, encode_hadron_token};
use utils::id::{NodeId, TenantId};
use whoami::username;
@@ -625,15 +625,17 @@ impl StorageController {
let jwt_token = private_key.encode_token(&claims)?;
args.push(format!("--jwt-token={jwt_token}"));
let peer_claims = Claims::new(None, Scope::Admin);
let peer_jwt_token = encode_from_key_file(&peer_claims, private_key)
.expect("failed to generate jwt token");
args.push(format!("--peer-jwt-token={peer_jwt_token}"));
if let StorageControllerPrivateKey::EdPrivateKey(key) = private_key {
let peer_claims = Claims::new(None, Scope::Admin);
let peer_jwt_token =
encode_from_key_file(&peer_claims, key).expect("failed to generate jwt token");
args.push(format!("--peer-jwt-token={peer_jwt_token}"));
let claims = Claims::new(None, Scope::SafekeeperData);
let jwt_token =
encode_from_key_file(&claims, private_key).expect("failed to generate jwt token");
args.push(format!("--safekeeper-jwt-token={jwt_token}"));
let claims = Claims::new(None, Scope::SafekeeperData);
let jwt_token =
encode_from_key_file(&claims, key).expect("failed to generate jwt token");
args.push(format!("--safekeeper-jwt-token={jwt_token}"));
}
}
if let Some(public_key) = &self.public_key {
@@ -685,7 +687,13 @@ impl StorageController {
self.env.base_data_dir.display()
));
if self.env.safekeepers.iter().any(|sk| sk.auth_enabled) && self.private_key.is_none() {
if self
.env
.safekeepers
.iter()
.any(|sk| sk.auth_type != AuthType::Trust)
&& self.private_key.is_none()
{
anyhow::bail!("Safekeeper set up for auth but no private key specified");
}

18
libs/utils/src/auth.rs
View File

@@ -1,9 +1,9 @@
// For details about authentication see docs/authentication.md
use std::borrow::Cow;
use std::fmt::Display;
use std::fs;
use std::sync::Arc;
use std::{borrow::Cow, io, path::Path};
use anyhow::Result;
use arc_swap::ArcSwap;
@@ -11,11 +11,11 @@ use camino::Utf8Path;
use jsonwebtoken::{
Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, decode, encode,
};
use oid_registry::OID_PKCS1_RSAENCRYPTION;
use pem::Pem;
use rustls_pki_types::CertificateDer;
use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned};
use uuid::Uuid;
use oid_registry::OID_PKCS1_RSAENCRYPTION;
use rustls_pki_types::CertificateDer;
use crate::id::TenantId;
@@ -259,7 +259,9 @@ impl JwtAuth {
anyhow::bail!("{cert_path} is neither a directory or a file")
}
if decoding_keys.is_empty() {
anyhow::bail!("Configured for JWT auth with zero decoding keys. All JWT gated requests would be rejected.");
anyhow::bail!(
"Configured for JWT auth with zero decoding keys. All JWT gated requests would be rejected."
);
}
// Note that we need to create a `JwtAuth` with a different `validation` from the default one created by `new()` in this case
@@ -319,13 +321,13 @@ pub fn encode_from_key_file<S: Serialize>(claims: &S, pem: &Pem) -> Result<Strin
/// Encode (i.e., sign) a Hadron auth token with the given claims and RSA private key. This is used
/// by HCC to sign tokens when deploying compute or returning the compute spec. The resulting token
/// is used by the compute node to authenticate with HCC and PS/SK.
pub fn encode_hadron_token(claims: &Claims, key_data: &[u8]) -> Result<String> {
pub fn encode_hadron_token<S: Serialize>(claims: &S, key_data: &[u8]) -> Result<String> {
let key = EncodingKey::from_rsa_pem(key_data)?;
encode_hadron_token_with_encoding_key(claims, &key)
}
pub fn encode_hadron_token_with_encoding_key(
claims: &Claims,
pub fn encode_hadron_token_with_encoding_key<S: Serialize>(
claims: &S,
encoding_key: &EncodingKey,
) -> Result<String> {
Ok(encode(
@@ -364,7 +366,6 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH
tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081").unwrap()),
endpoint_id: None,
scope: Scope::Tenant,
endpoint_id: None,
};
// A test token containing the following payload, signed using TEST_PRIV_KEY_ED25519:
@@ -394,7 +395,6 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH
tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081").unwrap()),
endpoint_id: None,
scope: Scope::Tenant,
endpoint_id: None,
};
let pem = pem::parse(TEST_PRIV_KEY_ED25519).unwrap();

7
pageserver/src/bin/pageserver.rs
View File

@@ -459,7 +459,10 @@ fn start_pageserver(
let http_auth;
let pg_auth;
let grpc_auth;
if [conf.http_auth_type, conf.pg_auth_type, conf.grpc_auth_type].iter().any(|auth_type| auth_type == AuthType::NeonJWT || auth_type == HadronJWT) {
if [conf.http_auth_type, conf.pg_auth_type, conf.grpc_auth_type]
.iter()
.any(|auth_type| *auth_type == AuthType::NeonJWT || *auth_type == AuthType::HadronJWT)
{
// unwrap is ok because check is performed when creating config, so path is set and exists
let key_path = conf.auth_validation_public_key_path.as_ref().unwrap();
info!("Loading public key(s) for verifying JWT tokens from {key_path:?}");
@@ -481,7 +484,7 @@ fn start_pageserver(
};
pg_auth = match conf.pg_auth_type {
AuthType::Trust => None,
AuthType::NeonJWT => Some(auth.clone()),
AuthType::NeonJWT | AuthType::HadronJWT => Some(auth.clone()),
};
grpc_auth = match conf.grpc_auth_type {
AuthType::Trust => None,

3
pageserver/src/http/routes.rs
View File

@@ -44,18 +44,19 @@ use pageserver_api::models::{
TopTenantShardItem, TopTenantShardsRequest, TopTenantShardsResponse,
};
use pageserver_api::shard::{ShardCount, TenantShardId};
use postgres_backend::AuthType;
use postgres_ffi::PgMajorVersion;
use remote_storage::{DownloadError, GenericRemoteStorage, TimeTravelError};
use scopeguard::defer;
use serde::{Deserialize, Serialize};
use serde_json::json;
use tenant_size_model::svg::SvgBranchKind;
use postgres_backend::AuthType;
use tenant_size_model::{SizeResult, StorageModel};
use tokio::time::Instant;
use tokio_util::io::StreamReader;
use tokio_util::sync::CancellationToken;
use tracing::*;
use utils::auth::JwtAuth;
use utils::auth::SwappableJwtAuth;
use utils::generation::Generation;
use utils::id::{TenantId, TimelineId};

61
safekeeper/src/bin/safekeeper.rs
View File

@@ -14,8 +14,8 @@ use futures::future::BoxFuture;
use futures::stream::FuturesUnordered;
use futures::{FutureExt, StreamExt};
use http_utils::tls_certs::ReloadingCertificateResolver;
use postgres_backend::AuthType;
use metrics::set_build_info_metric;
use postgres_backend::AuthType;
use remote_storage::RemoteStorageConfig;
use safekeeper::defaults::{
DEFAULT_CONTROL_FILE_SAVE_INTERVAL, DEFAULT_EVICTION_MIN_RESIDENT,
@@ -374,17 +374,17 @@ async fn main() -> anyhow::Result<()> {
Some(path) => {
info!("loading pg auth JWT key from {path}");
match args.token_auth_type {
AuthType::NeonJWT => {
Some(Arc::new(
JwtAuth::from_key_path(path).context("failed to load the auth key")?,
))
}
AuthType::HadronJWT => {
Some(Arc::new(
JwtAuth::from_cert_path(path).context("failed to load auth keys from certificates")?,
))
}
_ => panic!("AuthType {auth_type} is not allowed when --pg-auth-public-key-path is specified", auth_type = args.token_auth_type),
AuthType::NeonJWT => Some(Arc::new(
JwtAuth::from_key_path(path).context("failed to load the auth key")?,
)),
AuthType::HadronJWT => Some(Arc::new(
JwtAuth::from_cert_path(path)
.context("failed to load auth keys from certificates")?,
)),
_ => panic!(
"AuthType {auth_type} is not allowed when --pg-auth-public-key-path is specified",
auth_type = args.token_auth_type
),
}
}
};
@@ -396,17 +396,17 @@ async fn main() -> anyhow::Result<()> {
Some(path) => {
info!("loading pg tenant only auth JWT key from {path}");
match args.token_auth_type {
AuthType::NeonJWT => {
Some(Arc::new(
JwtAuth::from_key_path(path).context("failed to load the auth key")?,
))
}
AuthType::HadronJWT => {
Some(Arc::new(
JwtAuth::from_cert_path(path).context("failed to load auth keys from certificates")?,
))
}
_ => panic!("AuthType {auth_type} is not allowed when --pg-tenant-only-auth-public-key-path is specified", auth_type = args.token_auth_type),
AuthType::NeonJWT => Some(Arc::new(
JwtAuth::from_key_path(path).context("failed to load the auth key")?,
)),
AuthType::HadronJWT => Some(Arc::new(
JwtAuth::from_cert_path(path)
.context("failed to load auth keys from certificates")?,
)),
_ => panic!(
"AuthType {auth_type} is not allowed when --pg-tenant-only-auth-public-key-path is specified",
auth_type = args.token_auth_type
),
}
}
};
@@ -418,9 +418,15 @@ async fn main() -> anyhow::Result<()> {
Some(path) => {
info!("loading http auth JWT key(s) from {path}");
let jwt_auth = match args.token_auth_type {
AuthType::NeonJWT => JwtAuth::from_key_path(path).context("failed to load the auth key")?,
AuthType::HadronJWT => JwtAuth::from_cert_path(path).context("failed to load auth keys from certificates")?,
_ => panic!("AuthType {auth_type} is not allowed when --http-auth-public-key-path is specified", auth_type = args.token_auth_type),
AuthType::NeonJWT => {
JwtAuth::from_key_path(path).context("failed to load the auth key")?
}
AuthType::HadronJWT => JwtAuth::from_cert_path(path)
.context("failed to load auth keys from certificates")?,
_ => panic!(
"AuthType {auth_type} is not allowed when --http-auth-public-key-path is specified",
auth_type = args.token_auth_type
),
};
Some(Arc::new(SwappableJwtAuth::new(jwt_auth)))
}
@@ -456,7 +462,6 @@ async fn main() -> anyhow::Result<()> {
listen_http_addr: args.listen_http,
listen_https_addr: args.listen_https,
advertise_pg_addr: args.advertise_pg,
advertise_pg_addr_tenant_only: args.advertise_pg_tenant_only,
availability_zone: args.availability_zone,
no_sync: args.no_sync,
broker_endpoint: args.broker_endpoint,
@@ -495,7 +500,7 @@ async fn main() -> anyhow::Result<()> {
enable_tls_wal_service_api: args.enable_tls_wal_service_api,
force_metric_collection_on_scrape: args.force_metric_collection_on_scrape,
/* BEGIN_HADRON */
advertise_pg_addr_tenant_only: None,
advertise_pg_addr_tenant_only: args.advertise_pg_tenant_only,
enable_pull_timeline_on_startup: args.enable_pull_timeline_on_startup,
hcc_base_url: None,
global_disk_check_interval: args.global_disk_check_interval,

2
safekeeper/src/lib.rs
View File

@@ -106,7 +106,6 @@ pub struct SafeKeeperConf {
pub listen_http_addr: String,
pub listen_https_addr: Option<String>,
pub advertise_pg_addr: Option<String>,
pub advertise_pg_addr_tenant_only: Option<String>,
pub availability_zone: Option<String>,
pub no_sync: bool,
/* BEGIN_HADRON */
@@ -166,7 +165,6 @@ impl SafeKeeperConf {
listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
listen_https_addr: None,
advertise_pg_addr: None,
advertise_pg_addr_tenant_only: None,
availability_zone: None,
remote_storage: None,
my_id: NodeId(0),

2
safekeeper/tests/walproposer_sim/safekeeper.rs
View File

@@ -9,12 +9,12 @@ use std::time::Duration;
use anyhow::{Result, bail};
use bytes::{Bytes, BytesMut};
use camino::Utf8PathBuf;
use postgres_backend::AuthType;
use desim::executor::{self, PollSome};
use desim::network::TCP;
use desim::node_os::NodeOs;
use desim::proto::{AnyMessage, NetEvent, NodeEvent};
use http::Uri;
use postgres_backend::AuthType;
use safekeeper::SafeKeeperConf;
use safekeeper::safekeeper::{
ProposerAcceptorMessage, SK_PROTO_VERSION_3, SafeKeeper, UNKNOWN_SERVER_VERSION,

2
storage_controller/src/hadron_token.rs
View File

@@ -3,7 +3,7 @@ use camino::Utf8Path;
use jsonwebtoken::EncodingKey;
use std::fs;
use utils::{
auth::{encode_hadron_token_with_encoding_key, Claims, Scope},
auth::{Claims, Scope, encode_hadron_token_with_encoding_key},
id::TenantId,
};
use uuid::Uuid;

2
storage_controller/src/http.rs
View File

@@ -40,6 +40,7 @@ use tokio_util::sync::CancellationToken;
use tracing::warn;
use utils::auth::{Scope, SwappableJwtAuth};
use utils::id::{NodeId, TenantId, TimelineId};
use uuid::Uuid;
use crate::http;
use crate::metrics::{
@@ -1805,6 +1806,7 @@ fn check_permissions(request: &Request<Body>, required_scope: Scope) -> Result<(
/// Access by Admin-scope tokens is also permitted.
/// TODO(william.huang): Merge with the previous function by refactoring `Scope` to make it carry the dependent arguments.
/// E.g., `Scope::TenantEndpoint(EndpointId)`, `Scope::Tenant(TenantId)`, etc.
#[allow(unused)]
fn check_endpoint_permission(request: &Request<Body>, endpoint_id: Uuid) -> Result<(), ApiError> {
check_permission_with(
request,

9
storage_controller/src/main.rs
View File

@@ -515,17 +515,12 @@ async fn async_main() -> anyhow::Result<()> {
let persistence = Arc::new(Persistence::new(secrets.database_url).await);
let service = Service::spawn(
config,
persistence.clone(),
secrets.token_generator,
)
.await?;
let service = Service::spawn(config, persistence.clone(), secrets.token_generator).await?;
let jwt_auth = secrets
.public_key
.map(|jwt_auth| Arc::new(SwappableJwtAuth::new(jwt_auth)));
let router = make_router(service.clone(), auth, build_info)
let router = make_router(service.clone(), jwt_auth, build_info)
.build()
.map_err(|err| anyhow!(err))?;
let http_service =

9
storage_controller/src/service.rs
View File

@@ -4,6 +4,7 @@ pub(crate) mod safekeeper_reconciler;
mod safekeeper_service;
mod tenant_shard_iterator;
use crate::hadron_token::HadronTokenGenerator;
use std::borrow::Cow;
use std::cmp::Ordering;
use std::collections::{BTreeMap, HashMap, HashSet};
@@ -14,7 +15,6 @@ use std::path::PathBuf;
use std::str::FromStr;
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant, SystemTime};
use crate::hadron_token::HadronTokenGenerator;
use anyhow::Context;
use control_plane::storage_controller::{
@@ -518,6 +518,7 @@ pub struct Service {
persistence: Arc<Persistence>,
// HadronTokenGenerator to generate (sign) JWTs during compute deployment and compute-spec generation.
#[allow(unused)]
token_generator: Option<HadronTokenGenerator>,
compute_hook: Arc<ComputeHook>,
@@ -1661,7 +1662,11 @@ impl Service {
}
}
pub async fn spawn(config: Config, persistence: Arc<Persistence>, token_generator: Option<HadronTokenGenerator>) -> anyhow::Result<Arc<Self>> {
pub async fn spawn(
config: Config,
persistence: Arc<Persistence>,
token_generator: Option<HadronTokenGenerator>,
) -> anyhow::Result<Arc<Self>> {
let (result_tx, result_rx) = tokio::sync::mpsc::unbounded_channel();
let (abort_tx, abort_rx) = tokio::sync::mpsc::unbounded_channel();

3
test_runner/fixtures/neon_fixtures.py
View File

@@ -1132,7 +1132,8 @@ class NeonEnv:
or config.use_https_storage_broker_api
)
self.ssl_ca_file = (
self.repo_dir.joinpath("rootCA.crt") if self.generate_local_ssl_certs else None)
self.repo_dir.joinpath("rootCA.crt") if self.generate_local_ssl_certs else None
)
# The auth token type used in the test environment. neon_local is instruted to generate key pairs
# according to the auth token type. The keys are always generated but are only used if

2
vendor/postgres-v14 vendored

Submodule vendor/postgres-v14 updated: 4cacada8bd...8ce1f52303

2
vendor/postgres-v15 vendored

Submodule vendor/postgres-v15 updated: e5ee23d998...afd46987f3

2
vendor/postgres-v16 vendored

Submodule vendor/postgres-v16 updated: ad2b69b582...e08c8d5f15

2
vendor/postgres-v17 vendored

Submodule vendor/postgres-v17 updated: ba750903a9...353c725b0c