Compare commits

...

8 Commits

Author SHA1 Message Date
Conrad Ludgate
0e952cee8e fix deps 2025-07-11 14:46:56 +01:00
Conrad Ludgate
6d7ab67401 fix lints 2025-07-11 14:18:30 +01:00
Conrad Ludgate
3ff8afa32c update measured properly 2025-07-11 13:59:21 +01:00
Conrad Ludgate
682a54fa9e replace lasso completely 2025-07-11 13:59:21 +01:00
Conrad Ludgate
d5c17559ce bump 2025-07-11 13:59:21 +01:00
Conrad Ludgate
e8ccb4a4d1 fmt 2025-07-11 13:58:53 +01:00
Conrad Ludgate
bdd68bb069 bump 2025-07-11 13:58:53 +01:00
Conrad Ludgate
783260b88a switch to paracord 2025-07-11 13:57:59 +01:00
16 changed files with 139 additions and 307 deletions

118
Cargo.lock generated
View File

@@ -1009,6 +1009,12 @@ dependencies = [
"generic-array",
]
[[package]]
name = "boxcar"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26c4925bc979b677330a8c7fe7a8c94af2dbb4a2d37b4a20a80d884400f46baa"
[[package]]
name = "bstr"
version = "1.5.0"
@@ -1242,14 +1248,14 @@ checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7"
[[package]]
name = "clashmap"
version = "1.0.0"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93bd59c81e2bd87a775ae2de75f070f7e2bfe97363a6ad652f46824564c23e4d"
checksum = "e8a055b1f1bf558eae4959f6dd77cf2d7d50ae1483928e60ef21ca5a24fd4321"
dependencies = [
"crossbeam-utils",
"hashbrown 0.15.2",
"lock_api",
"parking_lot_core 0.9.8",
"parking_lot_core 0.9.10",
"polonius-the-crab",
"replace_with",
]
@@ -1753,19 +1759,6 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "dashmap"
version = "5.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d"
dependencies = [
"cfg-if",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core 0.9.8",
]
[[package]]
name = "dashmap"
version = "6.1.0"
@@ -1777,7 +1770,7 @@ dependencies = [
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core 0.9.8",
"parking_lot_core 0.9.10",
]
[[package]]
@@ -2343,6 +2336,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@@ -2582,7 +2581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "842dc78579ce01e6a1576ad896edc92fca002dd60c9c3746b7fc2bec6fb429d0"
dependencies = [
"cfg-if",
"dashmap 6.1.0",
"dashmap",
"futures-sink",
"futures-timer",
"futures-util",
@@ -3288,7 +3287,7 @@ dependencies = [
"clap",
"crossbeam-channel",
"crossbeam-utils",
"dashmap 6.1.0",
"dashmap",
"env_logger",
"indexmap 2.9.0",
"itoa",
@@ -3545,16 +3544,6 @@ dependencies = [
"libc",
]
[[package]]
name = "lasso"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4644821e1c3d7a560fe13d842d13f587c07348a1a05d3a797152d41c90c56df2"
dependencies = [
"dashmap 5.5.0",
"hashbrown 0.13.2",
]
[[package]]
name = "lazy_static"
version = "1.5.0"
@@ -3674,17 +3663,17 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "measured"
version = "0.0.22"
version = "0.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3051f3a030d55d680cdef6ca50e80abd1182f8da29f2344a7c9cb575721138f0"
checksum = "d22ae866c28b9c59afaeb488ad6e3bd148570cf5a2bacf6c4845def5b9a03470"
dependencies = [
"bytes",
"crossbeam-utils",
"hashbrown 0.14.5",
"itoa",
"lasso",
"measured-derive",
"memchr",
"paracord",
"parking_lot 0.12.1",
"rustc-hash 1.1.0",
"ryu",
@@ -3692,9 +3681,9 @@ dependencies = [
[[package]]
name = "measured-derive"
version = "0.0.22"
version = "0.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e6777fc80a575f9503d908c8b498782a6c3ee88a06cb416dc3941401e43b94"
checksum = "16d734ed9dbca22d87d56b54d220f254ce921cb5cce97d4a960075af0131d076"
dependencies = [
"heck",
"proc-macro2",
@@ -3704,15 +3693,26 @@ dependencies = [
[[package]]
name = "measured-process"
version = "0.0.22"
version = "0.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c4b80445aeb08e832d87bf1830049a924cdc1d6b7ef40b6b9b365bff17bf8ec"
checksum = "f71a318d2b9edcded1e5e0ccf3c9e1e1614217d7f07933631c771daa717743aa"
dependencies = [
"libc",
"measured",
"procfs",
]
[[package]]
name = "measured-tokio"
version = "0.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e79a936051c484268e2d71a5dd01219096c62c8ff09afad95f07c14a3b0c580"
dependencies = [
"itoa",
"measured",
"tokio",
]
[[package]]
name = "memchr"
version = "2.6.4"
@@ -4571,6 +4571,21 @@ dependencies = [
"seize",
]
[[package]]
name = "paracord"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e41e924113f9a05eecd561c6e695648f243a03b3ad8d9b3eff689342b95023b"
dependencies = [
"boxcar",
"clashmap",
"foldhash",
"hashbrown 0.15.2",
"serde",
"sync_wrapper 1.0.1",
"typed-arena",
]
[[package]]
name = "parking"
version = "2.1.1"
@@ -4595,7 +4610,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core 0.9.8",
"parking_lot_core 0.9.10",
]
[[package]]
@@ -4614,15 +4629,15 @@ dependencies = [
[[package]]
name = "parking_lot_core"
version = "0.9.8"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
"redox_syscall 0.3.5",
"redox_syscall 0.5.10",
"smallvec",
"windows-targets 0.48.0",
"windows-targets 0.52.6",
]
[[package]]
@@ -5328,13 +5343,14 @@ dependencies = [
"itoa",
"jose-jwa",
"jose-jwk",
"lasso",
"measured",
"measured-tokio",
"metrics",
"once_cell",
"opentelemetry",
"p256 0.13.2",
"papaya",
"paracord",
"parking_lot 0.12.1",
"parquet",
"parquet_derive",
@@ -5637,6 +5653,15 @@ dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
dependencies = [
"bitflags 2.8.0",
]
[[package]]
name = "regex"
version = "1.10.2"
@@ -6897,12 +6922,12 @@ dependencies = [
"hyper 0.14.30",
"itertools 0.10.5",
"json-structural-diff",
"lasso",
"measured",
"metrics",
"once_cell",
"pageserver_api",
"pageserver_client",
"paracord",
"postgres_connection",
"posthog_client_lite",
"rand 0.8.5",
@@ -8017,6 +8042,12 @@ dependencies = [
"static_assertions",
]
[[package]]
name = "typed-arena"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
[[package]]
name = "typed-json"
version = "0.1.1"
@@ -8761,6 +8792,7 @@ dependencies = [
"num-traits",
"once_cell",
"p256 0.13.2",
"paracord",
"parquet",
"prettyplease",
"proc-macro2",

View File

@@ -128,11 +128,11 @@ itertools = "0.10"
itoa = "1.0.11"
jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
jsonwebtoken = "9"
lasso = "0.7"
libc = "0.2"
md5 = "0.7.0"
measured = { version = "0.0.22", features=["lasso"] }
measured-process = { version = "0.0.22" }
measured = { version = "0.0.23", features = ["paracord"] }
measured-process = { version = "0.0.23" }
measured-tokio = { version = "0.0.23" }
memoffset = "0.9"
nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket", "signal", "poll"] }
# Do not update to >= 7.0.0, at least. The update will have a significant impact
@@ -145,6 +145,7 @@ opentelemetry = "0.27"
opentelemetry_sdk = "0.27"
opentelemetry-otlp = { version = "0.27", default-features = false, features = ["http-proto", "trace", "http", "reqwest-client"] }
opentelemetry-semantic-conventions = "0.27"
paracord = { version = "0.1.0", features = ["serde"] }
parking_lot = "0.12"
parquet = { version = "53", default-features = false, features = ["zstd"] }
parquet_derive = "53"

View File

@@ -52,6 +52,7 @@ exceptions = [
# Zlib license has some restrictions if we decide to change sth
{ allow = ["Zlib"], name = "const_format_proc_macros", version = "*" },
{ allow = ["Zlib"], name = "const_format", version = "*" },
{ allow = ["Zlib"], name = "foldhash", version = "*" },
]
[licenses.private]

View File

@@ -48,12 +48,13 @@ indexmap = { workspace = true, features = ["serde"] }
ipnet.workspace = true
itertools.workspace = true
itoa.workspace = true
lasso = { workspace = true, features = ["multi-threaded"] }
measured = { workspace = true, features = ["lasso"] }
measured = { workspace = true, features = ["paracord"] }
measured-tokio.workspace = true
metrics.workspace = true
once_cell.workspace = true
opentelemetry = { workspace = true, features = ["trace"] }
papaya = "0.2.0"
paracord.workspace = true
parking_lot.workspace = true
parquet.workspace = true
parquet_derive.workspace = true
@@ -127,4 +128,4 @@ rstest.workspace = true
walkdir.workspace = true
rand_distr = "0.4"
tokio-postgres.workspace = true
tracing-test = "0.2"
tracing-test = "0.2"

View File

@@ -87,7 +87,12 @@ impl JwkCacheEntry {
self.key_sets
.values()
// make sure our requested role has access to the key set
.filter(|key_set| key_set.role_names.iter().any(|role| **role == **role_name))
.filter(|key_set| {
key_set
.role_names
.iter()
.any(|role| *role.as_str() == **role_name)
})
// try and find the requested key-id in the key set
.find_map(|key_set| {
key_set

View File

@@ -12,7 +12,7 @@ use crate::context::RequestContext;
use crate::control_plane::NodeInfo;
use crate::control_plane::messages::{ColdStartInfo, EndpointJwksResponse, MetricsAuxInfo};
use crate::http;
use crate::intern::{BranchIdTag, EndpointIdTag, InternId, ProjectIdTag};
use crate::intern::{BranchIdInt, EndpointIdInt, ProjectIdInt};
use crate::types::EndpointId;
use crate::url::ApiUrl;
@@ -38,9 +38,9 @@ impl LocalBackend {
},
// TODO(conrad): make this better reflect compute info rather than endpoint info.
aux: MetricsAuxInfo {
endpoint_id: EndpointIdTag::get_interner().get_or_intern("local"),
project_id: ProjectIdTag::get_interner().get_or_intern("local"),
branch_id: BranchIdTag::get_interner().get_or_intern("local"),
endpoint_id: EndpointIdInt::new("local"),
project_id: ProjectIdInt::new("local"),
branch_id: BranchIdInt::new("local"),
compute_id: "local".into(),
cold_start_info: ColdStartInfo::WarmCached,
},

View File

@@ -192,6 +192,7 @@ pub async fn run() -> anyhow::Result<()> {
jemalloc,
neon_metrics,
proxy: crate::metrics::Metrics::get(),
tokio: measured_tokio::RuntimeCollector::current(),
},
));

View File

@@ -513,6 +513,7 @@ pub async fn run() -> anyhow::Result<()> {
jemalloc,
neon_metrics,
proxy: crate::metrics::Metrics::get(),
tokio: measured_tokio::RuntimeCollector::current(),
},
));
maintenance_tasks.spawn(control_plane::mgmt::task_main(mgmt_listener));

View File

@@ -194,7 +194,7 @@ impl ProjectInfoCacheImpl {
&self,
endpoint_id: &EndpointId,
) -> Option<Ref<'_, EndpointIdInt, EndpointInfo>> {
let endpoint_id = EndpointIdInt::get(endpoint_id)?;
let endpoint_id = EndpointIdInt::try_new_existing(endpoint_id)?;
self.cache.get(&endpoint_id)
}
@@ -204,7 +204,7 @@ impl ProjectInfoCacheImpl {
role_name: &RoleName,
) -> Option<RoleAccessControl> {
let valid_since = self.get_cache_times();
let role_name = RoleNameInt::get(role_name)?;
let role_name = RoleNameInt::try_new_existing(role_name)?;
let endpoint_info = self.get_endpoint_cache(endpoint_id)?;
endpoint_info.get_role_secret(role_name, valid_since)
}
@@ -297,10 +297,10 @@ impl ProjectInfoCacheImpl {
}
pub fn maybe_invalidate_role_secret(&self, endpoint_id: &EndpointId, role_name: &RoleName) {
let Some(endpoint_id) = EndpointIdInt::get(endpoint_id) else {
let Some(endpoint_id) = EndpointIdInt::try_new_existing(endpoint_id) else {
return;
};
let Some(role_name) = RoleNameInt::get(role_name) else {
let Some(role_name) = RoleNameInt::try_new_existing(role_name) else {
return;
};

View File

@@ -71,6 +71,8 @@ pub struct AppMetrics {
pub neon_metrics: NeonMetrics,
#[metric(flatten)]
pub proxy: &'static crate::metrics::Metrics,
#[metric(namespace = "tokio")]
pub tokio: measured_tokio::RuntimeCollector,
}
async fn prometheus_metrics_handler(

View File

@@ -1,272 +1,59 @@
use std::hash::BuildHasherDefault;
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use std::ops::Index;
use std::sync::OnceLock;
use lasso::{Capacity, MemoryLimits, Spur, ThreadedRodeo};
use rustc_hash::FxHasher;
use paracord::custom_key;
use crate::types::{AccountId, BranchId, EndpointId, ProjectId, RoleName};
pub trait InternId: Sized + 'static {
fn get_interner() -> &'static StringInterner<Self>;
}
custom_key!(pub struct RoleNameInt);
custom_key!(pub struct EndpointIdInt);
custom_key!(pub struct BranchIdInt);
custom_key!(pub struct ProjectIdInt);
custom_key!(pub struct AccountIdInt);
pub struct StringInterner<Id> {
inner: ThreadedRodeo<Spur, BuildHasherDefault<FxHasher>>,
_id: PhantomData<Id>,
}
#[derive(PartialEq, Debug, Clone, Copy, Eq, Hash)]
pub struct InternedString<Id> {
inner: Spur,
_id: PhantomData<Id>,
}
impl<Id: InternId> std::fmt::Display for InternedString<Id> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.as_str().fmt(f)
}
}
impl<Id: InternId> InternedString<Id> {
pub(crate) fn as_str(&self) -> &'static str {
Id::get_interner().inner.resolve(&self.inner)
}
pub(crate) fn get(s: &str) -> Option<Self> {
Id::get_interner().get(s)
}
}
impl<Id: InternId> AsRef<str> for InternedString<Id> {
fn as_ref(&self) -> &str {
self.as_str()
}
}
impl<Id: InternId> std::ops::Deref for InternedString<Id> {
type Target = str;
fn deref(&self) -> &str {
self.as_str()
}
}
impl<'de, Id: InternId> serde::de::Deserialize<'de> for InternedString<Id> {
fn deserialize<D: serde::de::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
struct Visitor<Id>(PhantomData<Id>);
impl<Id: InternId> serde::de::Visitor<'_> for Visitor<Id> {
type Value = InternedString<Id>;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("a string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(Id::get_interner().get_or_intern(v))
}
}
d.deserialize_str(Visitor::<Id>(PhantomData))
}
}
impl<Id: InternId> serde::Serialize for InternedString<Id> {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
self.as_str().serialize(s)
}
}
impl<Id: InternId> StringInterner<Id> {
pub(crate) fn new() -> Self {
StringInterner {
inner: ThreadedRodeo::with_capacity_memory_limits_and_hasher(
Capacity::new(2500, NonZeroUsize::new(1 << 16).expect("value is nonzero")),
// unbounded
MemoryLimits::for_memory_usage(usize::MAX),
BuildHasherDefault::<FxHasher>::default(),
),
_id: PhantomData,
}
}
#[cfg(test)]
fn len(&self) -> usize {
self.inner.len()
}
#[cfg(test)]
fn current_memory_usage(&self) -> usize {
self.inner.current_memory_usage()
}
pub(crate) fn get_or_intern(&self, s: &str) -> InternedString<Id> {
InternedString {
inner: self.inner.get_or_intern(s),
_id: PhantomData,
}
}
pub(crate) fn get(&self, s: &str) -> Option<InternedString<Id>> {
Some(InternedString {
inner: self.inner.get(s)?,
_id: PhantomData,
})
}
}
impl<Id: InternId> Index<InternedString<Id>> for StringInterner<Id> {
type Output = str;
fn index(&self, index: InternedString<Id>) -> &Self::Output {
self.inner.resolve(&index.inner)
}
}
impl<Id: InternId> Default for StringInterner<Id> {
fn default() -> Self {
Self::new()
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub struct RoleNameTag;
impl InternId for RoleNameTag {
fn get_interner() -> &'static StringInterner<Self> {
static ROLE_NAMES: OnceLock<StringInterner<RoleNameTag>> = OnceLock::new();
ROLE_NAMES.get_or_init(Default::default)
}
}
pub type RoleNameInt = InternedString<RoleNameTag>;
impl From<&RoleName> for RoleNameInt {
fn from(value: &RoleName) -> Self {
RoleNameTag::get_interner().get_or_intern(value)
RoleNameInt::new(value)
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub struct EndpointIdTag;
impl InternId for EndpointIdTag {
fn get_interner() -> &'static StringInterner<Self> {
static ROLE_NAMES: OnceLock<StringInterner<EndpointIdTag>> = OnceLock::new();
ROLE_NAMES.get_or_init(Default::default)
}
}
pub type EndpointIdInt = InternedString<EndpointIdTag>;
impl From<&EndpointId> for EndpointIdInt {
fn from(value: &EndpointId) -> Self {
EndpointIdTag::get_interner().get_or_intern(value)
EndpointIdInt::new(value)
}
}
impl From<EndpointId> for EndpointIdInt {
fn from(value: EndpointId) -> Self {
EndpointIdTag::get_interner().get_or_intern(&value)
EndpointIdInt::new(&value)
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub struct BranchIdTag;
impl InternId for BranchIdTag {
fn get_interner() -> &'static StringInterner<Self> {
static ROLE_NAMES: OnceLock<StringInterner<BranchIdTag>> = OnceLock::new();
ROLE_NAMES.get_or_init(Default::default)
}
}
pub type BranchIdInt = InternedString<BranchIdTag>;
impl From<&BranchId> for BranchIdInt {
fn from(value: &BranchId) -> Self {
BranchIdTag::get_interner().get_or_intern(value)
BranchIdInt::new(value)
}
}
impl From<BranchId> for BranchIdInt {
fn from(value: BranchId) -> Self {
BranchIdTag::get_interner().get_or_intern(&value)
BranchIdInt::new(&value)
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub struct ProjectIdTag;
impl InternId for ProjectIdTag {
fn get_interner() -> &'static StringInterner<Self> {
static ROLE_NAMES: OnceLock<StringInterner<ProjectIdTag>> = OnceLock::new();
ROLE_NAMES.get_or_init(Default::default)
}
}
pub type ProjectIdInt = InternedString<ProjectIdTag>;
impl From<&ProjectId> for ProjectIdInt {
fn from(value: &ProjectId) -> Self {
ProjectIdTag::get_interner().get_or_intern(value)
ProjectIdInt::new(value)
}
}
impl From<ProjectId> for ProjectIdInt {
fn from(value: ProjectId) -> Self {
ProjectIdTag::get_interner().get_or_intern(&value)
ProjectIdInt::new(&value)
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub struct AccountIdTag;
impl InternId for AccountIdTag {
fn get_interner() -> &'static StringInterner<Self> {
static ROLE_NAMES: OnceLock<StringInterner<AccountIdTag>> = OnceLock::new();
ROLE_NAMES.get_or_init(Default::default)
}
}
pub type AccountIdInt = InternedString<AccountIdTag>;
impl From<&AccountId> for AccountIdInt {
fn from(value: &AccountId) -> Self {
AccountIdTag::get_interner().get_or_intern(value)
AccountIdInt::new(value)
}
}
impl From<AccountId> for AccountIdInt {
fn from(value: AccountId) -> Self {
AccountIdTag::get_interner().get_or_intern(&value)
}
}
#[cfg(test)]
mod tests {
use std::sync::OnceLock;
use super::InternId;
use crate::intern::StringInterner;
struct MyId;
impl InternId for MyId {
fn get_interner() -> &'static StringInterner<Self> {
pub(crate) static ROLE_NAMES: OnceLock<StringInterner<MyId>> = OnceLock::new();
ROLE_NAMES.get_or_init(Default::default)
}
}
#[test]
fn push_many_strings() {
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use rand_distr::Zipf;
let endpoint_dist = Zipf::new(500000, 0.8).unwrap();
let endpoints = StdRng::seed_from_u64(272488357).sample_iter(endpoint_dist);
let interner = MyId::get_interner();
const N: usize = 100_000;
let mut verify = Vec::with_capacity(N);
for endpoint in endpoints.take(N) {
let endpoint = format!("ep-string-interning-{endpoint}");
let key = interner.get_or_intern(&endpoint);
verify.push((endpoint, key));
}
for (s, key) in verify {
assert_eq!(interner[key], s);
}
// 2031616/59861 = 34 bytes per string
assert_eq!(interner.len(), 59_861);
// will have other overhead for the internal hashmaps that are not accounted for.
assert_eq!(interner.current_memory_usage(), 2_031_616);
AccountIdInt::new(&value)
}
}

View File

@@ -1,6 +1,5 @@
use std::sync::{Arc, OnceLock};
use lasso::ThreadedRodeo;
use measured::label::{
FixedCardinalitySet, LabelGroupSet, LabelName, LabelSet, LabelValue, StaticLabelSet,
};
@@ -11,6 +10,7 @@ use measured::{
MetricGroup,
};
use metrics::{CounterPairAssoc, CounterPairVec, HyperLogLogVec};
use paracord::ParaCord;
use tokio::time::{self, Instant};
use crate::control_plane::messages::ColdStartInfo;
@@ -222,7 +222,7 @@ pub enum CacheOutcome {
#[derive(LabelGroup)]
#[label(set = ConsoleRequestSet)]
pub struct ConsoleRequest<'a> {
#[label(dynamic_with = ThreadedRodeo, default)]
#[label(dynamic_with = ParaCord, default)]
pub request: &'a str,
}
@@ -345,7 +345,7 @@ pub struct ConnectionFailuresBreakdownGroup {
#[derive(LabelGroup, Copy, Clone)]
#[label(set = RedisErrorsSet)]
pub struct RedisErrors<'a> {
#[label(dynamic_with = ThreadedRodeo, default)]
#[label(dynamic_with = ParaCord, default)]
pub channel: &'a str,
}

View File

@@ -1,4 +1,4 @@
use crate::intern::{EndpointIdInt, EndpointIdTag, InternId};
use crate::intern::EndpointIdInt;
macro_rules! smol_str_wrapper {
($name:ident) => {
@@ -85,7 +85,7 @@ impl EndpointId {
#[must_use]
pub fn normalize_intern(&self) -> EndpointIdInt {
EndpointIdTag::get_interner().get_or_intern(self.normalize_str())
EndpointIdInt::new(self.normalize_str())
}
}

View File

@@ -31,10 +31,10 @@ humantime.workspace = true
humantime-serde.workspace = true
itertools.workspace = true
json-structural-diff.workspace = true
lasso.workspace = true
once_cell.workspace = true
pageserver_api.workspace = true
pageserver_client.workspace = true
paracord.workspace = true
postgres_connection.workspace = true
posthog_client_lite.workspace = true
rand.workspace = true
@@ -72,4 +72,4 @@ http-utils = { path = "../libs/http-utils/" }
utils = { path = "../libs/utils/" }
metrics = { path = "../libs/metrics/" }
control_plane = { path = "../control_plane" }
workspace_hack = { version = "0.1", path = "../workspace_hack" }
workspace_hack = { version = "0.1", path = "../workspace_hack" }

View File

@@ -183,9 +183,9 @@ impl Default for StorageControllerMetrics {
#[derive(measured::LabelGroup, Clone)]
#[label(set = NodeLabelGroupSet)]
pub(crate) struct NodeLabelGroup<'a> {
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) az: &'a str,
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) node_id: &'a str,
}
@@ -198,7 +198,7 @@ pub(crate) struct ReconcileCompleteLabelGroup {
#[derive(measured::LabelGroup)]
#[label(set = HttpRequestStatusLabelGroupSet)]
pub(crate) struct HttpRequestStatusLabelGroup<'a> {
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) path: &'a str,
pub(crate) method: Method,
pub(crate) status: StatusCode,
@@ -207,7 +207,7 @@ pub(crate) struct HttpRequestStatusLabelGroup<'a> {
#[derive(measured::LabelGroup)]
#[label(set = HttpRequestLatencyLabelGroupSet)]
pub(crate) struct HttpRequestLatencyLabelGroup<'a> {
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) path: &'a str,
pub(crate) method: Method,
}
@@ -215,9 +215,9 @@ pub(crate) struct HttpRequestLatencyLabelGroup<'a> {
#[derive(measured::LabelGroup, Clone)]
#[label(set = PageserverRequestLabelGroupSet)]
pub(crate) struct PageserverRequestLabelGroup<'a> {
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) pageserver_id: &'a str,
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) path: &'a str,
pub(crate) method: Method,
}
@@ -225,9 +225,9 @@ pub(crate) struct PageserverRequestLabelGroup<'a> {
#[derive(measured::LabelGroup, Clone)]
#[label(set = SafekeeperRequestLabelGroupSet)]
pub(crate) struct SafekeeperRequestLabelGroup<'a> {
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) safekeeper_id: &'a str,
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) path: &'a str,
pub(crate) method: Method,
}
@@ -254,11 +254,11 @@ pub(crate) struct LeadershipStatusGroup {
#[derive(measured::LabelGroup, Clone)]
#[label(set = ReconcileLongRunningLabelGroupSet)]
pub(crate) struct ReconcileLongRunningLabelGroup<'a> {
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) tenant_id: &'a str,
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) shard_number: &'a str,
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) sequence: &'a str,
}
@@ -282,11 +282,11 @@ pub(crate) enum Method {
#[derive(measured::LabelGroup, Clone)]
#[label(set = SafekeeperReconcilerLabelGroupSet)]
pub(crate) struct SafekeeperReconcilerLabelGroup<'a> {
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) sk_az: &'a str,
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) sk_node_id: &'a str,
#[label(dynamic_with = lasso::ThreadedRodeo, default)]
#[label(dynamic_with = paracord::ParaCord, default)]
pub(crate) sk_hostname: &'a str,
}

View File

@@ -72,6 +72,7 @@ num-rational = { version = "0.4", default-features = false, features = ["num-big
num-traits = { version = "0.2", features = ["i128", "libm"] }
once_cell = { version = "1" }
p256 = { version = "0.13", features = ["jwk"] }
paracord = { version = "0.1", features = ["serde"] }
parquet = { version = "53", default-features = false, features = ["zstd"] }
prost = { version = "0.13", features = ["no-recursion-limit", "prost-derive"] }
rand = { version = "0.8", features = ["small_rng"] }