diff --git a/Cargo.lock b/Cargo.lock index 750ac0edc2..a9de71420b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -441,6 +441,7 @@ dependencies = [ "regex", "reqwest", "serde", + "serde_with", "tar", "thiserror", "toml", @@ -600,6 +601,41 @@ dependencies = [ "libc", ] +[[package]] +name = "darling" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.9.0" @@ -1038,6 +1074,12 @@ dependencies = [ "tokio-rustls 0.23.2", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -1422,7 +1464,6 @@ dependencies = [ "daemonize", "fail", "futures", - "hex", "hex-literal", "humantime", "hyper", @@ -1440,6 +1481,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", + "serde_with", "signal-hook", "tar", "tempfile", @@ -2075,6 +2117,12 @@ dependencies = [ "rustls 0.19.1", ] +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + [[package]] name = "ryu" version = "1.0.9" @@ -2187,6 +2235,29 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec1e6ec4d8950e5b1e894eac0d360742f3b1407a6078a604a731c4b3f49cefbc" +dependencies = [ + "rustversion", + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha2" version = "0.9.9" @@ -3056,6 +3127,7 @@ dependencies = [ "rustls-split", "serde", "serde_json", + "serde_with", "signal-hook", "tempfile", "thiserror", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index eff6b3ef2d..b52c7ad5a9 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" tar = "0.4.33" postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" } serde = { version = "1.0", features = ["derive"] } +serde_with = "1.12.0" toml = "0.5" lazy_static = "1.4" regex = "1" diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 2a1d51fe08..00ace431e6 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -5,6 +5,7 @@ use anyhow::{bail, ensure, Context}; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use std::collections::HashMap; use std::env; use std::fs; @@ -12,9 +13,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ - HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId, -}; +use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}; use crate::safekeeper::SafekeeperNode; @@ -25,6 +24,7 @@ use crate::safekeeper::SafekeeperNode; // to 'zenith init --config=' option. See control_plane/simple.conf for // an example. // +#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct LocalEnv { // Base directory for all the nodes (the pageserver, safekeepers and @@ -50,7 +50,8 @@ pub struct LocalEnv { // Default tenant ID to use with the 'zenith' command line utility, when // --tenantid is not explicitly specified. #[serde(default)] - pub default_tenant_id: Option, + #[serde_as(as = "Option")] + pub default_tenant_id: Option, // used to issue tokens during e.g pg start #[serde(default)] @@ -66,7 +67,8 @@ pub struct LocalEnv { // A `HashMap>` would be more appropriate here, // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table". - branch_name_mappings: HashMap>, + #[serde_as(as = "HashMap<_, Vec<(DisplayFromStr, DisplayFromStr)>>")] + branch_name_mappings: HashMap>, } #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] @@ -164,9 +166,6 @@ impl LocalEnv { .entry(branch_name.clone()) .or_default(); - let tenant_id = HexZTenantId::from(tenant_id); - let timeline_id = HexZTimelineId::from(timeline_id); - let existing_ids = existing_values .iter() .find(|(existing_tenant_id, _)| existing_tenant_id == &tenant_id); @@ -193,7 +192,6 @@ impl LocalEnv { branch_name: &str, tenant_id: ZTenantId, ) -> Option { - let tenant_id = HexZTenantId::from(tenant_id); self.branch_name_mappings .get(branch_name)? .iter() @@ -207,13 +205,7 @@ impl LocalEnv { .iter() .flat_map(|(name, tenant_timelines)| { tenant_timelines.iter().map(|&(tenant_id, timeline_id)| { - ( - ZTenantTimelineId::new( - ZTenantId::from(tenant_id), - ZTimelineId::from(timeline_id), - ), - name.clone(), - ) + (ZTenantTimelineId::new(tenant_id, timeline_id), name.clone()) }) }) .collect() @@ -259,7 +251,7 @@ impl LocalEnv { // If no initial tenant ID was given, generate it. if env.default_tenant_id.is_none() { - env.default_tenant_id = Some(HexZTenantId::from(ZTenantId::generate())); + env.default_tenant_id = Some(ZTenantId::generate()); } env.base_data_dir = base_path(); diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index ef43ba3c1e..835c93bf1d 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -18,7 +18,7 @@ use thiserror::Error; use zenith_utils::http::error::HttpErrorBody; use zenith_utils::lsn::Lsn; use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{HexZTenantId, HexZTimelineId, ZTenantId, ZTimelineId}; +use zenith_utils::zid::{ZTenantId, ZTimelineId}; use crate::local_env::LocalEnv; use crate::{fill_rust_env_vars, read_pidfile}; @@ -337,9 +337,7 @@ impl PageServerNode { ) -> anyhow::Result> { let tenant_id_string = self .http_request(Method::POST, format!("{}/tenant", self.http_base_url)) - .json(&TenantCreateRequest { - new_tenant_id: new_tenant_id.map(HexZTenantId::from), - }) + .json(&TenantCreateRequest { new_tenant_id }) .send()? .error_from_body()? .json::>()?; @@ -382,9 +380,9 @@ impl PageServerNode { format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id), ) .json(&TimelineCreateRequest { - new_timeline_id: new_timeline_id.map(HexZTimelineId::from), + new_timeline_id, ancestor_start_lsn, - ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from), + ancestor_timeline_id, }) .send()? .error_from_body()? diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index cfcb453732..efd2fa4a38 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -25,11 +25,12 @@ tokio-stream = "0.1.8" anyhow = { version = "1.0", features = ["backtrace"] } crc32c = "0.6.0" thiserror = "1.0" -hex = { version = "0.4.3", features = ["serde"] } tar = "0.4.33" humantime = "2.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1" +serde_with = "1.12.0" + toml_edit = { version = "0.13", features = ["easy"] } scopeguard = "1.1.0" async-trait = "0.1" diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index 8827713f11..c28cd0def7 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,24 +1,39 @@ -use anyhow::Context; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use zenith_utils::{ lsn::Lsn, - zid::{HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTimelineId}, + zid::{ZNodeId, ZTenantId, ZTimelineId}, }; use crate::timelines::{LocalTimelineInfo, TimelineInfo}; +#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - pub new_timeline_id: Option, - pub ancestor_timeline_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub new_timeline_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub ancestor_timeline_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] pub ancestor_start_lsn: Option, } +#[serde_as] #[derive(Serialize, Deserialize)] pub struct TenantCreateRequest { - pub new_tenant_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub new_tenant_id: Option, } +#[serde_as] +#[derive(Serialize, Deserialize)] +#[serde(transparent)] +pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub ZTenantId); + #[derive(Clone)] pub enum TimelineInfoV1 { Local { @@ -39,18 +54,24 @@ pub enum TimelineInfoV1 { }, } +#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineInfoResponseV1 { pub kind: String, - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] timeline_id: ZTimelineId, - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] tenant_id: ZTenantId, - disk_consistent_lsn: String, - last_record_lsn: Option, - prev_record_lsn: Option, - ancestor_timeline_id: Option, - ancestor_lsn: Option, + #[serde_as(as = "DisplayFromStr")] + disk_consistent_lsn: Lsn, + #[serde_as(as = "Option")] + last_record_lsn: Option, + #[serde_as(as = "Option")] + prev_record_lsn: Option, + #[serde_as(as = "Option")] + ancestor_timeline_id: Option, + #[serde_as(as = "Option")] + ancestor_lsn: Option, current_logical_size: Option, current_logical_size_non_incremental: Option, } @@ -72,11 +93,11 @@ impl From for TimelineInfoResponseV1 { kind: "Local".to_owned(), timeline_id, tenant_id, - disk_consistent_lsn: disk_consistent_lsn.to_string(), - last_record_lsn: Some(last_record_lsn.to_string()), - prev_record_lsn: prev_record_lsn.map(|lsn| lsn.to_string()), - ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from), - ancestor_lsn: ancestor_lsn.map(|lsn| lsn.to_string()), + disk_consistent_lsn, + last_record_lsn: Some(last_record_lsn), + prev_record_lsn, + ancestor_timeline_id, + ancestor_lsn, current_logical_size, current_logical_size_non_incremental, }, @@ -88,7 +109,7 @@ impl From for TimelineInfoResponseV1 { kind: "Remote".to_owned(), timeline_id, tenant_id, - disk_consistent_lsn: disk_consistent_lsn.to_string(), + disk_consistent_lsn, last_record_lsn: None, prev_record_lsn: None, ancestor_timeline_id: None, @@ -104,37 +125,24 @@ impl TryFrom for TimelineInfoV1 { type Error = anyhow::Error; fn try_from(other: TimelineInfoResponseV1) -> anyhow::Result { - let parse_lsn_hex_string = |lsn_string: String| { - lsn_string - .parse::() - .with_context(|| format!("Failed to parse Lsn as hex string from '{}'", lsn_string)) - }; - - let disk_consistent_lsn = parse_lsn_hex_string(other.disk_consistent_lsn)?; Ok(match other.kind.as_str() { "Local" => TimelineInfoV1::Local { timeline_id: other.timeline_id, tenant_id: other.tenant_id, - last_record_lsn: other - .last_record_lsn - .ok_or(anyhow::anyhow!( - "Local timeline should have last_record_lsn" - )) - .and_then(parse_lsn_hex_string)?, - prev_record_lsn: other - .prev_record_lsn - .map(parse_lsn_hex_string) - .transpose()?, + last_record_lsn: other.last_record_lsn.ok_or(anyhow::anyhow!( + "Local timeline should have last_record_lsn" + ))?, + prev_record_lsn: other.prev_record_lsn, ancestor_timeline_id: other.ancestor_timeline_id.map(ZTimelineId::from), - ancestor_lsn: other.ancestor_lsn.map(parse_lsn_hex_string).transpose()?, - disk_consistent_lsn, + ancestor_lsn: other.ancestor_lsn, + disk_consistent_lsn: other.disk_consistent_lsn, current_logical_size: other.current_logical_size, current_logical_size_non_incremental: other.current_logical_size_non_incremental, }, "Remote" => TimelineInfoV1::Remote { timeline_id: other.timeline_id, tenant_id: other.tenant_id, - disk_consistent_lsn, + disk_consistent_lsn: other.disk_consistent_lsn, }, unknown => anyhow::bail!("Unknown timeline kind: {}", unknown), }) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 2d913afe4e..a1249f463a 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -17,11 +17,11 @@ use zenith_utils::http::{ request::parse_request_param, }; use zenith_utils::http::{RequestExt, RouterBuilder}; -use zenith_utils::zid::{HexZTenantId, ZTenantTimelineId, ZTimelineId}; +use zenith_utils::zid::{ZTenantTimelineId, ZTimelineId}; use super::models::{ - StatusResponse, TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponseV1, - TimelineInfoV1, + StatusResponse, TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest, + TimelineInfoResponseV1, TimelineInfoV1, }; use crate::remote_storage::{schedule_timeline_download, RemoteTimelineIndex}; use crate::timelines::{ @@ -308,7 +308,7 @@ async fn tenant_create_handler(mut request: Request) -> Result json_response(StatusCode::CREATED, HexZTenantId::from(id))?, + Some(id) => json_response(StatusCode::CREATED, TenantCreateResponse(id))?, None => json_response(StatusCode::CONFLICT, ())?, }) } diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 8584bdd424..4d6dfd7488 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -15,6 +15,7 @@ use anyhow::{Context, Result}; use lazy_static::lazy_static; use log::*; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::fmt; @@ -267,9 +268,10 @@ pub fn get_timeline_for_tenant_load( .with_context(|| format!("Timeline {} not found for tenant {}", timelineid, tenantid)) } +#[serde_as] #[derive(Serialize, Deserialize, Clone)] pub struct TenantInfo { - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] pub id: ZTenantId, pub state: TenantState, } diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 9cfc21b413..00dd0f8f9c 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Context, Result}; use postgres_ffi::ControlFileData; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; use std::{ fs, path::Path, @@ -13,9 +14,9 @@ use std::{ }; use tracing::*; +use zenith_utils::lsn::Lsn; use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; use zenith_utils::{crashsafe_dir, logging}; -use zenith_utils::{lsn::Lsn, zid::HexZTimelineId}; use crate::{ config::PageServerConf, @@ -28,12 +29,18 @@ use crate::{layered_repository::LayeredRepository, walredo::WalRedoManager}; use crate::{repository::RepositoryTimeline, tenant_mgr}; use crate::{repository::Timeline, CheckpointConfig}; +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct LocalTimelineInfo { - pub ancestor_timeline_id: Option, + #[serde_as(as = "Option")] + pub ancestor_timeline_id: Option, + #[serde_as(as = "Option")] pub ancestor_lsn: Option, + #[serde_as(as = "DisplayFromStr")] pub last_record_lsn: Lsn, + #[serde_as(as = "Option")] pub prev_record_lsn: Option, + #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, pub current_logical_size: Option, // is None when timeline is Unloaded pub current_logical_size_non_incremental: Option, @@ -47,9 +54,7 @@ impl LocalTimelineInfo { ) -> anyhow::Result { let last_record_lsn = timeline.get_last_record_lsn(); let info = LocalTimelineInfo { - ancestor_timeline_id: timeline - .get_ancestor_timeline_id() - .map(HexZTimelineId::from), + ancestor_timeline_id: timeline.get_ancestor_timeline_id(), ancestor_lsn: { match timeline.get_ancestor_lsn() { Lsn(0) => None, @@ -72,7 +77,7 @@ impl LocalTimelineInfo { pub fn from_unloaded_timeline(metadata: &TimelineMetadata) -> Self { LocalTimelineInfo { - ancestor_timeline_id: metadata.ancestor_timeline().map(HexZTimelineId::from), + ancestor_timeline_id: metadata.ancestor_timeline(), ancestor_lsn: { match metadata.ancestor_lsn() { Lsn(0) => None, @@ -103,17 +108,20 @@ impl LocalTimelineInfo { } } +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct RemoteTimelineInfo { + #[serde_as(as = "Option")] pub remote_consistent_lsn: Option, pub awaits_download: bool, } +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimelineInfo { - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] pub tenant_id: ZTenantId, - #[serde(with = "hex")] + #[serde_as(as = "DisplayFromStr")] pub timeline_id: ZTimelineId, pub local: Option, pub remote: Option, diff --git a/test_runner/batch_others/test_remote_storage.py b/test_runner/batch_others/test_remote_storage.py index 8689838089..07a122ede9 100644 --- a/test_runner/batch_others/test_remote_storage.py +++ b/test_runner/batch_others/test_remote_storage.py @@ -7,6 +7,7 @@ from pathlib import Path from uuid import UUID from fixtures.zenith_fixtures import ZenithEnvBuilder, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload from fixtures.log_helper import log +from fixtures.utils import lsn_from_hex import pytest @@ -56,7 +57,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder, INSERT INTO t1 VALUES ({data_id}, '{data_secret}'); ''') cur.execute("SELECT pg_current_wal_flush_lsn()") - current_lsn = int(cur.fetchone()[0].split('/')[1], base=16) + current_lsn = lsn_from_hex(cur.fetchone()[0]) # wait until pageserver receives that data wait_for_last_record_lsn(client, UUID(tenant_id), UUID(timeline_id), current_lsn) diff --git a/test_runner/batch_others/test_tenant_relocation.py b/test_runner/batch_others/test_tenant_relocation.py index e4492e5393..12ce3eb760 100644 --- a/test_runner/batch_others/test_tenant_relocation.py +++ b/test_runner/batch_others/test_tenant_relocation.py @@ -11,6 +11,7 @@ import signal import pytest from fixtures.zenith_fixtures import PgProtocol, PortDistributor, Postgres, ZenithEnvBuilder, ZenithPageserverHttpClient, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload, zenith_binpath, pg_distrib_dir +from fixtures.utils import lsn_from_hex def assert_abs_margin_ratio(a: float, b: float, margin_ratio: float): @@ -134,7 +135,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, assert cur.fetchone() == (500500, ) cur.execute("SELECT pg_current_wal_flush_lsn()") - current_lsn = int(cur.fetchone()[0].split('/')[1], base=16) + current_lsn = lsn_from_hex(cur.fetchone()[0]) pageserver_http = env.pageserver.http_client() @@ -189,8 +190,8 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder, # when load is active these checks can break because lsns are not static # so lets check with some margin - assert_abs_margin_ratio(new_timeline_detail['local']['disk_consistent_lsn'], - timeline_detail['local']['disk_consistent_lsn'], + assert_abs_margin_ratio(lsn_from_hex(new_timeline_detail['local']['disk_consistent_lsn']), + lsn_from_hex(timeline_detail['local']['disk_consistent_lsn']), 0.03) # callmemaybe to start replication from safekeeper to the new pageserver diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index c44a6e431f..fa68c4f476 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -33,7 +33,7 @@ from typing_extensions import Literal import requests import backoff # type: ignore -from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture) +from .utils import (get_self_dir, lsn_from_hex, mkdir_if_needed, subprocess_capture) from fixtures.log_helper import log """ This file contains pytest fixtures. A fixture is a test resource that can be @@ -1900,8 +1900,10 @@ def remote_consistent_lsn(pageserver_http_client: ZenithPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID) -> int: detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) - assert isinstance(detail['remote']['remote_consistent_lsn'], int) - return detail['remote']['remote_consistent_lsn'] + + lsn_str = detail['remote']['remote_consistent_lsn'] + assert isinstance(lsn_str, str) + return lsn_from_hex(lsn_str) def wait_for_upload(pageserver_http_client: ZenithPageserverHttpClient, @@ -1917,8 +1919,10 @@ def last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient, tenant: uuid.UUID, timeline: uuid.UUID) -> int: detail = pageserver_http_client.timeline_detail_v2(tenant, timeline) - assert isinstance(detail['local']['last_record_lsn'], int) - return detail['local']['last_record_lsn'] + + lsn_str = detail['local']['last_record_lsn'] + assert isinstance(lsn_str, str) + return lsn_from_hex(lsn_str) def wait_for_last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient, diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 389c394103..f5d4184e63 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -316,7 +316,7 @@ fn print_timelines_tree( timeline.local.as_ref().and_then(|l| l.ancestor_timeline_id) { timelines_hash - .get_mut(&ZTimelineId::from(ancestor_timeline_id)) + .get_mut(&ancestor_timeline_id) .context("missing timeline info in the HashMap")? .children .insert(timeline.timeline_id); @@ -437,8 +437,8 @@ fn get_timeline_infos( fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::Result { if let Some(tenant_id_from_arguments) = parse_tenant_id(sub_match).transpose() { tenant_id_from_arguments - } else if let Some(tenantid_conf) = env.default_tenant_id { - Ok(ZTenantId::from(tenantid_conf)) + } else if let Some(default_id) = env.default_tenant_id { + Ok(default_id) } else { bail!("No tenant id. Use --tenant-id, or set 'default_tenant_id' in the config file"); } @@ -479,7 +479,7 @@ fn handle_init(init_match: &ArgMatches) -> Result { .context("Failed to initialize zenith repository")?; // default_tenantid was generated by the `env.init()` call above - let initial_tenant_id = ZTenantId::from(env.default_tenant_id.unwrap()); + let initial_tenant_id = env.default_tenant_id.unwrap(); // Call 'pageserver init'. let pageserver = PageServerNode::from_env(&env); diff --git a/zenith_utils/Cargo.toml b/zenith_utils/Cargo.toml index daaf345f8f..8e7f5f233c 100644 --- a/zenith_utils/Cargo.toml +++ b/zenith_utils/Cargo.toml @@ -27,6 +27,7 @@ hex = { version = "0.4.3", features = ["serde"] } rustls = "0.19.1" rustls-split = "0.2.1" git-version = "0.3.5" +serde_with = "1.12.0" zenith_metrics = { path = "../zenith_metrics" } workspace_hack = { path = "../workspace_hack" } diff --git a/zenith_utils/src/auth.rs b/zenith_utils/src/auth.rs index cbc4fcee61..8271121c63 100644 --- a/zenith_utils/src/auth.rs +++ b/zenith_utils/src/auth.rs @@ -14,8 +14,9 @@ use jsonwebtoken::{ decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, }; use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, DisplayFromStr}; -use crate::zid::{HexZTenantId, ZTenantId}; +use crate::zid::ZTenantId; const JWT_ALGORITHM: Algorithm = Algorithm::RS256; @@ -26,18 +27,18 @@ pub enum Scope { PageServerApi, } +#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct Claims { - pub tenant_id: Option, + #[serde(default)] + #[serde_as(as = "Option")] + pub tenant_id: Option, pub scope: Scope, } impl Claims { pub fn new(tenant_id: Option, scope: Scope) -> Self { - Self { - tenant_id: tenant_id.map(HexZTenantId::from), - scope, - } + Self { tenant_id, scope } } } @@ -47,7 +48,7 @@ pub fn check_permission(claims: &Claims, tenantid: Option) -> Result< bail!("Attempt to access management api with tenant scope. Permission denied") } (Scope::Tenant, Some(tenantid)) => { - if ZTenantId::from(claims.tenant_id.unwrap()) != tenantid { + if claims.tenant_id.unwrap() != tenantid { bail!("Tenant id mismatch. Permission denied") } Ok(()) diff --git a/zenith_utils/src/zid.rs b/zenith_utils/src/zid.rs index e047e38da7..fce5ed97c1 100644 --- a/zenith_utils/src/zid.rs +++ b/zenith_utils/src/zid.rs @@ -2,100 +2,19 @@ use std::{fmt, str::FromStr}; use hex::FromHex; use rand::Rng; -use serde::{ - de::{self, Visitor}, - Deserialize, Serialize, -}; - -macro_rules! mutual_from { - ($id1:ident, $id2:ident) => { - impl From<$id1> for $id2 { - fn from(id1: $id1) -> Self { - Self(id1.0.into()) - } - } - - impl From<$id2> for $id1 { - fn from(id2: $id2) -> Self { - Self(id2.0.into()) - } - } - }; -} +use serde::{Deserialize, Serialize}; /// Zenith ID is a 128-bit random ID. /// Used to represent various identifiers. Provides handy utility methods and impls. /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// Use [`HexZId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// +/// Use `#[serde_as(as = "DisplayFromStr")]` to (de)serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// Check the `serde_with::serde_as` documentation for options for more complex types. #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] struct ZId([u8; 16]); -/// [`ZId`] version that serializes and deserializes as a hex string. -/// Useful for various json serializations, where hex byte array from original id is not convenient. -/// -/// Plain `ZId` could be (de)serialized into hex string with `#[serde(with = "hex")]` attribute. -/// This however won't work on nested types like `Option` or `Vec`, see https://github.com/serde-rs/serde/issues/723 for the details. -/// Every separate type currently needs a new (de)serializing method for every type separately. -/// -/// To provide a generic way to serialize the ZId as a hex string where `#[serde(with = "hex")]` is not enough, this wrapper is created. -/// The default wrapper serialization is left unchanged due to -/// * byte array (de)serialization being faster and simpler -/// * byte deserialization being used in Safekeeper already, with those bytes coming from compute (see `ProposerGreeting` in safekeeper) -/// * current `HexZId`'s deserialization impl breaks on compute byte array deserialization, having it by default is dangerous -#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -struct HexZId([u8; 16]); - -impl Serialize for HexZId { - fn serialize(&self, ser: S) -> Result - where - S: serde::Serializer, - { - hex::encode(self.0).serialize(ser) - } -} - -impl<'de> Deserialize<'de> for HexZId { - fn deserialize(de: D) -> Result - where - D: serde::Deserializer<'de>, - { - de.deserialize_bytes(HexVisitor) - } -} - -struct HexVisitor; - -impl<'de> Visitor<'de> for HexVisitor { - type Value = HexZId; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "A hexadecimal representation of a 128-bit random Zenith ID" - ) - } - - fn visit_bytes(self, hex_bytes: &[u8]) -> Result - where - E: de::Error, - { - ZId::from_hex(hex_bytes) - .map(HexZId::from) - .map_err(de::Error::custom) - } - - fn visit_str(self, hex_bytes_str: &str) -> Result - where - E: de::Error, - { - Self::visit_bytes(self, hex_bytes_str.as_bytes()) - } -} - -mutual_from!(ZId, HexZId); - impl ZId { pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZId { let mut arr = [0u8; 16]; @@ -256,76 +175,22 @@ macro_rules! zid_newtype { /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// Use [`HexZTimelineId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// See [`ZId`] for alternative ways to serialize it. #[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] pub struct ZTimelineId(ZId); -/// A [`ZTimelineId`] version that gets (de)serialized as a hex string. -/// Use in complex types, where `#[serde(with = "hex")]` does not work. -/// See [`HexZId`] for more details. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] -pub struct HexZTimelineId(HexZId); - -impl std::fmt::Debug for HexZTimelineId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTimelineId::from(*self).fmt(f) - } -} - -impl std::fmt::Display for HexZTimelineId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTimelineId::from(*self).fmt(f) - } -} - -impl FromStr for HexZTimelineId { - type Err = ::Err; - - fn from_str(s: &str) -> Result { - Ok(HexZTimelineId::from(ZTimelineId::from_str(s)?)) - } -} - zid_newtype!(ZTimelineId); -mutual_from!(ZTimelineId, HexZTimelineId); /// Zenith Tenant Id represents identifiar of a particular tenant. /// Is used for distinguishing requests and data belonging to different users. /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// Use [`HexZTenantId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. +/// See [`ZId`] for alternative ways to serialize it. #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct ZTenantId(ZId); -/// A [`ZTenantId`] version that gets (de)serialized as a hex string. -/// Use in complex types, where `#[serde(with = "hex")]` does not work. -/// See [`HexZId`] for more details. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)] -pub struct HexZTenantId(HexZId); - -impl std::fmt::Debug for HexZTenantId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTenantId::from(*self).fmt(f) - } -} - -impl std::fmt::Display for HexZTenantId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ZTenantId::from(*self).fmt(f) - } -} - -impl FromStr for HexZTenantId { - type Err = ::Err; - - fn from_str(s: &str) -> Result { - Ok(HexZTenantId::from(ZTenantId::from_str(s)?)) - } -} - zid_newtype!(ZTenantId); -mutual_from!(ZTenantId, HexZTenantId); // A pair uniquely identifying Zenith instance. #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)] @@ -368,55 +233,3 @@ impl fmt::Display for ZNodeId { write!(f, "{}", self.0) } } - -#[cfg(test)] -mod tests { - use std::fmt::Display; - - use super::*; - use hex::FromHexError; - use hex_literal::hex; - - #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] - struct TestStruct + Display> { - field: Option, - } - - #[test] - fn test_hex_serializations_tenant_id() { - let original_struct = TestStruct { - field: Some(HexZTenantId::from(ZTenantId::from_array(hex!( - "11223344556677881122334455667788" - )))), - }; - - let serialized_string = serde_json::to_string(&original_struct).unwrap(); - assert_eq!( - serialized_string, - r#"{"field":"11223344556677881122334455667788"}"# - ); - - let deserialized_struct: TestStruct = - serde_json::from_str(&serialized_string).unwrap(); - assert_eq!(original_struct, deserialized_struct); - } - - #[test] - fn test_hex_serializations_timeline_id() { - let original_struct = TestStruct { - field: Some(HexZTimelineId::from(ZTimelineId::from_array(hex!( - "AA223344556677881122334455667788" - )))), - }; - - let serialized_string = serde_json::to_string(&original_struct).unwrap(); - assert_eq!( - serialized_string, - r#"{"field":"aa223344556677881122334455667788"}"# - ); - - let deserialized_struct: TestStruct = - serde_json::from_str(&serialized_string).unwrap(); - assert_eq!(original_struct, deserialized_struct); - } -}