Use serde_with to (de)serialize ZId and Lsn to hex

2026-06-04 22:10:39 +00:00 · 2022-03-19 02:38:29 +02:00
parent 3b069f5aef
commit 063f9ba81d
16 changed files with 192 additions and 289 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -441,6 +441,7 @@ dependencies = [
 "regex",
 "reqwest",
 "serde",
+ "serde_with",
 "tar",
 "thiserror",
 "toml",
@@ -600,6 +601,41 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "darling"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim 0.10.0",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "digest"
 version = "0.9.0"
@@ -1038,6 +1074,12 @@ dependencies = [
 "tokio-rustls 0.23.2",
 ]

+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
 [[package]]
 name = "idna"
 version = "0.2.3"
@@ -1422,7 +1464,6 @@ dependencies = [
 "daemonize",
 "fail",
 "futures",
- "hex",
 "hex-literal",
 "humantime",
 "hyper",
@@ -1440,6 +1481,7 @@ dependencies = [
 "scopeguard",
 "serde",
 "serde_json",
+ "serde_with",
 "signal-hook",
 "tar",
 "tempfile",
@@ -2075,6 +2117,12 @@ dependencies = [
 "rustls 0.19.1",
 ]

+[[package]]
+name = "rustversion"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f"
+
 [[package]]
 name = "ryu"
 version = "1.0.9"
@@ -2187,6 +2235,29 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "serde_with"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec1e6ec4d8950e5b1e894eac0d360742f3b1407a6078a604a731c4b3f49cefbc"
+dependencies = [
+ "rustversion",
+ "serde",
+ "serde_with_macros",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "sha2"
 version = "0.9.9"
@@ -3056,6 +3127,7 @@ dependencies = [
 "rustls-split",
 "serde",
 "serde_json",
+ "serde_with",
 "signal-hook",
 "tempfile",
 "thiserror",
--- a/control_plane/Cargo.toml
+++ b/control_plane/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 tar = "0.4.33"
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 serde = { version = "1.0", features = ["derive"] }
+serde_with = "1.12.0"
 toml = "0.5"
 lazy_static = "1.4"
 regex = "1"
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -5,6 +5,7 @@

 use anyhow::{bail, ensure, Context};
 use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
 use std::collections::HashMap;
 use std::env;
 use std::fs;
@@ -12,9 +13,7 @@ use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use zenith_utils::auth::{encode_from_key_file, Claims, Scope};
 use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{
-    HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId,
-};
+use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId};

 use crate::safekeeper::SafekeeperNode;

@@ -25,6 +24,7 @@ use crate::safekeeper::SafekeeperNode;
 // to 'zenith init --config=<path>' option. See control_plane/simple.conf for
 // an example.
 //
+#[serde_as]
 #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
 pub struct LocalEnv {
    // Base directory for all the nodes (the pageserver, safekeepers and
@@ -50,7 +50,8 @@ pub struct LocalEnv {
    // Default tenant ID to use with the 'zenith' command line utility, when
    // --tenantid is not explicitly specified.
    #[serde(default)]
-    pub default_tenant_id: Option<HexZTenantId>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub default_tenant_id: Option<ZTenantId>,

    // used to issue tokens during e.g pg start
    #[serde(default)]
@@ -66,7 +67,8 @@ pub struct LocalEnv {
    // A `HashMap<String, HashMap<ZTenantId, ZTimelineId>>` would be more appropriate here,
    // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error.
    // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table".
-    branch_name_mappings: HashMap<String, Vec<(HexZTenantId, HexZTimelineId)>>,
+    #[serde_as(as = "HashMap<_, Vec<(DisplayFromStr, DisplayFromStr)>>")]
+    branch_name_mappings: HashMap<String, Vec<(ZTenantId, ZTimelineId)>>,
 }

 #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
@@ -164,9 +166,6 @@ impl LocalEnv {
            .entry(branch_name.clone())
            .or_default();

-        let tenant_id = HexZTenantId::from(tenant_id);
-        let timeline_id = HexZTimelineId::from(timeline_id);
-
        let existing_ids = existing_values
            .iter()
            .find(|(existing_tenant_id, _)| existing_tenant_id == &tenant_id);
@@ -193,7 +192,6 @@ impl LocalEnv {
        branch_name: &str,
        tenant_id: ZTenantId,
    ) -> Option<ZTimelineId> {
-        let tenant_id = HexZTenantId::from(tenant_id);
        self.branch_name_mappings
            .get(branch_name)?
            .iter()
@@ -207,13 +205,7 @@ impl LocalEnv {
            .iter()
            .flat_map(|(name, tenant_timelines)| {
                tenant_timelines.iter().map(|&(tenant_id, timeline_id)| {
-                    (
-                        ZTenantTimelineId::new(
-                            ZTenantId::from(tenant_id),
-                            ZTimelineId::from(timeline_id),
-                        ),
-                        name.clone(),
-                    )
+                    (ZTenantTimelineId::new(tenant_id, timeline_id), name.clone())
                })
            })
            .collect()
@@ -259,7 +251,7 @@ impl LocalEnv {

        // If no initial tenant ID was given, generate it.
        if env.default_tenant_id.is_none() {
-            env.default_tenant_id = Some(HexZTenantId::from(ZTenantId::generate()));
+            env.default_tenant_id = Some(ZTenantId::generate());
        }

        env.base_data_dir = base_path();
--- a/control_plane/src/storage.rs
+++ b/control_plane/src/storage.rs
@@ -18,7 +18,7 @@ use thiserror::Error;
 use zenith_utils::http::error::HttpErrorBody;
 use zenith_utils::lsn::Lsn;
 use zenith_utils::postgres_backend::AuthType;
-use zenith_utils::zid::{HexZTenantId, HexZTimelineId, ZTenantId, ZTimelineId};
+use zenith_utils::zid::{ZTenantId, ZTimelineId};

 use crate::local_env::LocalEnv;
 use crate::{fill_rust_env_vars, read_pidfile};
@@ -337,9 +337,7 @@ impl PageServerNode {
    ) -> anyhow::Result<Option<ZTenantId>> {
        let tenant_id_string = self
            .http_request(Method::POST, format!("{}/tenant", self.http_base_url))
-            .json(&TenantCreateRequest {
-                new_tenant_id: new_tenant_id.map(HexZTenantId::from),
-            })
+            .json(&TenantCreateRequest { new_tenant_id })
            .send()?
            .error_from_body()?
            .json::<Option<String>>()?;
@@ -382,9 +380,9 @@ impl PageServerNode {
                format!("{}/tenant/{}/timeline", self.http_base_url, tenant_id),
            )
            .json(&TimelineCreateRequest {
-                new_timeline_id: new_timeline_id.map(HexZTimelineId::from),
+                new_timeline_id,
                ancestor_start_lsn,
-                ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from),
+                ancestor_timeline_id,
            })
            .send()?
            .error_from_body()?
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -25,11 +25,12 @@ tokio-stream = "0.1.8"
 anyhow = { version = "1.0", features = ["backtrace"] }
 crc32c = "0.6.0"
 thiserror = "1.0"
-hex = { version = "0.4.3", features = ["serde"] }
 tar = "0.4.33"
 humantime = "2.1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
+serde_with = "1.12.0"
+
 toml_edit = { version = "0.13", features = ["easy"] }
 scopeguard = "1.1.0"
 async-trait = "0.1"
--- a/pageserver/src/http/models.rs
+++ b/pageserver/src/http/models.rs
@@ -1,24 +1,39 @@
-use anyhow::Context;
 use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
 use zenith_utils::{
    lsn::Lsn,
-    zid::{HexZTenantId, HexZTimelineId, ZNodeId, ZTenantId, ZTimelineId},
+    zid::{ZNodeId, ZTenantId, ZTimelineId},
 };

 use crate::timelines::{LocalTimelineInfo, TimelineInfo};

+#[serde_as]
 #[derive(Serialize, Deserialize)]
 pub struct TimelineCreateRequest {
-    pub new_timeline_id: Option<HexZTimelineId>,
-    pub ancestor_timeline_id: Option<HexZTimelineId>,
+    #[serde(default)]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub new_timeline_id: Option<ZTimelineId>,
+    #[serde(default)]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub ancestor_timeline_id: Option<ZTimelineId>,
+    #[serde(default)]
+    #[serde_as(as = "Option<DisplayFromStr>")]
    pub ancestor_start_lsn: Option<Lsn>,
 }

+#[serde_as]
 #[derive(Serialize, Deserialize)]
 pub struct TenantCreateRequest {
-    pub new_tenant_id: Option<HexZTenantId>,
+    #[serde(default)]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub new_tenant_id: Option<ZTenantId>,
 }

+#[serde_as]
+#[derive(Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub ZTenantId);
+
 #[derive(Clone)]
 pub enum TimelineInfoV1 {
    Local {
@@ -39,18 +54,24 @@ pub enum TimelineInfoV1 {
    },
 }

+#[serde_as]
 #[derive(Serialize, Deserialize)]
 pub struct TimelineInfoResponseV1 {
    pub kind: String,
-    #[serde(with = "hex")]
+    #[serde_as(as = "DisplayFromStr")]
    timeline_id: ZTimelineId,
-    #[serde(with = "hex")]
+    #[serde_as(as = "DisplayFromStr")]
    tenant_id: ZTenantId,
-    disk_consistent_lsn: String,
-    last_record_lsn: Option<String>,
-    prev_record_lsn: Option<String>,
-    ancestor_timeline_id: Option<HexZTimelineId>,
-    ancestor_lsn: Option<String>,
+    #[serde_as(as = "DisplayFromStr")]
+    disk_consistent_lsn: Lsn,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    last_record_lsn: Option<Lsn>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    prev_record_lsn: Option<Lsn>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    ancestor_timeline_id: Option<ZTimelineId>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    ancestor_lsn: Option<Lsn>,
    current_logical_size: Option<usize>,
    current_logical_size_non_incremental: Option<usize>,
 }
@@ -72,11 +93,11 @@ impl From<TimelineInfoV1> for TimelineInfoResponseV1 {
                kind: "Local".to_owned(),
                timeline_id,
                tenant_id,
-                disk_consistent_lsn: disk_consistent_lsn.to_string(),
-                last_record_lsn: Some(last_record_lsn.to_string()),
-                prev_record_lsn: prev_record_lsn.map(|lsn| lsn.to_string()),
-                ancestor_timeline_id: ancestor_timeline_id.map(HexZTimelineId::from),
-                ancestor_lsn: ancestor_lsn.map(|lsn| lsn.to_string()),
+                disk_consistent_lsn,
+                last_record_lsn: Some(last_record_lsn),
+                prev_record_lsn,
+                ancestor_timeline_id,
+                ancestor_lsn,
                current_logical_size,
                current_logical_size_non_incremental,
            },
@@ -88,7 +109,7 @@ impl From<TimelineInfoV1> for TimelineInfoResponseV1 {
                kind: "Remote".to_owned(),
                timeline_id,
                tenant_id,
-                disk_consistent_lsn: disk_consistent_lsn.to_string(),
+                disk_consistent_lsn,
                last_record_lsn: None,
                prev_record_lsn: None,
                ancestor_timeline_id: None,
@@ -104,37 +125,24 @@ impl TryFrom<TimelineInfoResponseV1> for TimelineInfoV1 {
    type Error = anyhow::Error;

    fn try_from(other: TimelineInfoResponseV1) -> anyhow::Result<Self> {
-        let parse_lsn_hex_string = |lsn_string: String| {
-            lsn_string
-                .parse::<Lsn>()
-                .with_context(|| format!("Failed to parse Lsn as hex string from '{}'", lsn_string))
-        };
-
-        let disk_consistent_lsn = parse_lsn_hex_string(other.disk_consistent_lsn)?;
        Ok(match other.kind.as_str() {
            "Local" => TimelineInfoV1::Local {
                timeline_id: other.timeline_id,
                tenant_id: other.tenant_id,
-                last_record_lsn: other
-                    .last_record_lsn
-                    .ok_or(anyhow::anyhow!(
-                        "Local timeline should have last_record_lsn"
-                    ))
-                    .and_then(parse_lsn_hex_string)?,
-                prev_record_lsn: other
-                    .prev_record_lsn
-                    .map(parse_lsn_hex_string)
-                    .transpose()?,
+                last_record_lsn: other.last_record_lsn.ok_or(anyhow::anyhow!(
+                    "Local timeline should have last_record_lsn"
+                ))?,
+                prev_record_lsn: other.prev_record_lsn,
                ancestor_timeline_id: other.ancestor_timeline_id.map(ZTimelineId::from),
-                ancestor_lsn: other.ancestor_lsn.map(parse_lsn_hex_string).transpose()?,
-                disk_consistent_lsn,
+                ancestor_lsn: other.ancestor_lsn,
+                disk_consistent_lsn: other.disk_consistent_lsn,
                current_logical_size: other.current_logical_size,
                current_logical_size_non_incremental: other.current_logical_size_non_incremental,
            },
            "Remote" => TimelineInfoV1::Remote {
                timeline_id: other.timeline_id,
                tenant_id: other.tenant_id,
-                disk_consistent_lsn,
+                disk_consistent_lsn: other.disk_consistent_lsn,
            },
            unknown => anyhow::bail!("Unknown timeline kind: {}", unknown),
        })
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -17,11 +17,11 @@ use zenith_utils::http::{
    request::parse_request_param,
 };
 use zenith_utils::http::{RequestExt, RouterBuilder};
-use zenith_utils::zid::{HexZTenantId, ZTenantTimelineId, ZTimelineId};
+use zenith_utils::zid::{ZTenantTimelineId, ZTimelineId};

 use super::models::{
-    StatusResponse, TenantCreateRequest, TimelineCreateRequest, TimelineInfoResponseV1,
-    TimelineInfoV1,
+    StatusResponse, TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest,
+    TimelineInfoResponseV1, TimelineInfoV1,
 };
 use crate::remote_storage::{schedule_timeline_download, RemoteTimelineIndex};
 use crate::timelines::{
@@ -308,7 +308,7 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
    .map_err(ApiError::from_err)??;

    Ok(match new_tenant_id {
-        Some(id) => json_response(StatusCode::CREATED, HexZTenantId::from(id))?,
+        Some(id) => json_response(StatusCode::CREATED, TenantCreateResponse(id))?,
        None => json_response(StatusCode::CONFLICT, ())?,
    })
 }
--- a/pageserver/src/tenant_mgr.rs
+++ b/pageserver/src/tenant_mgr.rs
@@ -15,6 +15,7 @@ use anyhow::{Context, Result};
 use lazy_static::lazy_static;
 use log::*;
 use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
 use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 use std::fmt;
@@ -267,9 +268,10 @@ pub fn get_timeline_for_tenant_load(
        .with_context(|| format!("Timeline {} not found for tenant {}", timelineid, tenantid))
 }

+#[serde_as]
 #[derive(Serialize, Deserialize, Clone)]
 pub struct TenantInfo {
-    #[serde(with = "hex")]
+    #[serde_as(as = "DisplayFromStr")]
    pub id: ZTenantId,
    pub state: TenantState,
 }
--- a/pageserver/src/timelines.rs
+++ b/pageserver/src/timelines.rs
@@ -5,6 +5,7 @@
 use anyhow::{bail, Context, Result};
 use postgres_ffi::ControlFileData;
 use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
 use std::{
    fs,
    path::Path,
@@ -13,9 +14,9 @@ use std::{
 };
 use tracing::*;

+use zenith_utils::lsn::Lsn;
 use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};
 use zenith_utils::{crashsafe_dir, logging};
-use zenith_utils::{lsn::Lsn, zid::HexZTimelineId};

 use crate::{
    config::PageServerConf,
@@ -28,12 +29,18 @@ use crate::{layered_repository::LayeredRepository, walredo::WalRedoManager};
 use crate::{repository::RepositoryTimeline, tenant_mgr};
 use crate::{repository::Timeline, CheckpointConfig};

+#[serde_as]
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct LocalTimelineInfo {
-    pub ancestor_timeline_id: Option<HexZTimelineId>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub ancestor_timeline_id: Option<ZTimelineId>,
+    #[serde_as(as = "Option<DisplayFromStr>")]
    pub ancestor_lsn: Option<Lsn>,
+    #[serde_as(as = "DisplayFromStr")]
    pub last_record_lsn: Lsn,
+    #[serde_as(as = "Option<DisplayFromStr>")]
    pub prev_record_lsn: Option<Lsn>,
+    #[serde_as(as = "DisplayFromStr")]
    pub disk_consistent_lsn: Lsn,
    pub current_logical_size: Option<usize>, // is None when timeline is Unloaded
    pub current_logical_size_non_incremental: Option<usize>,
@@ -47,9 +54,7 @@ impl LocalTimelineInfo {
    ) -> anyhow::Result<Self> {
        let last_record_lsn = timeline.get_last_record_lsn();
        let info = LocalTimelineInfo {
-            ancestor_timeline_id: timeline
-                .get_ancestor_timeline_id()
-                .map(HexZTimelineId::from),
+            ancestor_timeline_id: timeline.get_ancestor_timeline_id(),
            ancestor_lsn: {
                match timeline.get_ancestor_lsn() {
                    Lsn(0) => None,
@@ -72,7 +77,7 @@ impl LocalTimelineInfo {

    pub fn from_unloaded_timeline(metadata: &TimelineMetadata) -> Self {
        LocalTimelineInfo {
-            ancestor_timeline_id: metadata.ancestor_timeline().map(HexZTimelineId::from),
+            ancestor_timeline_id: metadata.ancestor_timeline(),
            ancestor_lsn: {
                match metadata.ancestor_lsn() {
                    Lsn(0) => None,
@@ -103,17 +108,20 @@ impl LocalTimelineInfo {
    }
 }

+#[serde_as]
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct RemoteTimelineInfo {
+    #[serde_as(as = "Option<DisplayFromStr>")]
    pub remote_consistent_lsn: Option<Lsn>,
    pub awaits_download: bool,
 }

+#[serde_as]
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct TimelineInfo {
-    #[serde(with = "hex")]
+    #[serde_as(as = "DisplayFromStr")]
    pub tenant_id: ZTenantId,
-    #[serde(with = "hex")]
+    #[serde_as(as = "DisplayFromStr")]
    pub timeline_id: ZTimelineId,
    pub local: Option<LocalTimelineInfo>,
    pub remote: Option<RemoteTimelineInfo>,
--- a/test_runner/batch_others/test_remote_storage.py
+++ b/test_runner/batch_others/test_remote_storage.py
@@ -7,6 +7,7 @@ from pathlib import Path
 from uuid import UUID
 from fixtures.zenith_fixtures import ZenithEnvBuilder, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload
 from fixtures.log_helper import log
+from fixtures.utils import lsn_from_hex
 import pytest


@@ -56,7 +57,7 @@ def test_remote_storage_backup_and_restore(zenith_env_builder: ZenithEnvBuilder,
                INSERT INTO t1 VALUES ({data_id}, '{data_secret}');
            ''')
            cur.execute("SELECT pg_current_wal_flush_lsn()")
-            current_lsn = int(cur.fetchone()[0].split('/')[1], base=16)
+            current_lsn = lsn_from_hex(cur.fetchone()[0])

    # wait until pageserver receives that data
    wait_for_last_record_lsn(client, UUID(tenant_id), UUID(timeline_id), current_lsn)
--- a/test_runner/batch_others/test_tenant_relocation.py
+++ b/test_runner/batch_others/test_tenant_relocation.py
@@ -11,6 +11,7 @@ import signal
 import pytest

 from fixtures.zenith_fixtures import PgProtocol, PortDistributor, Postgres, ZenithEnvBuilder, ZenithPageserverHttpClient, assert_local, wait_for, wait_for_last_record_lsn, wait_for_upload, zenith_binpath, pg_distrib_dir
+from fixtures.utils import lsn_from_hex


 def assert_abs_margin_ratio(a: float, b: float, margin_ratio: float):
@@ -134,7 +135,7 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder,
            assert cur.fetchone() == (500500, )
            cur.execute("SELECT pg_current_wal_flush_lsn()")

-            current_lsn = int(cur.fetchone()[0].split('/')[1], base=16)
+            current_lsn = lsn_from_hex(cur.fetchone()[0])

    pageserver_http = env.pageserver.http_client()

@@ -189,8 +190,8 @@ def test_tenant_relocation(zenith_env_builder: ZenithEnvBuilder,

        # when load is active these checks can break because lsns are not static
        # so lets check with some margin
-        assert_abs_margin_ratio(new_timeline_detail['local']['disk_consistent_lsn'],
-                                timeline_detail['local']['disk_consistent_lsn'],
+        assert_abs_margin_ratio(lsn_from_hex(new_timeline_detail['local']['disk_consistent_lsn']),
+                                lsn_from_hex(timeline_detail['local']['disk_consistent_lsn']),
                                0.03)

        # callmemaybe to start replication from safekeeper to the new pageserver
--- a/test_runner/fixtures/zenith_fixtures.py
+++ b/test_runner/fixtures/zenith_fixtures.py
@@ -33,7 +33,7 @@ from typing_extensions import Literal
 import requests
 import backoff  # type: ignore

-from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture)
+from .utils import (get_self_dir, lsn_from_hex, mkdir_if_needed, subprocess_capture)
 from fixtures.log_helper import log
 """
 This file contains pytest fixtures. A fixture is a test resource that can be
@@ -1900,8 +1900,10 @@ def remote_consistent_lsn(pageserver_http_client: ZenithPageserverHttpClient,
                          tenant: uuid.UUID,
                          timeline: uuid.UUID) -> int:
    detail = pageserver_http_client.timeline_detail_v2(tenant, timeline)
-    assert isinstance(detail['remote']['remote_consistent_lsn'], int)
-    return detail['remote']['remote_consistent_lsn']
+
+    lsn_str = detail['remote']['remote_consistent_lsn']
+    assert isinstance(lsn_str, str)
+    return lsn_from_hex(lsn_str)


 def wait_for_upload(pageserver_http_client: ZenithPageserverHttpClient,
@@ -1917,8 +1919,10 @@ def last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient,
                    tenant: uuid.UUID,
                    timeline: uuid.UUID) -> int:
    detail = pageserver_http_client.timeline_detail_v2(tenant, timeline)
-    assert isinstance(detail['local']['last_record_lsn'], int)
-    return detail['local']['last_record_lsn']
+
+    lsn_str = detail['local']['last_record_lsn']
+    assert isinstance(lsn_str, str)
+    return lsn_from_hex(lsn_str)


 def wait_for_last_record_lsn(pageserver_http_client: ZenithPageserverHttpClient,
--- a/zenith/src/main.rs
+++ b/zenith/src/main.rs
@@ -316,7 +316,7 @@ fn print_timelines_tree(
            timeline.local.as_ref().and_then(|l| l.ancestor_timeline_id)
        {
            timelines_hash
-                .get_mut(&ZTimelineId::from(ancestor_timeline_id))
+                .get_mut(&ancestor_timeline_id)
                .context("missing timeline info in the HashMap")?
                .children
                .insert(timeline.timeline_id);
@@ -437,8 +437,8 @@ fn get_timeline_infos(
 fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::Result<ZTenantId> {
    if let Some(tenant_id_from_arguments) = parse_tenant_id(sub_match).transpose() {
        tenant_id_from_arguments
-    } else if let Some(tenantid_conf) = env.default_tenant_id {
-        Ok(ZTenantId::from(tenantid_conf))
+    } else if let Some(default_id) = env.default_tenant_id {
+        Ok(default_id)
    } else {
        bail!("No tenant id. Use --tenant-id, or set 'default_tenant_id' in the config file");
    }
@@ -479,7 +479,7 @@ fn handle_init(init_match: &ArgMatches) -> Result<LocalEnv> {
        .context("Failed to initialize zenith repository")?;

    // default_tenantid was generated by the `env.init()` call above
-    let initial_tenant_id = ZTenantId::from(env.default_tenant_id.unwrap());
+    let initial_tenant_id = env.default_tenant_id.unwrap();

    // Call 'pageserver init'.
    let pageserver = PageServerNode::from_env(&env);
--- a/zenith_utils/Cargo.toml
+++ b/zenith_utils/Cargo.toml
@@ -27,6 +27,7 @@ hex = { version = "0.4.3", features = ["serde"] }
 rustls = "0.19.1"
 rustls-split = "0.2.1"
 git-version = "0.3.5"
+serde_with = "1.12.0"

 zenith_metrics = { path = "../zenith_metrics" }
 workspace_hack = { path = "../workspace_hack" }
--- a/zenith_utils/src/auth.rs
+++ b/zenith_utils/src/auth.rs
@@ -14,8 +14,9 @@ use jsonwebtoken::{
    decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation,
 };
 use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};

-use crate::zid::{HexZTenantId, ZTenantId};
+use crate::zid::ZTenantId;

 const JWT_ALGORITHM: Algorithm = Algorithm::RS256;

@@ -26,18 +27,18 @@ pub enum Scope {
    PageServerApi,
 }

+#[serde_as]
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Claims {
-    pub tenant_id: Option<HexZTenantId>,
+    #[serde(default)]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    pub tenant_id: Option<ZTenantId>,
    pub scope: Scope,
 }

 impl Claims {
    pub fn new(tenant_id: Option<ZTenantId>, scope: Scope) -> Self {
-        Self {
-            tenant_id: tenant_id.map(HexZTenantId::from),
-            scope,
-        }
+        Self { tenant_id, scope }
    }
 }

@@ -47,7 +48,7 @@ pub fn check_permission(claims: &Claims, tenantid: Option<ZTenantId>) -> Result<
            bail!("Attempt to access management api with tenant scope. Permission denied")
        }
        (Scope::Tenant, Some(tenantid)) => {
-            if ZTenantId::from(claims.tenant_id.unwrap()) != tenantid {
+            if claims.tenant_id.unwrap() != tenantid {
                bail!("Tenant id mismatch. Permission denied")
            }
            Ok(())
--- a/zenith_utils/src/zid.rs
+++ b/zenith_utils/src/zid.rs
@@ -2,100 +2,19 @@ use std::{fmt, str::FromStr};

 use hex::FromHex;
 use rand::Rng;
-use serde::{
-    de::{self, Visitor},
-    Deserialize, Serialize,
-};
-
-macro_rules! mutual_from {
-    ($id1:ident, $id2:ident) => {
-        impl From<$id1> for $id2 {
-            fn from(id1: $id1) -> Self {
-                Self(id1.0.into())
-            }
-        }
-
-        impl From<$id2> for $id1 {
-            fn from(id2: $id2) -> Self {
-                Self(id2.0.into())
-            }
-        }
-    };
-}
+use serde::{Deserialize, Serialize};

 /// Zenith ID is a 128-bit random ID.
 /// Used to represent various identifiers. Provides handy utility methods and impls.
 ///
 /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look
 /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`.
-/// Use [`HexZId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`.
+///
+/// Use `#[serde_as(as = "DisplayFromStr")]` to (de)serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`.
+/// Check the `serde_with::serde_as` documentation for options for more complex types.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
 struct ZId([u8; 16]);

-/// [`ZId`] version that serializes and deserializes as a hex string.
-/// Useful for various json serializations, where hex byte array from original id is not convenient.
-///
-/// Plain `ZId` could be (de)serialized into hex string with `#[serde(with = "hex")]` attribute.
-/// This however won't work on nested types like `Option<ZId>` or `Vec<ZId>`, see https://github.com/serde-rs/serde/issues/723 for the details.
-/// Every separate type currently needs a new (de)serializing method for every type separately.
-///
-/// To provide a generic way to serialize the ZId as a hex string where `#[serde(with = "hex")]` is not enough, this wrapper is created.
-/// The default wrapper serialization is left unchanged due to
-/// * byte array (de)serialization being faster and simpler
-/// * byte deserialization being used in Safekeeper already, with those bytes coming from compute (see `ProposerGreeting` in safekeeper)
-/// * current `HexZId`'s deserialization impl breaks on compute byte array deserialization, having it by default is dangerous
-#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
-struct HexZId([u8; 16]);
-
-impl Serialize for HexZId {
-    fn serialize<S>(&self, ser: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        hex::encode(self.0).serialize(ser)
-    }
-}
-
-impl<'de> Deserialize<'de> for HexZId {
-    fn deserialize<D>(de: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        de.deserialize_bytes(HexVisitor)
-    }
-}
-
-struct HexVisitor;
-
-impl<'de> Visitor<'de> for HexVisitor {
-    type Value = HexZId;
-
-    fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "A hexadecimal representation of a 128-bit random Zenith ID"
-        )
-    }
-
-    fn visit_bytes<E>(self, hex_bytes: &[u8]) -> Result<Self::Value, E>
-    where
-        E: de::Error,
-    {
-        ZId::from_hex(hex_bytes)
-            .map(HexZId::from)
-            .map_err(de::Error::custom)
-    }
-
-    fn visit_str<E>(self, hex_bytes_str: &str) -> Result<Self::Value, E>
-    where
-        E: de::Error,
-    {
-        Self::visit_bytes(self, hex_bytes_str.as_bytes())
-    }
-}
-
-mutual_from!(ZId, HexZId);
-
 impl ZId {
    pub fn get_from_buf(buf: &mut dyn bytes::Buf) -> ZId {
        let mut arr = [0u8; 16];
@@ -256,76 +175,22 @@ macro_rules! zid_newtype {
 ///
 /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look
 /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`.
-/// Use [`HexZTimelineId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`.
+/// See [`ZId`] for alternative ways to serialize it.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)]
 pub struct ZTimelineId(ZId);

-/// A [`ZTimelineId`] version that gets (de)serialized as a hex string.
-/// Use in complex types, where `#[serde(with = "hex")]` does not work.
-/// See [`HexZId`] for more details.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)]
-pub struct HexZTimelineId(HexZId);
-
-impl std::fmt::Debug for HexZTimelineId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        ZTimelineId::from(*self).fmt(f)
-    }
-}
-
-impl std::fmt::Display for HexZTimelineId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        ZTimelineId::from(*self).fmt(f)
-    }
-}
-
-impl FromStr for HexZTimelineId {
-    type Err = <ZTimelineId as FromStr>::Err;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(HexZTimelineId::from(ZTimelineId::from_str(s)?))
-    }
-}
-
 zid_newtype!(ZTimelineId);
-mutual_from!(ZTimelineId, HexZTimelineId);

 /// Zenith Tenant Id represents identifiar of a particular tenant.
 /// Is used for distinguishing requests and data belonging to different users.
 ///
 /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look
 /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`.
-/// Use [`HexZTenantId`] to serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`.
+/// See [`ZId`] for alternative ways to serialize it.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)]
 pub struct ZTenantId(ZId);

-/// A [`ZTenantId`] version that gets (de)serialized as a hex string.
-/// Use in complex types, where `#[serde(with = "hex")]` does not work.
-/// See [`HexZId`] for more details.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)]
-pub struct HexZTenantId(HexZId);
-
-impl std::fmt::Debug for HexZTenantId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        ZTenantId::from(*self).fmt(f)
-    }
-}
-
-impl std::fmt::Display for HexZTenantId {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        ZTenantId::from(*self).fmt(f)
-    }
-}
-
-impl FromStr for HexZTenantId {
-    type Err = <ZTenantId as FromStr>::Err;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(HexZTenantId::from(ZTenantId::from_str(s)?))
-    }
-}
-
 zid_newtype!(ZTenantId);
-mutual_from!(ZTenantId, HexZTenantId);

 // A pair uniquely identifying Zenith instance.
 #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash)]
@@ -368,55 +233,3 @@ impl fmt::Display for ZNodeId {
        write!(f, "{}", self.0)
    }
 }
-
-#[cfg(test)]
-mod tests {
-    use std::fmt::Display;
-
-    use super::*;
-    use hex::FromHexError;
-    use hex_literal::hex;
-
-    #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
-    struct TestStruct<E: Display, T: FromStr<Err = E> + Display> {
-        field: Option<T>,
-    }
-
-    #[test]
-    fn test_hex_serializations_tenant_id() {
-        let original_struct = TestStruct {
-            field: Some(HexZTenantId::from(ZTenantId::from_array(hex!(
-                "11223344556677881122334455667788"
-            )))),
-        };
-
-        let serialized_string = serde_json::to_string(&original_struct).unwrap();
-        assert_eq!(
-            serialized_string,
-            r#"{"field":"11223344556677881122334455667788"}"#
-        );
-
-        let deserialized_struct: TestStruct<FromHexError, HexZTenantId> =
-            serde_json::from_str(&serialized_string).unwrap();
-        assert_eq!(original_struct, deserialized_struct);
-    }
-
-    #[test]
-    fn test_hex_serializations_timeline_id() {
-        let original_struct = TestStruct {
-            field: Some(HexZTimelineId::from(ZTimelineId::from_array(hex!(
-                "AA223344556677881122334455667788"
-            )))),
-        };
-
-        let serialized_string = serde_json::to_string(&original_struct).unwrap();
-        assert_eq!(
-            serialized_string,
-            r#"{"field":"aa223344556677881122334455667788"}"#
-        );
-
-        let deserialized_struct: TestStruct<FromHexError, HexZTimelineId> =
-            serde_json::from_str(&serialized_string).unwrap();
-        assert_eq!(original_struct, deserialized_struct);
-    }
-}