diff --git a/Cargo.lock b/Cargo.lock index 1cf8562787..508b56125d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,9 +225,6 @@ name = "cc" version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" -dependencies = [ - "jobserver", -] [[package]] name = "cexpr" @@ -368,8 +365,8 @@ dependencies = [ "thiserror", "toml", "url", + "utils", "workspace_hack", - "zenith_utils", ] [[package]] @@ -1137,15 +1134,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" -[[package]] -name = "jobserver" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.56" @@ -1272,6 +1260,17 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.1.0" +dependencies = [ + "lazy_static", + "libc", + "once_cell", + "prometheus", + "workspace_hack", +] + [[package]] name = "mime" version = "0.3.16" @@ -1514,6 +1513,7 @@ dependencies = [ "hyper", "itertools", "lazy_static", + "metrics", "nix", "once_cell", "postgres", @@ -1539,9 +1539,8 @@ dependencies = [ "toml_edit", "tracing", "url", + "utils", "workspace_hack", - "zenith_metrics", - "zenith_utils", ] [[package]] @@ -1744,8 +1743,8 @@ dependencies = [ "regex", "serde", "thiserror", + "utils", "workspace_hack", - "zenith_utils", ] [[package]] @@ -1853,6 +1852,7 @@ dependencies = [ "hyper", "lazy_static", "md5", + "metrics", "parking_lot", "pin-project-lite", "rand", @@ -1872,9 +1872,8 @@ dependencies = [ "tokio-postgres", "tokio-postgres-rustls", "tokio-rustls", + "utils", "workspace_hack", - "zenith_metrics", - "zenith_utils", ] [[package]] @@ -2267,6 +2266,7 @@ dependencies = [ "humantime", "hyper", "lazy_static", + "metrics", "postgres", "postgres-protocol", "postgres_ffi", @@ -2283,10 +2283,9 @@ dependencies = [ "tokio-util 0.7.0", "tracing", "url", + "utils", "walkdir", "workspace_hack", - "zenith_metrics", - "zenith_utils", ] [[package]] @@ -3063,6 +3062,43 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "byteorder", + "bytes", + "criterion", + "git-version", + "hex", + "hex-literal", + "hyper", + "jsonwebtoken", + "lazy_static", + "metrics", + "nix", + "pin-project-lite", + "postgres", + "postgres-protocol", + "rand", + "routerify", + "rustls", + "rustls-pemfile", + "rustls-split", + "serde", + "serde_json", + "serde_with", + "signal-hook", + "tempfile", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", + "workspace_hack", +] + [[package]] name = "valuable" version = "0.1.0" @@ -3272,7 +3308,6 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", - "cc", "chrono", "clap 2.34.0", "either", @@ -3331,56 +3366,8 @@ dependencies = [ "postgres_ffi", "safekeeper", "serde_json", + "utils", "workspace_hack", - "zenith_utils", -] - -[[package]] -name = "zenith_metrics" -version = "0.1.0" -dependencies = [ - "lazy_static", - "libc", - "once_cell", - "prometheus", - "workspace_hack", -] - -[[package]] -name = "zenith_utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "bincode", - "byteorder", - "bytes", - "criterion", - "git-version", - "hex", - "hex-literal", - "hyper", - "jsonwebtoken", - "lazy_static", - "nix", - "pin-project-lite", - "postgres", - "postgres-protocol", - "rand", - "routerify", - "rustls", - "rustls-pemfile", - "rustls-split", - "serde", - "serde_json", - "serde_with", - "signal-hook", - "tempfile", - "thiserror", - "tokio", - "tracing", - "tracing-subscriber", - "workspace_hack", - "zenith_metrics", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1405f26517..35c18ba237 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,13 +3,11 @@ members = [ "compute_tools", "control_plane", "pageserver", - "postgres_ffi", "proxy", "safekeeper", "workspace_hack", "zenith", - "zenith_metrics", - "zenith_utils", + "libs/*", ] [profile.release] diff --git a/compute_tools/src/bin/zenith_ctl.rs b/compute_tools/src/bin/zenith_ctl.rs index 372afbc633..a5dfb1c875 100644 --- a/compute_tools/src/bin/zenith_ctl.rs +++ b/compute_tools/src/bin/zenith_ctl.rs @@ -157,7 +157,7 @@ fn run_compute(state: &Arc>) -> Result { } fn main() -> Result<()> { - // TODO: re-use `zenith_utils::logging` later + // TODO: re-use `utils::logging` later init_logger(DEFAULT_LOG_LEVEL)?; // Env variable is set by `cargo` diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 80b6c00dd2..33d01f7556 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -19,5 +19,5 @@ reqwest = { version = "0.11", default-features = false, features = ["blocking", pageserver = { path = "../pageserver" } safekeeper = { path = "../safekeeper" } -zenith_utils = { path = "../zenith_utils" } +utils = { path = "../libs/utils" } workspace_hack = { version = "0.1", path = "../workspace_hack" } diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index c078c274cf..2549baca5d 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -11,11 +11,12 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{Context, Result}; -use zenith_utils::connstring::connection_host_port; -use zenith_utils::lsn::Lsn; -use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::ZTenantId; -use zenith_utils::zid::ZTimelineId; +use utils::{ + connstring::connection_host_port, + lsn::Lsn, + postgres_backend::AuthType, + zid::{ZTenantId, ZTimelineId}, +}; use crate::local_env::LocalEnv; use crate::postgresql_conf::PostgresConf; diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 2bdc76e876..12ee88cdc9 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -11,9 +11,11 @@ use std::env; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use zenith_utils::auth::{encode_from_key_file, Claims, Scope}; -use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}; +use utils::{ + auth::{encode_from_key_file, Claims, Scope}, + postgres_backend::AuthType, + zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}, +}; use crate::safekeeper::SafekeeperNode; diff --git a/control_plane/src/safekeeper.rs b/control_plane/src/safekeeper.rs index 6f11a4e03d..b094016131 100644 --- a/control_plane/src/safekeeper.rs +++ b/control_plane/src/safekeeper.rs @@ -15,13 +15,15 @@ use reqwest::blocking::{Client, RequestBuilder, Response}; use reqwest::{IntoUrl, Method}; use safekeeper::http::models::TimelineCreateRequest; use thiserror::Error; -use zenith_utils::http::error::HttpErrorBody; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; +use utils::{ + connstring::connection_address, + http::error::HttpErrorBody, + zid::{ZNodeId, ZTenantId, ZTimelineId}, +}; use crate::local_env::{LocalEnv, SafekeeperConf}; use crate::storage::PageServerNode; use crate::{fill_rust_env_vars, read_pidfile}; -use zenith_utils::connstring::connection_address; #[derive(Error, Debug)] pub enum SafekeeperHttpError { diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index c49d5743a9..a01ffd30f6 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -15,15 +15,17 @@ use postgres::{Config, NoTls}; use reqwest::blocking::{Client, RequestBuilder, Response}; use reqwest::{IntoUrl, Method}; use thiserror::Error; -use zenith_utils::http::error::HttpErrorBody; -use zenith_utils::lsn::Lsn; -use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::{ + connstring::connection_address, + http::error::HttpErrorBody, + lsn::Lsn, + postgres_backend::AuthType, + zid::{ZTenantId, ZTimelineId}, +}; use crate::local_env::LocalEnv; use crate::{fill_rust_env_vars, read_pidfile}; use pageserver::tenant_mgr::TenantInfo; -use zenith_utils::connstring::connection_address; #[derive(Error, Debug)] pub enum PageserverHttpError { diff --git a/docs/README.md b/docs/README.md index a3fcd20bd2..99d635bb33 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ - [multitenancy.md](multitenancy.md) — how multitenancy is organized in the pageserver and Zenith CLI. - [sourcetree.md](sourcetree.md) — Overview of the source tree layeout. - [pageserver/README](/pageserver/README) — pageserver overview. -- [postgres_ffi/README](/postgres_ffi/README) — Postgres FFI overview. +- [postgres_ffi/README](/libs/postgres_ffi/README) — Postgres FFI overview. - [test_runner/README.md](/test_runner/README.md) — tests infrastructure overview. - [safekeeper/README](/safekeeper/README) — WAL service overview. - [core_changes.md](core_changes.md) - Description of Zenith changes in Postgres core diff --git a/docs/authentication.md b/docs/authentication.md index de408624ae..7200ffc62f 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -27,4 +27,4 @@ management_token = jwt.encode({"scope": "pageserverapi"}, auth_keys.priv, algori tenant_token = jwt.encode({"scope": "tenant", "tenant_id": ps.initial_tenant}, auth_keys.priv, algorithm="RS256") ``` -Utility functions to work with jwts in rust are located in zenith_utils/src/auth.rs +Utility functions to work with jwts in rust are located in libs/utils/src/auth.rs diff --git a/docs/sourcetree.md b/docs/sourcetree.md index b15294d67f..5fd5fe19e5 100644 --- a/docs/sourcetree.md +++ b/docs/sourcetree.md @@ -30,11 +30,6 @@ The pageserver has a few different duties: For more detailed info, see `/pageserver/README` -`/postgres_ffi`: - -Utility functions for interacting with PostgreSQL file formats. -Misc constants, copied from PostgreSQL headers. - `/proxy`: Postgres protocol proxy/router. @@ -74,14 +69,21 @@ We use [cargo-hakari](https://crates.io/crates/cargo-hakari) for automation. Main entry point for the 'zenith' CLI utility. TODO: Doesn't it belong to control_plane? -`/zenith_metrics`: +`/libs`: +Unites granular neon helper crates under the hood. +`/libs/postgres_ffi`: + +Utility functions for interacting with PostgreSQL file formats. +Misc constants, copied from PostgreSQL headers. + +`/libs/utils`: +Generic helpers that are shared between other crates in this repository. +A subject for future modularization. + +`/libs/metrics`: Helpers for exposing Prometheus metrics from the server. -`/zenith_utils`: - -Helpers that are shared between other crates in this repository. - ## Using Python Note that Debian/Ubuntu Python packages are stale, as it commonly happens, so manual installation of dependencies is not recommended. diff --git a/zenith_metrics/Cargo.toml b/libs/metrics/Cargo.toml similarity index 69% rename from zenith_metrics/Cargo.toml rename to libs/metrics/Cargo.toml index 906c5a1d64..3b6ff4691d 100644 --- a/zenith_metrics/Cargo.toml +++ b/libs/metrics/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "zenith_metrics" +name = "metrics" version = "0.1.0" edition = "2021" @@ -8,4 +8,4 @@ prometheus = {version = "0.13", default_features=false} # removes protobuf depen libc = "0.2" lazy_static = "1.4" once_cell = "1.8.0" -workspace_hack = { version = "0.1", path = "../workspace_hack" } +workspace_hack = { version = "0.1", path = "../../workspace_hack" } diff --git a/zenith_metrics/src/lib.rs b/libs/metrics/src/lib.rs similarity index 100% rename from zenith_metrics/src/lib.rs rename to libs/metrics/src/lib.rs diff --git a/zenith_metrics/src/wrappers.rs b/libs/metrics/src/wrappers.rs similarity index 96% rename from zenith_metrics/src/wrappers.rs rename to libs/metrics/src/wrappers.rs index 48202bc15e..de334add99 100644 --- a/zenith_metrics/src/wrappers.rs +++ b/libs/metrics/src/wrappers.rs @@ -8,8 +8,8 @@ use std::io::{Read, Result, Write}; /// /// ``` /// # use std::io::{Result, Read}; -/// # use zenith_metrics::{register_int_counter, IntCounter}; -/// # use zenith_metrics::CountedReader; +/// # use metrics::{register_int_counter, IntCounter}; +/// # use metrics::CountedReader; /// # /// # lazy_static::lazy_static! { /// # static ref INT_COUNTER: IntCounter = register_int_counter!( @@ -83,8 +83,8 @@ impl Read for CountedReader<'_, T> { /// /// ``` /// # use std::io::{Result, Write}; -/// # use zenith_metrics::{register_int_counter, IntCounter}; -/// # use zenith_metrics::CountedWriter; +/// # use metrics::{register_int_counter, IntCounter}; +/// # use metrics::CountedWriter; /// # /// # lazy_static::lazy_static! { /// # static ref INT_COUNTER: IntCounter = register_int_counter!( diff --git a/postgres_ffi/Cargo.toml b/libs/postgres_ffi/Cargo.toml similarity index 77% rename from postgres_ffi/Cargo.toml rename to libs/postgres_ffi/Cargo.toml index e8d471cb12..7be5ca1b93 100644 --- a/postgres_ffi/Cargo.toml +++ b/libs/postgres_ffi/Cargo.toml @@ -17,8 +17,8 @@ log = "0.4.14" memoffset = "0.6.2" thiserror = "1.0" serde = { version = "1.0", features = ["derive"] } -zenith_utils = { path = "../zenith_utils" } -workspace_hack = { version = "0.1", path = "../workspace_hack" } +utils = { path = "../utils" } +workspace_hack = { version = "0.1", path = "../../workspace_hack" } [build-dependencies] bindgen = "0.59.1" diff --git a/postgres_ffi/README b/libs/postgres_ffi/README similarity index 100% rename from postgres_ffi/README rename to libs/postgres_ffi/README diff --git a/postgres_ffi/build.rs b/libs/postgres_ffi/build.rs similarity index 96% rename from postgres_ffi/build.rs rename to libs/postgres_ffi/build.rs index 3b4b37f9ee..0043b9ab58 100644 --- a/postgres_ffi/build.rs +++ b/libs/postgres_ffi/build.rs @@ -88,8 +88,8 @@ fn main() { // 'pg_config --includedir-server' would perhaps be the more proper way to find it, // but this will do for now. // - .clang_arg("-I../tmp_install/include/server") - .clang_arg("-I../tmp_install/include/postgresql/server") + .clang_arg("-I../../tmp_install/include/server") + .clang_arg("-I../../tmp_install/include/postgresql/server") // // Finish the builder and generate the bindings. // diff --git a/postgres_ffi/pg_control_ffi.h b/libs/postgres_ffi/pg_control_ffi.h similarity index 100% rename from postgres_ffi/pg_control_ffi.h rename to libs/postgres_ffi/pg_control_ffi.h diff --git a/postgres_ffi/samples/pg_hba.conf b/libs/postgres_ffi/samples/pg_hba.conf similarity index 100% rename from postgres_ffi/samples/pg_hba.conf rename to libs/postgres_ffi/samples/pg_hba.conf diff --git a/postgres_ffi/src/controlfile_utils.rs b/libs/postgres_ffi/src/controlfile_utils.rs similarity index 97% rename from postgres_ffi/src/controlfile_utils.rs rename to libs/postgres_ffi/src/controlfile_utils.rs index b72c86c71c..4df2342b90 100644 --- a/postgres_ffi/src/controlfile_utils.rs +++ b/libs/postgres_ffi/src/controlfile_utils.rs @@ -43,7 +43,7 @@ impl ControlFileData { /// Interpret a slice of bytes as a Postgres control file. /// pub fn decode(buf: &[u8]) -> Result { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; // Check that the slice has the expected size. The control file is // padded with zeros up to a 512 byte sector size, so accept a @@ -77,7 +77,7 @@ impl ControlFileData { /// /// The CRC is recomputed to match the contents of the fields. pub fn encode(&self) -> Bytes { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; // Serialize into a new buffer. let b = self.ser().unwrap(); diff --git a/postgres_ffi/src/lib.rs b/libs/postgres_ffi/src/lib.rs similarity index 100% rename from postgres_ffi/src/lib.rs rename to libs/postgres_ffi/src/lib.rs diff --git a/postgres_ffi/src/nonrelfile_utils.rs b/libs/postgres_ffi/src/nonrelfile_utils.rs similarity index 100% rename from postgres_ffi/src/nonrelfile_utils.rs rename to libs/postgres_ffi/src/nonrelfile_utils.rs diff --git a/postgres_ffi/src/pg_constants.rs b/libs/postgres_ffi/src/pg_constants.rs similarity index 100% rename from postgres_ffi/src/pg_constants.rs rename to libs/postgres_ffi/src/pg_constants.rs diff --git a/postgres_ffi/src/relfile_utils.rs b/libs/postgres_ffi/src/relfile_utils.rs similarity index 100% rename from postgres_ffi/src/relfile_utils.rs rename to libs/postgres_ffi/src/relfile_utils.rs diff --git a/postgres_ffi/src/waldecoder.rs b/libs/postgres_ffi/src/waldecoder.rs similarity index 99% rename from postgres_ffi/src/waldecoder.rs rename to libs/postgres_ffi/src/waldecoder.rs index ce5aaf722d..9d1089ed46 100644 --- a/postgres_ffi/src/waldecoder.rs +++ b/libs/postgres_ffi/src/waldecoder.rs @@ -18,7 +18,7 @@ use crc32c::*; use log::*; use std::cmp::min; use thiserror::Error; -use zenith_utils::lsn::Lsn; +use utils::lsn::Lsn; pub struct WalStreamDecoder { lsn: Lsn, diff --git a/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs similarity index 98% rename from postgres_ffi/src/xlog_utils.rs rename to libs/postgres_ffi/src/xlog_utils.rs index 89fdbbf7ac..1645c44de5 100644 --- a/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -28,7 +28,7 @@ use std::io::prelude::*; use std::io::SeekFrom; use std::path::{Path, PathBuf}; use std::time::SystemTime; -use zenith_utils::lsn::Lsn; +use utils::lsn::Lsn; pub const XLOG_FNAME_LEN: usize = 24; pub const XLOG_BLCKSZ: usize = 8192; @@ -351,17 +351,17 @@ pub fn main() { impl XLogRecord { pub fn from_slice(buf: &[u8]) -> XLogRecord { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; XLogRecord::des(buf).unwrap() } pub fn from_bytes(buf: &mut B) -> XLogRecord { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; XLogRecord::des_from(&mut buf.reader()).unwrap() } pub fn encode(&self) -> Bytes { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; self.ser().unwrap().into() } @@ -373,19 +373,19 @@ impl XLogRecord { impl XLogPageHeaderData { pub fn from_bytes(buf: &mut B) -> XLogPageHeaderData { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; XLogPageHeaderData::des_from(&mut buf.reader()).unwrap() } } impl XLogLongPageHeaderData { pub fn from_bytes(buf: &mut B) -> XLogLongPageHeaderData { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; XLogLongPageHeaderData::des_from(&mut buf.reader()).unwrap() } pub fn encode(&self) -> Bytes { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; self.ser().unwrap().into() } } @@ -394,12 +394,12 @@ pub const SIZEOF_CHECKPOINT: usize = std::mem::size_of::(); impl CheckPoint { pub fn encode(&self) -> Bytes { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; self.ser().unwrap().into() } pub fn decode(buf: &[u8]) -> Result { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; Ok(CheckPoint::des(buf)?) } @@ -477,7 +477,9 @@ mod tests { #[test] pub fn test_find_end_of_wal() { // 1. Run initdb to generate some WAL - let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".."); + let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(".."); let data_dir = top_path.join("test_output/test_find_end_of_wal"); let initdb_path = top_path.join("tmp_install/bin/initdb"); let lib_path = top_path.join("tmp_install/lib"); diff --git a/zenith_utils/Cargo.toml b/libs/utils/Cargo.toml similarity index 88% rename from zenith_utils/Cargo.toml rename to libs/utils/Cargo.toml index dd83fa4a92..35eb443809 100644 --- a/zenith_utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "zenith_utils" +name = "utils" version = "0.1.0" edition = "2021" @@ -29,8 +29,8 @@ rustls-split = "0.3.0" git-version = "0.3.5" serde_with = "1.12.0" -zenith_metrics = { path = "../zenith_metrics" } -workspace_hack = { version = "0.1", path = "../workspace_hack" } +metrics = { path = "../metrics" } +workspace_hack = { version = "0.1", path = "../../workspace_hack" } [dev-dependencies] byteorder = "1.4.3" diff --git a/zenith_utils/benches/benchmarks.rs b/libs/utils/benches/benchmarks.rs similarity index 96% rename from zenith_utils/benches/benchmarks.rs rename to libs/utils/benches/benchmarks.rs index c945d5021c..0339939934 100644 --- a/zenith_utils/benches/benchmarks.rs +++ b/libs/utils/benches/benchmarks.rs @@ -1,7 +1,7 @@ #![allow(unused)] use criterion::{criterion_group, criterion_main, Criterion}; -use zenith_utils::zid; +use utils::zid; pub fn bench_zid_stringify(c: &mut Criterion) { // Can only use public methods. diff --git a/zenith_utils/build.rs b/libs/utils/build.rs similarity index 100% rename from zenith_utils/build.rs rename to libs/utils/build.rs diff --git a/zenith_utils/scripts/restore_from_wal.sh b/libs/utils/scripts/restore_from_wal.sh similarity index 100% rename from zenith_utils/scripts/restore_from_wal.sh rename to libs/utils/scripts/restore_from_wal.sh diff --git a/zenith_utils/scripts/restore_from_wal_archive.sh b/libs/utils/scripts/restore_from_wal_archive.sh similarity index 100% rename from zenith_utils/scripts/restore_from_wal_archive.sh rename to libs/utils/scripts/restore_from_wal_archive.sh diff --git a/zenith_utils/src/accum.rs b/libs/utils/src/accum.rs similarity index 96% rename from zenith_utils/src/accum.rs rename to libs/utils/src/accum.rs index d3ad61e514..0fb0190a92 100644 --- a/zenith_utils/src/accum.rs +++ b/libs/utils/src/accum.rs @@ -5,7 +5,7 @@ /// For example, to calculate the smallest value among some integers: /// /// ``` -/// use zenith_utils::accum::Accum; +/// use utils::accum::Accum; /// /// let values = [1, 2, 3]; /// diff --git a/zenith_utils/src/auth.rs b/libs/utils/src/auth.rs similarity index 100% rename from zenith_utils/src/auth.rs rename to libs/utils/src/auth.rs diff --git a/zenith_utils/src/bin_ser.rs b/libs/utils/src/bin_ser.rs similarity index 100% rename from zenith_utils/src/bin_ser.rs rename to libs/utils/src/bin_ser.rs diff --git a/zenith_utils/src/connstring.rs b/libs/utils/src/connstring.rs similarity index 100% rename from zenith_utils/src/connstring.rs rename to libs/utils/src/connstring.rs diff --git a/zenith_utils/src/crashsafe_dir.rs b/libs/utils/src/crashsafe_dir.rs similarity index 100% rename from zenith_utils/src/crashsafe_dir.rs rename to libs/utils/src/crashsafe_dir.rs diff --git a/zenith_utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs similarity index 97% rename from zenith_utils/src/http/endpoint.rs rename to libs/utils/src/http/endpoint.rs index 7669f18cd2..77acab496f 100644 --- a/zenith_utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -5,12 +5,11 @@ use anyhow::anyhow; use hyper::header::AUTHORIZATION; use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server}; use lazy_static::lazy_static; +use metrics::{new_common_metric_name, register_int_counter, Encoder, IntCounter, TextEncoder}; use routerify::ext::RequestExt; use routerify::RequestInfo; use routerify::{Middleware, Router, RouterBuilder, RouterService}; use tracing::info; -use zenith_metrics::{new_common_metric_name, register_int_counter, IntCounter}; -use zenith_metrics::{Encoder, TextEncoder}; use std::future::Future; use std::net::TcpListener; @@ -36,7 +35,7 @@ async fn prometheus_metrics_handler(_req: Request) -> Result anyhow::Result<()> { /// # Ok(()) diff --git a/zenith_utils/src/seqwait.rs b/libs/utils/src/seqwait.rs similarity index 100% rename from zenith_utils/src/seqwait.rs rename to libs/utils/src/seqwait.rs diff --git a/zenith_utils/src/seqwait_async.rs b/libs/utils/src/seqwait_async.rs similarity index 100% rename from zenith_utils/src/seqwait_async.rs rename to libs/utils/src/seqwait_async.rs diff --git a/zenith_utils/src/shutdown.rs b/libs/utils/src/shutdown.rs similarity index 100% rename from zenith_utils/src/shutdown.rs rename to libs/utils/src/shutdown.rs diff --git a/zenith_utils/src/signals.rs b/libs/utils/src/signals.rs similarity index 100% rename from zenith_utils/src/signals.rs rename to libs/utils/src/signals.rs diff --git a/zenith_utils/src/sock_split.rs b/libs/utils/src/sock_split.rs similarity index 100% rename from zenith_utils/src/sock_split.rs rename to libs/utils/src/sock_split.rs diff --git a/zenith_utils/src/sync.rs b/libs/utils/src/sync.rs similarity index 99% rename from zenith_utils/src/sync.rs rename to libs/utils/src/sync.rs index 5e61480bc3..48f0ff6384 100644 --- a/zenith_utils/src/sync.rs +++ b/libs/utils/src/sync.rs @@ -29,7 +29,7 @@ impl SyncFuture { /// Example: /// /// ``` - /// # use zenith_utils::sync::SyncFuture; + /// # use utils::sync::SyncFuture; /// # use std::future::Future; /// # use tokio::io::AsyncReadExt; /// # diff --git a/zenith_utils/src/tcp_listener.rs b/libs/utils/src/tcp_listener.rs similarity index 100% rename from zenith_utils/src/tcp_listener.rs rename to libs/utils/src/tcp_listener.rs diff --git a/zenith_utils/src/vec_map.rs b/libs/utils/src/vec_map.rs similarity index 100% rename from zenith_utils/src/vec_map.rs rename to libs/utils/src/vec_map.rs diff --git a/zenith_utils/src/zid.rs b/libs/utils/src/zid.rs similarity index 100% rename from zenith_utils/src/zid.rs rename to libs/utils/src/zid.rs diff --git a/zenith_utils/tests/bin_ser_test.rs b/libs/utils/tests/bin_ser_test.rs similarity index 96% rename from zenith_utils/tests/bin_ser_test.rs rename to libs/utils/tests/bin_ser_test.rs index ada43a1189..f357837a55 100644 --- a/zenith_utils/tests/bin_ser_test.rs +++ b/libs/utils/tests/bin_ser_test.rs @@ -2,7 +2,7 @@ use bytes::{Buf, BytesMut}; use hex_literal::hex; use serde::Deserialize; use std::io::Read; -use zenith_utils::bin_ser::LeSer; +use utils::bin_ser::LeSer; #[derive(Debug, PartialEq, Deserialize)] pub struct HeaderData { diff --git a/zenith_utils/tests/cert.pem b/libs/utils/tests/cert.pem similarity index 100% rename from zenith_utils/tests/cert.pem rename to libs/utils/tests/cert.pem diff --git a/zenith_utils/tests/key.pem b/libs/utils/tests/key.pem similarity index 100% rename from zenith_utils/tests/key.pem rename to libs/utils/tests/key.pem diff --git a/zenith_utils/tests/ssl_test.rs b/libs/utils/tests/ssl_test.rs similarity index 98% rename from zenith_utils/tests/ssl_test.rs rename to libs/utils/tests/ssl_test.rs index 0e330c44f8..002361667b 100644 --- a/zenith_utils/tests/ssl_test.rs +++ b/libs/utils/tests/ssl_test.rs @@ -9,7 +9,7 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use bytes::{Buf, BufMut, Bytes, BytesMut}; use lazy_static::lazy_static; -use zenith_utils::postgres_backend::{AuthType, Handler, PostgresBackend}; +use utils::postgres_backend::{AuthType, Handler, PostgresBackend}; fn make_tcp_pair() -> (TcpStream, TcpStream) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 1a533af95f..7b44dafb09 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -47,9 +47,9 @@ rusoto_core = "0.47" rusoto_s3 = "0.47" async-trait = "0.1" -postgres_ffi = { path = "../postgres_ffi" } -zenith_metrics = { path = "../zenith_metrics" } -zenith_utils = { path = "../zenith_utils" } +postgres_ffi = { path = "../libs/postgres_ffi" } +metrics = { path = "../libs/metrics" } +utils = { path = "../libs/utils" } workspace_hack = { version = "0.1", path = "../workspace_hack" } [dev-dependencies] diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index 077e7c9f83..78a27e460f 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -25,7 +25,7 @@ use crate::repository::Timeline; use crate::DatadirTimelineImpl; use postgres_ffi::xlog_utils::*; use postgres_ffi::*; -use zenith_utils::lsn::Lsn; +use utils::lsn::Lsn; /// This is short-living object only for the time of tarball creation, /// created mostly to avoid passing a lot of parameters between various functions diff --git a/pageserver/src/bin/dump_layerfile.rs b/pageserver/src/bin/dump_layerfile.rs index 7cf39566ac..af73ef6bdb 100644 --- a/pageserver/src/bin/dump_layerfile.rs +++ b/pageserver/src/bin/dump_layerfile.rs @@ -7,7 +7,7 @@ use pageserver::layered_repository::dump_layerfile_from_path; use pageserver::page_cache; use pageserver::virtual_file; use std::path::PathBuf; -use zenith_utils::GIT_VERSION; +use utils::GIT_VERSION; fn main() -> Result<()> { let arg_matches = App::new("Zenith dump_layerfile utility") diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 1610a26239..867bea1b06 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -2,14 +2,6 @@ use std::{env, path::Path, str::FromStr}; use tracing::*; -use zenith_utils::{ - auth::JwtAuth, - logging, - postgres_backend::AuthType, - tcp_listener, - zid::{ZTenantId, ZTimelineId}, - GIT_VERSION, -}; use anyhow::{bail, Context, Result}; @@ -25,12 +17,20 @@ use pageserver::{ thread_mgr::ThreadKind, timelines, virtual_file, LOG_FILE_NAME, }; -use zenith_utils::http::endpoint; -use zenith_utils::shutdown::exit_now; -use zenith_utils::signals::{self, Signal}; +use utils::{ + auth::JwtAuth, + http::endpoint, + logging, + postgres_backend::AuthType, + shutdown::exit_now, + signals::{self, Signal}, + tcp_listener, + zid::{ZTenantId, ZTimelineId}, + GIT_VERSION, +}; fn main() -> anyhow::Result<()> { - zenith_metrics::set_common_metrics_prefix("pageserver"); + metrics::set_common_metrics_prefix("pageserver"); let arg_matches = App::new("Zenith page server") .about("Materializes WAL stream to pages and serves them to the postgres") .version(GIT_VERSION) diff --git a/pageserver/src/bin/update_metadata.rs b/pageserver/src/bin/update_metadata.rs index bfbb6179c5..fae5e5c2e3 100644 --- a/pageserver/src/bin/update_metadata.rs +++ b/pageserver/src/bin/update_metadata.rs @@ -6,8 +6,7 @@ use clap::{App, Arg}; use pageserver::layered_repository::metadata::TimelineMetadata; use std::path::PathBuf; use std::str::FromStr; -use zenith_utils::lsn::Lsn; -use zenith_utils::GIT_VERSION; +use utils::{lsn::Lsn, GIT_VERSION}; fn main() -> Result<()> { let arg_matches = App::new("Zenith update metadata utility") diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 067073cd9b..0cba3f48f8 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -7,8 +7,10 @@ use anyhow::{bail, ensure, Context, Result}; use toml_edit; use toml_edit::{Document, Item}; -use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; +use utils::{ + postgres_backend::AuthType, + zid::{ZNodeId, ZTenantId, ZTimelineId}, +}; use std::convert::TryInto; use std::env; diff --git a/pageserver/src/http/models.rs b/pageserver/src/http/models.rs index d1dfb911ba..9b51e48477 100644 --- a/pageserver/src/http/models.rs +++ b/pageserver/src/http/models.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; -use zenith_utils::{ +use utils::{ lsn::Lsn, zid::{ZNodeId, ZTenantId, ZTimelineId}, }; diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index f49b1d7ba3..82ea5d1d09 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -4,19 +4,6 @@ use anyhow::{Context, Result}; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use tracing::*; -use zenith_utils::auth::JwtAuth; -use zenith_utils::http::endpoint::attach_openapi_ui; -use zenith_utils::http::endpoint::auth_middleware; -use zenith_utils::http::endpoint::check_permission; -use zenith_utils::http::error::ApiError; -use zenith_utils::http::{ - endpoint, - error::HttpErrorBody, - json::{json_request, json_response}, - request::parse_request_param, -}; -use zenith_utils::http::{RequestExt, RouterBuilder}; -use zenith_utils::zid::{ZTenantTimelineId, ZTimelineId}; use super::models::{ StatusResponse, TenantCreateRequest, TenantCreateResponse, TimelineCreateRequest, @@ -27,7 +14,18 @@ use crate::remote_storage::{ }; use crate::repository::Repository; use crate::timelines::{LocalTimelineInfo, RemoteTimelineInfo, TimelineInfo}; -use crate::{config::PageServerConf, tenant_mgr, timelines, ZTenantId}; +use crate::{config::PageServerConf, tenant_mgr, timelines}; +use utils::{ + auth::JwtAuth, + http::{ + endpoint::{self, attach_openapi_ui, auth_middleware, check_permission}, + error::{ApiError, HttpErrorBody}, + json::{json_request, json_response}, + request::parse_request_param, + RequestExt, RouterBuilder, + }, + zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}, +}; struct State { conf: &'static PageServerConf, diff --git a/pageserver/src/import_datadir.rs b/pageserver/src/import_datadir.rs index 232892973e..8f49903e6c 100644 --- a/pageserver/src/import_datadir.rs +++ b/pageserver/src/import_datadir.rs @@ -20,7 +20,7 @@ use postgres_ffi::waldecoder::*; use postgres_ffi::xlog_utils::*; use postgres_ffi::{pg_constants, ControlFileData, DBState_DB_SHUTDOWNED}; use postgres_ffi::{Oid, TransactionId}; -use zenith_utils::lsn::Lsn; +use utils::lsn::Lsn; /// /// Import all relation data pages from local disk into the repository. diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 59a3def1fb..7525bdb94e 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -46,15 +46,17 @@ use crate::virtual_file::VirtualFile; use crate::walreceiver::IS_WAL_RECEIVER; use crate::walredo::WalRedoManager; use crate::CheckpointConfig; -use crate::{ZTenantId, ZTimelineId}; -use zenith_metrics::{ +use metrics::{ register_histogram_vec, register_int_counter, register_int_counter_vec, register_int_gauge_vec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, }; -use zenith_utils::crashsafe_dir; -use zenith_utils::lsn::{AtomicLsn, Lsn, RecordLsn}; -use zenith_utils::seqwait::SeqWait; +use utils::{ + crashsafe_dir, + lsn::{AtomicLsn, Lsn, RecordLsn}, + seqwait::SeqWait, + zid::{ZTenantId, ZTimelineId}, +}; mod blob_io; pub mod block_io; diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index 03b7e453b3..c5530a5789 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -35,7 +35,6 @@ use crate::page_cache::{PageReadGuard, PAGE_SZ}; use crate::repository::{Key, Value, KEY_SIZE}; use crate::virtual_file::VirtualFile; use crate::walrecord; -use crate::{ZTenantId, ZTimelineId}; use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION}; use anyhow::{bail, ensure, Context, Result}; use serde::{Deserialize, Serialize}; @@ -51,8 +50,11 @@ use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use zenith_utils::bin_ser::BeSer; -use zenith_utils::lsn::Lsn; +use utils::{ + bin_ser::BeSer, + lsn::Lsn, + zid::{ZTenantId, ZTimelineId}, +}; /// /// Header stored in the beginning of the file diff --git a/pageserver/src/layered_repository/ephemeral_file.rs b/pageserver/src/layered_repository/ephemeral_file.rs index a2f8cda461..9537d3939c 100644 --- a/pageserver/src/layered_repository/ephemeral_file.rs +++ b/pageserver/src/layered_repository/ephemeral_file.rs @@ -17,8 +17,7 @@ use std::ops::DerefMut; use std::path::PathBuf; use std::sync::{Arc, RwLock}; use tracing::*; -use zenith_utils::zid::ZTenantId; -use zenith_utils::zid::ZTimelineId; +use utils::zid::{ZTenantId, ZTimelineId}; use std::os::unix::fs::FileExt; diff --git a/pageserver/src/layered_repository/filename.rs b/pageserver/src/layered_repository/filename.rs index 497912b408..f088088277 100644 --- a/pageserver/src/layered_repository/filename.rs +++ b/pageserver/src/layered_repository/filename.rs @@ -8,7 +8,7 @@ use std::fmt; use std::ops::Range; use std::path::PathBuf; -use zenith_utils::lsn::Lsn; +use utils::lsn::Lsn; // Note: LayeredTimeline::load_layer_map() relies on this sort order #[derive(Debug, PartialEq, Eq, Clone)] diff --git a/pageserver/src/layered_repository/image_layer.rs b/pageserver/src/layered_repository/image_layer.rs index fa91198a79..0e38d46e7a 100644 --- a/pageserver/src/layered_repository/image_layer.rs +++ b/pageserver/src/layered_repository/image_layer.rs @@ -30,7 +30,6 @@ use crate::layered_repository::storage_layer::{ use crate::page_cache::PAGE_SZ; use crate::repository::{Key, Value, KEY_SIZE}; use crate::virtual_file::VirtualFile; -use crate::{ZTenantId, ZTimelineId}; use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION}; use anyhow::{bail, ensure, Context, Result}; use bytes::Bytes; @@ -44,8 +43,11 @@ use std::path::{Path, PathBuf}; use std::sync::{RwLock, RwLockReadGuard}; use tracing::*; -use zenith_utils::bin_ser::BeSer; -use zenith_utils::lsn::Lsn; +use utils::{ + bin_ser::BeSer, + lsn::Lsn, + zid::{ZTenantId, ZTimelineId}, +}; /// /// Header stored in the beginning of the file diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index 33e1eabd8e..714a0bc579 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -14,19 +14,21 @@ use crate::layered_repository::storage_layer::{ }; use crate::repository::{Key, Value}; use crate::walrecord; -use crate::{ZTenantId, ZTimelineId}; use anyhow::{bail, ensure, Result}; use std::collections::HashMap; use tracing::*; +use utils::{ + bin_ser::BeSer, + lsn::Lsn, + vec_map::VecMap, + zid::{ZTenantId, ZTimelineId}, +}; // avoid binding to Write (conflicts with std::io::Write) // while being able to use std::fmt::Write's methods use std::fmt::Write as _; use std::ops::Range; use std::path::PathBuf; use std::sync::RwLock; -use zenith_utils::bin_ser::BeSer; -use zenith_utils::lsn::Lsn; -use zenith_utils::vec_map::VecMap; pub struct InMemoryLayer { conf: &'static PageServerConf, diff --git a/pageserver/src/layered_repository/layer_map.rs b/pageserver/src/layered_repository/layer_map.rs index 3984ee550f..03ee8b8ef1 100644 --- a/pageserver/src/layered_repository/layer_map.rs +++ b/pageserver/src/layered_repository/layer_map.rs @@ -16,12 +16,12 @@ use crate::layered_repository::InMemoryLayer; use crate::repository::Key; use anyhow::Result; use lazy_static::lazy_static; +use metrics::{register_int_gauge, IntGauge}; use std::collections::VecDeque; use std::ops::Range; use std::sync::Arc; use tracing::*; -use zenith_metrics::{register_int_gauge, IntGauge}; -use zenith_utils::lsn::Lsn; +use utils::lsn::Lsn; lazy_static! { static ref NUM_ONDISK_LAYERS: IntGauge = diff --git a/pageserver/src/layered_repository/metadata.rs b/pageserver/src/layered_repository/metadata.rs index 7daf899ba2..0b47f8d697 100644 --- a/pageserver/src/layered_repository/metadata.rs +++ b/pageserver/src/layered_repository/metadata.rs @@ -10,7 +10,7 @@ use std::path::PathBuf; use anyhow::ensure; use serde::{Deserialize, Serialize}; -use zenith_utils::{ +use utils::{ bin_ser::BeSer, lsn::Lsn, zid::{ZTenantId, ZTimelineId}, diff --git a/pageserver/src/layered_repository/storage_layer.rs b/pageserver/src/layered_repository/storage_layer.rs index e413f311c3..aad631c5c4 100644 --- a/pageserver/src/layered_repository/storage_layer.rs +++ b/pageserver/src/layered_repository/storage_layer.rs @@ -4,13 +4,15 @@ use crate::repository::{Key, Value}; use crate::walrecord::ZenithWalRecord; -use crate::{ZTenantId, ZTimelineId}; use anyhow::Result; use bytes::Bytes; use std::ops::Range; use std::path::PathBuf; -use zenith_utils::lsn::Lsn; +use utils::{ + lsn::Lsn, + zid::{ZTenantId, ZTimelineId}, +}; pub fn range_overlaps(a: &Range, b: &Range) -> bool where diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 6dddef5f27..e6ac159ef2 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -22,13 +22,10 @@ pub mod walredo; use lazy_static::lazy_static; use tracing::info; -use zenith_metrics::{register_int_gauge_vec, IntGaugeVec}; -use zenith_utils::{ - postgres_backend, - zid::{ZTenantId, ZTimelineId}, -}; +use utils::postgres_backend; use crate::thread_mgr::ThreadKind; +use metrics::{register_int_gauge_vec, IntGaugeVec}; use layered_repository::LayeredRepository; use pgdatadir_mapping::DatadirTimeline; diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index bd44384a44..0c179b95c5 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -47,7 +47,7 @@ use std::{ use once_cell::sync::OnceCell; use tracing::error; -use zenith_utils::{ +use utils::{ lsn::Lsn, zid::{ZTenantId, ZTimelineId}, }; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index c09b032e48..8f5ea2e845 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -20,15 +20,13 @@ use std::str; use std::str::FromStr; use std::sync::{Arc, RwLockReadGuard}; use tracing::*; -use zenith_metrics::{register_histogram_vec, HistogramVec}; -use zenith_utils::auth::{self, JwtAuth}; -use zenith_utils::auth::{Claims, Scope}; -use zenith_utils::lsn::Lsn; -use zenith_utils::postgres_backend::is_socket_read_timed_out; -use zenith_utils::postgres_backend::PostgresBackend; -use zenith_utils::postgres_backend::{self, AuthType}; -use zenith_utils::pq_proto::{BeMessage, FeMessage, RowDescriptor, SINGLE_COL_ROWDESC}; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::{ + auth::{self, Claims, JwtAuth, Scope}, + lsn::Lsn, + postgres_backend::{self, is_socket_read_timed_out, AuthType, PostgresBackend}, + pq_proto::{BeMessage, FeMessage, RowDescriptor, SINGLE_COL_ROWDESC}, + zid::{ZTenantId, ZTimelineId}, +}; use crate::basebackup; use crate::config::PageServerConf; @@ -41,6 +39,7 @@ use crate::thread_mgr; use crate::thread_mgr::ThreadKind; use crate::walreceiver; use crate::CheckpointConfig; +use metrics::{register_histogram_vec, HistogramVec}; // Wrapped in libpq CopyData enum PagestreamFeMessage { diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 0b9ea7c7a7..071eccc05d 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -20,8 +20,7 @@ use std::ops::Range; use std::sync::atomic::{AtomicIsize, Ordering}; use std::sync::{Arc, Mutex, RwLockReadGuard}; use tracing::{debug, error, trace, warn}; -use zenith_utils::bin_ser::BeSer; -use zenith_utils::lsn::Lsn; +use utils::{bin_ser::BeSer, lsn::Lsn}; /// Block number within a relation or SLRU. This matches PostgreSQL's BlockNumber type. pub type BlockNumber = u32; @@ -1212,7 +1211,7 @@ pub fn key_to_slru_block(key: Key) -> Result<(SlruKind, u32, BlockNumber)> { #[cfg(test)] pub fn create_test_timeline( repo: R, - timeline_id: zenith_utils::zid::ZTimelineId, + timeline_id: utils::zid::ZTimelineId, ) -> Result>> { let tline = repo.create_empty_timeline(timeline_id, Lsn(8))?; let tline = DatadirTimeline::new(tline, 256 * 1024); diff --git a/pageserver/src/remote_storage.rs b/pageserver/src/remote_storage.rs index effc8dcdf4..8a09f7b9ca 100644 --- a/pageserver/src/remote_storage.rs +++ b/pageserver/src/remote_storage.rs @@ -117,7 +117,7 @@ use crate::{ metadata::{TimelineMetadata, METADATA_FILE_NAME}, }, }; -use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; +use utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; /// A timeline status to share with pageserver's sync counterpart, /// after comparing local and remote timeline state. diff --git a/pageserver/src/remote_storage/storage_sync.rs b/pageserver/src/remote_storage/storage_sync.rs index 649e563dbc..4d1ec2e225 100644 --- a/pageserver/src/remote_storage/storage_sync.rs +++ b/pageserver/src/remote_storage/storage_sync.rs @@ -86,10 +86,7 @@ use self::{ index::{IndexPart, RemoteIndex, RemoteTimeline, RemoteTimelineIndex}, upload::{upload_index_part, upload_timeline_layers, UploadedTimeline}, }; -use super::{ - LocalTimelineInitStatus, LocalTimelineInitStatuses, RemoteStorage, SyncStartupData, - ZTenantTimelineId, -}; +use super::{LocalTimelineInitStatus, LocalTimelineInitStatuses, RemoteStorage, SyncStartupData}; use crate::{ config::PageServerConf, layered_repository::metadata::{metadata_path, TimelineMetadata}, @@ -99,11 +96,11 @@ use crate::{ thread_mgr::ThreadKind, }; -use zenith_metrics::{ +use metrics::{ register_histogram_vec, register_int_counter, register_int_gauge, HistogramVec, IntCounter, IntGauge, }; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; pub use self::download::download_index_part; @@ -145,7 +142,7 @@ mod sync_queue { use tracing::{debug, warn}; use super::SyncTask; - use zenith_utils::zid::ZTenantTimelineId; + use utils::zid::ZTenantTimelineId; static SENDER: OnceCell> = OnceCell::new(); static LENGTH: AtomicUsize = AtomicUsize::new(0); @@ -1197,7 +1194,7 @@ fn register_sync_status(sync_start: Instant, sync_name: &str, sync_status: Optio #[cfg(test)] mod test_utils { - use zenith_utils::lsn::Lsn; + use utils::lsn::Lsn; use crate::repository::repo_harness::RepoHarness; @@ -1246,7 +1243,7 @@ mod tests { use std::collections::BTreeSet; use super::{test_utils::dummy_metadata, *}; - use zenith_utils::lsn::Lsn; + use utils::lsn::Lsn; #[test] fn download_sync_tasks_merge() { diff --git a/pageserver/src/remote_storage/storage_sync/download.rs b/pageserver/src/remote_storage/storage_sync/download.rs index eb805cd0cc..7fe25ab36e 100644 --- a/pageserver/src/remote_storage/storage_sync/download.rs +++ b/pageserver/src/remote_storage/storage_sync/download.rs @@ -12,9 +12,10 @@ use crate::{ layered_repository::metadata::metadata_path, remote_storage::{ storage_sync::{sync_queue, SyncTask}, - RemoteStorage, ZTenantTimelineId, + RemoteStorage, }, }; +use utils::zid::ZTenantTimelineId; use super::{ index::{IndexPart, RemoteTimeline}, @@ -182,7 +183,7 @@ mod tests { use std::collections::{BTreeSet, HashSet}; use tempfile::tempdir; - use zenith_utils::lsn::Lsn; + use utils::lsn::Lsn; use crate::{ remote_storage::{ diff --git a/pageserver/src/remote_storage/storage_sync/index.rs b/pageserver/src/remote_storage/storage_sync/index.rs index 918bda1039..d847e03a24 100644 --- a/pageserver/src/remote_storage/storage_sync/index.rs +++ b/pageserver/src/remote_storage/storage_sync/index.rs @@ -13,11 +13,8 @@ use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use tokio::sync::RwLock; -use crate::{ - config::PageServerConf, layered_repository::metadata::TimelineMetadata, - remote_storage::ZTenantTimelineId, -}; -use zenith_utils::lsn::Lsn; +use crate::{config::PageServerConf, layered_repository::metadata::TimelineMetadata}; +use utils::{lsn::Lsn, zid::ZTenantTimelineId}; /// A part of the filesystem path, that needs a root to become a path again. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] diff --git a/pageserver/src/remote_storage/storage_sync/upload.rs b/pageserver/src/remote_storage/storage_sync/upload.rs index b4a2f6f989..d2ff77e92e 100644 --- a/pageserver/src/remote_storage/storage_sync/upload.rs +++ b/pageserver/src/remote_storage/storage_sync/upload.rs @@ -12,9 +12,10 @@ use crate::{ layered_repository::metadata::metadata_path, remote_storage::{ storage_sync::{index::RemoteTimeline, sync_queue, SyncTask}, - RemoteStorage, ZTenantTimelineId, + RemoteStorage, }, }; +use utils::zid::ZTenantTimelineId; use super::{index::IndexPart, SyncData, TimelineUpload}; @@ -208,7 +209,7 @@ mod tests { use std::collections::{BTreeSet, HashSet}; use tempfile::tempdir; - use zenith_utils::lsn::Lsn; + use utils::lsn::Lsn; use crate::{ remote_storage::{ diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index d75b4efe71..fc438cce9c 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -11,8 +11,10 @@ use std::fmt::Display; use std::ops::{AddAssign, Range}; use std::sync::{Arc, RwLockReadGuard}; use std::time::Duration; -use zenith_utils::lsn::{Lsn, RecordLsn}; -use zenith_utils::zid::ZTimelineId; +use utils::{ + lsn::{Lsn, RecordLsn}, + zid::ZTimelineId, +}; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] /// Key used in the Repository kv-store. @@ -431,7 +433,7 @@ pub mod repo_harness { use super::*; use hex_literal::hex; - use zenith_utils::zid::ZTenantId; + use utils::zid::ZTenantId; pub const TIMELINE_ID: ZTimelineId = ZTimelineId::from_array(hex!("11223344556677881122334455667788")); diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 71e85c58e6..33bb4dc2e0 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::fmt; use std::sync::{Arc, Mutex, MutexGuard}; use tracing::*; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::zid::{ZTenantId, ZTimelineId}; lazy_static! { static ref TENANTS: Mutex> = Mutex::new(HashMap::new()); diff --git a/pageserver/src/tenant_threads.rs b/pageserver/src/tenant_threads.rs index 0d9a94cc5b..4dcc15f817 100644 --- a/pageserver/src/tenant_threads.rs +++ b/pageserver/src/tenant_threads.rs @@ -7,7 +7,7 @@ use crate::tenant_mgr::TenantState; use anyhow::Result; use std::time::Duration; use tracing::*; -use zenith_utils::zid::ZTenantId; +use utils::zid::ZTenantId; /// /// Compaction thread's main loop diff --git a/pageserver/src/thread_mgr.rs b/pageserver/src/thread_mgr.rs index 4484bb1db1..2866c6be44 100644 --- a/pageserver/src/thread_mgr.rs +++ b/pageserver/src/thread_mgr.rs @@ -47,7 +47,7 @@ use tracing::{debug, error, info, warn}; use lazy_static::lazy_static; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::zid::{ZTenantId, ZTimelineId}; use crate::shutdown_pageserver; diff --git a/pageserver/src/timelines.rs b/pageserver/src/timelines.rs index 586d27d5b1..abbabc8b31 100644 --- a/pageserver/src/timelines.rs +++ b/pageserver/src/timelines.rs @@ -14,9 +14,11 @@ use std::{ }; use tracing::*; -use zenith_utils::lsn::Lsn; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; -use zenith_utils::{crashsafe_dir, logging}; +use utils::{ + crashsafe_dir, logging, + lsn::Lsn, + zid::{ZTenantId, ZTimelineId}, +}; use crate::{ config::PageServerConf, diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index 64f9db2338..4ce245a74f 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -11,15 +11,15 @@ //! src/backend/storage/file/fd.c //! use lazy_static::lazy_static; +use once_cell::sync::OnceCell; use std::fs::{File, OpenOptions}; use std::io::{Error, ErrorKind, Read, Seek, SeekFrom, Write}; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{RwLock, RwLockWriteGuard}; -use zenith_metrics::{register_histogram_vec, register_int_gauge_vec, HistogramVec, IntGaugeVec}; -use once_cell::sync::OnceCell; +use metrics::{register_histogram_vec, register_int_gauge_vec, HistogramVec, IntGaugeVec}; // Metrics collected on disk IO operations const STORAGE_IO_TIME_BUCKETS: &[f64] = &[ diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index c6c6e89854..583cdecb1d 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -38,7 +38,7 @@ use postgres_ffi::nonrelfile_utils::mx_offset_to_member_segment; use postgres_ffi::xlog_utils::*; use postgres_ffi::TransactionId; use postgres_ffi::{pg_constants, CheckPoint}; -use zenith_utils::lsn::Lsn; +use utils::lsn::Lsn; static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]); diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index e09af09820..ce4e4d45fb 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -29,11 +29,11 @@ use tokio_postgres::replication::ReplicationStream; use tokio_postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow}; use tokio_stream::StreamExt; use tracing::*; -use zenith_utils::lsn::Lsn; -use zenith_utils::pq_proto::ZenithFeedback; -use zenith_utils::zid::ZTenantId; -use zenith_utils::zid::ZTenantTimelineId; -use zenith_utils::zid::ZTimelineId; +use utils::{ + lsn::Lsn, + pq_proto::ZenithFeedback, + zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}, +}; // // We keep one WAL Receiver active per timeline. diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index b7c6ecf726..dcffcda6bb 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -35,17 +35,14 @@ use std::sync::Mutex; use std::time::Duration; use std::time::Instant; use tracing::*; -use zenith_metrics::{register_histogram, register_int_counter, Histogram, IntCounter}; -use zenith_utils::bin_ser::BeSer; -use zenith_utils::lsn::Lsn; -use zenith_utils::nonblock::set_nonblock; -use zenith_utils::zid::ZTenantId; +use utils::{bin_ser::BeSer, lsn::Lsn, nonblock::set_nonblock, zid::ZTenantId}; use crate::config::PageServerConf; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; use crate::reltag::{RelTag, SlruKind}; use crate::repository::Key; use crate::walrecord::ZenithWalRecord; +use metrics::{register_histogram, register_int_counter, Histogram, IntCounter}; use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_bitshift; use postgres_ffi::nonrelfile_utils::mx_offset_to_flags_offset; use postgres_ffi::nonrelfile_utils::mx_offset_to_member_offset; diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index a4bd99db38..81086a0cad 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -33,8 +33,8 @@ tokio = { version = "1.17", features = ["macros"] } tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" } tokio-rustls = "0.23.0" -zenith_utils = { path = "../zenith_utils" } -zenith_metrics = { path = "../zenith_metrics" } +utils = { path = "../libs/utils" } +metrics = { path = "../libs/metrics" } workspace_hack = { version = "0.1", path = "../workspace_hack" } [dev-dependencies] diff --git a/proxy/src/auth.rs b/proxy/src/auth.rs index bda14d67a1..4c54e2f9eb 100644 --- a/proxy/src/auth.rs +++ b/proxy/src/auth.rs @@ -12,7 +12,7 @@ use crate::waiters; use std::io; use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite}; -use zenith_utils::pq_proto::{BeMessage as Be, BeParameterStatusMessage}; +use utils::pq_proto::{BeMessage as Be, BeParameterStatusMessage}; pub use credentials::ClientCredentials; diff --git a/proxy/src/auth/flow.rs b/proxy/src/auth/flow.rs index 0fafaa2f47..bcfd94a9ed 100644 --- a/proxy/src/auth/flow.rs +++ b/proxy/src/auth/flow.rs @@ -5,7 +5,7 @@ use crate::stream::PqStream; use crate::{sasl, scram}; use std::io; use tokio::io::{AsyncRead, AsyncWrite}; -use zenith_utils::pq_proto::{BeAuthenticationSaslMessage, BeMessage, BeMessage as Be}; +use utils::pq_proto::{BeAuthenticationSaslMessage, BeMessage, BeMessage as Be}; /// Every authentication selector is supposed to implement this trait. pub trait AuthMethod { diff --git a/proxy/src/cancellation.rs b/proxy/src/cancellation.rs index 07d3bcc71a..a801313635 100644 --- a/proxy/src/cancellation.rs +++ b/proxy/src/cancellation.rs @@ -4,7 +4,7 @@ use parking_lot::Mutex; use std::net::SocketAddr; use tokio::net::TcpStream; use tokio_postgres::{CancelToken, NoTls}; -use zenith_utils::pq_proto::CancelKeyData; +use utils::pq_proto::CancelKeyData; /// Enables serving `CancelRequest`s. #[derive(Default)] diff --git a/proxy/src/http.rs b/proxy/src/http.rs index 33d134678f..5a75718742 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -1,10 +1,7 @@ use anyhow::anyhow; use hyper::{Body, Request, Response, StatusCode}; use std::net::TcpListener; -use zenith_utils::http::endpoint; -use zenith_utils::http::error::ApiError; -use zenith_utils::http::json::json_response; -use zenith_utils::http::{RouterBuilder, RouterService}; +use utils::http::{endpoint, error::ApiError, json::json_response, RouterBuilder, RouterService}; async fn status_handler(_: Request) -> Result, ApiError> { json_response(StatusCode::OK, "") diff --git a/proxy/src/main.rs b/proxy/src/main.rs index 862152bb7b..8df46619ec 100644 --- a/proxy/src/main.rs +++ b/proxy/src/main.rs @@ -30,7 +30,7 @@ use config::ProxyConfig; use futures::FutureExt; use std::future::Future; use tokio::{net::TcpListener, task::JoinError}; -use zenith_utils::GIT_VERSION; +use utils::GIT_VERSION; use crate::config::{ClientAuthMethod, RouterConfig}; @@ -43,7 +43,7 @@ async fn flatten_err( #[tokio::main] async fn main() -> anyhow::Result<()> { - zenith_metrics::set_common_metrics_prefix("zenith_proxy"); + metrics::set_common_metrics_prefix("zenith_proxy"); let arg_matches = App::new("Zenith proxy/router") .version(GIT_VERSION) .arg( diff --git a/proxy/src/mgmt.rs b/proxy/src/mgmt.rs index ab6fdff040..23ad8a2013 100644 --- a/proxy/src/mgmt.rs +++ b/proxy/src/mgmt.rs @@ -5,7 +5,7 @@ use std::{ net::{TcpListener, TcpStream}, thread, }; -use zenith_utils::{ +use utils::{ postgres_backend::{self, AuthType, PostgresBackend}, pq_proto::{BeMessage, SINGLE_COL_ROWDESC}, }; diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 788179252b..f7de1618df 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -5,10 +5,10 @@ use crate::stream::{MetricsStream, PqStream, Stream}; use anyhow::{bail, Context}; use futures::TryFutureExt; use lazy_static::lazy_static; +use metrics::{new_common_metric_name, register_int_counter, IntCounter}; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -use zenith_metrics::{new_common_metric_name, register_int_counter, IntCounter}; -use zenith_utils::pq_proto::{BeMessage as Be, *}; +use utils::pq_proto::{BeMessage as Be, *}; const ERR_INSECURE_CONNECTION: &str = "connection is insecure (try using `sslmode=require`)"; const ERR_PROTO_VIOLATION: &str = "protocol violation"; diff --git a/proxy/src/sasl/messages.rs b/proxy/src/sasl/messages.rs index b1ae8cc426..58be6268fe 100644 --- a/proxy/src/sasl/messages.rs +++ b/proxy/src/sasl/messages.rs @@ -1,9 +1,9 @@ //! Definitions for SASL messages. use crate::parse::{split_at_const, split_cstr}; -use zenith_utils::pq_proto::{BeAuthenticationSaslMessage, BeMessage}; +use utils::pq_proto::{BeAuthenticationSaslMessage, BeMessage}; -/// SASL-specific payload of [`PasswordMessage`](zenith_utils::pq_proto::FeMessage::PasswordMessage). +/// SASL-specific payload of [`PasswordMessage`](utils::pq_proto::FeMessage::PasswordMessage). #[derive(Debug)] pub struct FirstMessage<'a> { /// Authentication method, e.g. `"SCRAM-SHA-256"`. @@ -31,7 +31,7 @@ impl<'a> FirstMessage<'a> { /// A single SASL message. /// This struct is deliberately decoupled from lower-level -/// [`BeAuthenticationSaslMessage`](zenith_utils::pq_proto::BeAuthenticationSaslMessage). +/// [`BeAuthenticationSaslMessage`](utils::pq_proto::BeAuthenticationSaslMessage). #[derive(Debug)] pub(super) enum ServerMessage { /// We expect to see more steps. diff --git a/proxy/src/stream.rs b/proxy/src/stream.rs index fb0be84584..42b0185fde 100644 --- a/proxy/src/stream.rs +++ b/proxy/src/stream.rs @@ -9,7 +9,7 @@ use std::{io, task}; use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf}; use tokio_rustls::server::TlsStream; -use zenith_utils::pq_proto::{BeMessage, FeMessage, FeStartupPacket}; +use utils::pq_proto::{BeMessage, FeMessage, FeStartupPacket}; pin_project! { /// Stream wrapper which implements libpq's protocol. diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index ca5e2a6b55..76d40cdc2e 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -33,9 +33,9 @@ tokio-util = { version = "0.7", features = ["io"] } rusoto_core = "0.47" rusoto_s3 = "0.47" -postgres_ffi = { path = "../postgres_ffi" } -zenith_metrics = { path = "../zenith_metrics" } -zenith_utils = { path = "../zenith_utils" } +postgres_ffi = { path = "../libs/postgres_ffi" } +metrics = { path = "../libs/metrics" } +utils = { path = "../libs/utils" } workspace_hack = { version = "0.1", path = "../workspace_hack" } [dev-dependencies] diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index e191cb52fd..7434f921cb 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -10,11 +10,9 @@ use std::fs::{self, File}; use std::io::{ErrorKind, Write}; use std::path::{Path, PathBuf}; use std::thread; +use tokio::sync::mpsc; use tracing::*; use url::{ParseError, Url}; -use zenith_utils::http::endpoint; -use zenith_utils::zid::ZNodeId; -use zenith_utils::{logging, tcp_listener, GIT_VERSION}; use safekeeper::control_file::{self}; use safekeeper::defaults::{DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_PG_LISTEN_ADDR}; @@ -23,15 +21,15 @@ use safekeeper::s3_offload; use safekeeper::wal_service; use safekeeper::SafeKeeperConf; use safekeeper::{broker, callmemaybe}; -use tokio::sync::mpsc; -use zenith_utils::shutdown::exit_now; -use zenith_utils::signals; +use utils::{ + http::endpoint, logging, shutdown::exit_now, signals, tcp_listener, zid::ZNodeId, GIT_VERSION, +}; const LOCK_FILE_NAME: &str = "safekeeper.lock"; const ID_FILE_NAME: &str = "safekeeper.id"; fn main() -> Result<()> { - zenith_metrics::set_common_metrics_prefix("safekeeper"); + metrics::set_common_metrics_prefix("safekeeper"); let arg_matches = App::new("Zenith safekeeper") .about("Store WAL stream to local file system and push it to WAL receivers") .version(GIT_VERSION) diff --git a/safekeeper/src/broker.rs b/safekeeper/src/broker.rs index 147497d673..b84b5cf789 100644 --- a/safekeeper/src/broker.rs +++ b/safekeeper/src/broker.rs @@ -17,14 +17,12 @@ use std::time::Duration; use tokio::task::JoinHandle; use tokio::{runtime, time::sleep}; use tracing::*; -use zenith_utils::zid::ZTenantId; -use zenith_utils::zid::ZTimelineId; -use zenith_utils::{ - lsn::Lsn, - zid::{ZNodeId, ZTenantTimelineId}, -}; use crate::{safekeeper::Term, timeline::GlobalTimelines, SafeKeeperConf}; +use utils::{ + lsn::Lsn, + zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}, +}; const RETRY_INTERVAL_MSEC: u64 = 1000; const PUSH_INTERVAL_MSEC: u64 = 1000; diff --git a/safekeeper/src/callmemaybe.rs b/safekeeper/src/callmemaybe.rs index 1e52ec927b..8c3fbe26ba 100644 --- a/safekeeper/src/callmemaybe.rs +++ b/safekeeper/src/callmemaybe.rs @@ -16,8 +16,10 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio::task; use tokio_postgres::NoTls; use tracing::*; -use zenith_utils::connstring::connection_host_port; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::{ + connstring::connection_host_port, + zid::{ZTenantId, ZTimelineId}, +}; async fn request_callback( pageserver_connstr: String, diff --git a/safekeeper/src/control_file.rs b/safekeeper/src/control_file.rs index 7cc53edeb0..c49b4c058a 100644 --- a/safekeeper/src/control_file.rs +++ b/safekeeper/src/control_file.rs @@ -10,13 +10,11 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use tracing::*; -use zenith_metrics::{register_histogram_vec, Histogram, HistogramVec, DISK_WRITE_SECONDS_BUCKETS}; -use zenith_utils::bin_ser::LeSer; - -use zenith_utils::zid::ZTenantTimelineId; use crate::control_file_upgrade::upgrade_control_file; use crate::safekeeper::{SafeKeeperState, SK_FORMAT_VERSION, SK_MAGIC}; +use metrics::{register_histogram_vec, Histogram, HistogramVec, DISK_WRITE_SECONDS_BUCKETS}; +use utils::{bin_ser::LeSer, zid::ZTenantTimelineId}; use crate::SafeKeeperConf; @@ -251,10 +249,10 @@ impl Storage for FileStorage { mod test { use super::FileStorage; use super::*; - use crate::{safekeeper::SafeKeeperState, SafeKeeperConf, ZTenantTimelineId}; + use crate::{safekeeper::SafeKeeperState, SafeKeeperConf}; use anyhow::Result; use std::fs; - use zenith_utils::lsn::Lsn; + use utils::{lsn::Lsn, zid::ZTenantTimelineId}; fn stub_conf() -> SafeKeeperConf { let workdir = tempfile::tempdir().unwrap().into_path(); diff --git a/safekeeper/src/control_file_upgrade.rs b/safekeeper/src/control_file_upgrade.rs index 9effe42f8d..0cb14298cb 100644 --- a/safekeeper/src/control_file_upgrade.rs +++ b/safekeeper/src/control_file_upgrade.rs @@ -5,7 +5,7 @@ use crate::safekeeper::{ use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; use tracing::*; -use zenith_utils::{ +use utils::{ bin_ser::LeSer, lsn::Lsn, pq_proto::SystemId, diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index bb14049787..7d86523b0e 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -14,11 +14,12 @@ use regex::Regex; use std::str::FromStr; use std::sync::Arc; use tracing::info; -use zenith_utils::lsn::Lsn; -use zenith_utils::postgres_backend; -use zenith_utils::postgres_backend::PostgresBackend; -use zenith_utils::pq_proto::{BeMessage, FeStartupPacket, RowDescriptor, INT4_OID, TEXT_OID}; -use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}; +use utils::{ + lsn::Lsn, + postgres_backend::{self, PostgresBackend}, + pq_proto::{BeMessage, FeStartupPacket, RowDescriptor, INT4_OID, TEXT_OID}, + zid::{ZTenantId, ZTenantTimelineId, ZTimelineId}, +}; use crate::callmemaybe::CallmeEvent; use tokio::sync::mpsc::UnboundedSender; diff --git a/safekeeper/src/http/models.rs b/safekeeper/src/http/models.rs index 8a6ed7a812..ca18e64096 100644 --- a/safekeeper/src/http/models.rs +++ b/safekeeper/src/http/models.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; +use utils::zid::{ZNodeId, ZTenantId, ZTimelineId}; #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 26b23cddcc..2d22332db9 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -4,21 +4,22 @@ use serde::Serialize; use serde::Serializer; use std::fmt::Display; use std::sync::Arc; -use zenith_utils::http::json::json_request; -use zenith_utils::http::{RequestExt, RouterBuilder}; -use zenith_utils::lsn::Lsn; -use zenith_utils::zid::ZNodeId; -use zenith_utils::zid::ZTenantTimelineId; use crate::safekeeper::Term; use crate::safekeeper::TermHistory; use crate::timeline::GlobalTimelines; use crate::SafeKeeperConf; -use zenith_utils::http::endpoint; -use zenith_utils::http::error::ApiError; -use zenith_utils::http::json::json_response; -use zenith_utils::http::request::parse_request_param; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::{ + http::{ + endpoint, + error::ApiError, + json::{json_request, json_response}, + request::parse_request_param, + RequestExt, RouterBuilder, + }, + lsn::Lsn, + zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}, +}; use super::models::TimelineCreateRequest; diff --git a/safekeeper/src/json_ctrl.rs b/safekeeper/src/json_ctrl.rs index ad5d790105..407fafd990 100644 --- a/safekeeper/src/json_ctrl.rs +++ b/safekeeper/src/json_ctrl.rs @@ -22,9 +22,11 @@ use crate::timeline::TimelineTools; use postgres_ffi::pg_constants; use postgres_ffi::xlog_utils; use postgres_ffi::{uint32, uint64, Oid, XLogRecord}; -use zenith_utils::lsn::Lsn; -use zenith_utils::postgres_backend::PostgresBackend; -use zenith_utils::pq_proto::{BeMessage, RowDescriptor, TEXT_OID}; +use utils::{ + lsn::Lsn, + postgres_backend::PostgresBackend, + pq_proto::{BeMessage, RowDescriptor, TEXT_OID}, +}; #[derive(Serialize, Deserialize, Debug)] pub struct AppendLogicalMessage { @@ -191,7 +193,7 @@ struct XlLogicalMessage { impl XlLogicalMessage { pub fn encode(&self) -> Bytes { - use zenith_utils::bin_ser::LeSer; + use utils::bin_ser::LeSer; self.ser().unwrap().into() } } diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 69423d42d8..8951e8f680 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::time::Duration; use url::Url; -use zenith_utils::zid::{ZNodeId, ZTenantTimelineId}; +use utils::zid::{ZNodeId, ZTenantTimelineId}; pub mod broker; pub mod callmemaybe; diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index e6b12a0d81..3ad99ab0df 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -7,7 +7,6 @@ use anyhow::{anyhow, bail, Result}; use bytes::BytesMut; use tokio::sync::mpsc::UnboundedSender; use tracing::*; -use zenith_utils::sock_split::ReadStream; use crate::timeline::Timeline; @@ -23,8 +22,11 @@ use crate::safekeeper::ProposerAcceptorMessage; use crate::handler::SafekeeperPostgresHandler; use crate::timeline::TimelineTools; -use zenith_utils::postgres_backend::PostgresBackend; -use zenith_utils::pq_proto::{BeMessage, FeMessage}; +use utils::{ + postgres_backend::PostgresBackend, + pq_proto::{BeMessage, FeMessage}, + sock_split::ReadStream, +}; use crate::callmemaybe::CallmeEvent; diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index cf56261ee6..59174f34a2 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -11,8 +11,6 @@ use std::cmp::min; use std::fmt; use std::io::Read; use tracing::*; -use zenith_utils::zid::ZNodeId; -use zenith_utils::zid::ZTenantTimelineId; use lazy_static::lazy_static; @@ -20,13 +18,14 @@ use crate::broker::SafekeeperInfo; use crate::control_file; use crate::send_wal::HotStandbyFeedback; use crate::wal_storage; +use metrics::{register_gauge_vec, Gauge, GaugeVec}; use postgres_ffi::xlog_utils::MAX_SEND_SIZE; -use zenith_metrics::{register_gauge_vec, Gauge, GaugeVec}; -use zenith_utils::bin_ser::LeSer; -use zenith_utils::lsn::Lsn; -use zenith_utils::pq_proto::SystemId; -use zenith_utils::pq_proto::ZenithFeedback; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::{ + bin_ser::LeSer, + lsn::Lsn, + pq_proto::{SystemId, ZenithFeedback}, + zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}, +}; pub const SK_MAGIC: u32 = 0xcafeceefu32; pub const SK_FORMAT_VERSION: u32 = 4; diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index f12fb5cb4a..960f70d154 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -19,13 +19,14 @@ use std::time::Duration; use std::{str, thread}; use tokio::sync::mpsc::UnboundedSender; use tracing::*; -use zenith_utils::bin_ser::BeSer; -use zenith_utils::lsn::Lsn; -use zenith_utils::postgres_backend::PostgresBackend; -use zenith_utils::pq_proto::{BeMessage, FeMessage, WalSndKeepAlive, XLogDataBody, ZenithFeedback}; -use zenith_utils::sock_split::ReadStream; - -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use utils::{ + bin_ser::BeSer, + lsn::Lsn, + postgres_backend::PostgresBackend, + pq_proto::{BeMessage, FeMessage, WalSndKeepAlive, XLogDataBody, ZenithFeedback}, + sock_split::ReadStream, + zid::{ZTenantId, ZTimelineId}, +}; // See: https://www.postgresql.org/docs/13/protocol-replication.html const HOT_STANDBY_FEEDBACK_TAG_BYTE: u8 = b'h'; diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 777db7eb2b..fbae34251c 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -14,8 +14,11 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tracing::*; -use zenith_utils::lsn::Lsn; -use zenith_utils::zid::{ZNodeId, ZTenantTimelineId}; +use utils::{ + lsn::Lsn, + pq_proto::ZenithFeedback, + zid::{ZNodeId, ZTenantTimelineId}, +}; use crate::broker::SafekeeperInfo; use crate::callmemaybe::{CallmeEvent, SubscriptionStateKey}; @@ -30,8 +33,6 @@ use crate::wal_storage; use crate::wal_storage::Storage as wal_storage_iface; use crate::SafeKeeperConf; -use zenith_utils::pq_proto::ZenithFeedback; - const POLL_STATE_TIMEOUT: Duration = Duration::from_secs(1); /// Replica status update + hot standby feedback diff --git a/safekeeper/src/wal_service.rs b/safekeeper/src/wal_service.rs index 305e59bcd3..468ac28526 100644 --- a/safekeeper/src/wal_service.rs +++ b/safekeeper/src/wal_service.rs @@ -12,7 +12,7 @@ use crate::callmemaybe::CallmeEvent; use crate::handler::SafekeeperPostgresHandler; use crate::SafeKeeperConf; use tokio::sync::mpsc::UnboundedSender; -use zenith_utils::postgres_backend::{AuthType, PostgresBackend}; +use utils::postgres_backend::{AuthType, PostgresBackend}; /// Accept incoming TCP connections and spawn them into a background thread. pub fn thread_main( diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 7cef525bee..69a4fb11e1 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -20,8 +20,7 @@ use std::path::{Path, PathBuf}; use tracing::*; -use zenith_utils::lsn::Lsn; -use zenith_utils::zid::ZTenantTimelineId; +use utils::{lsn::Lsn, zid::ZTenantTimelineId}; use crate::safekeeper::SafeKeeperState; @@ -30,7 +29,7 @@ use postgres_ffi::xlog_utils::{XLogFileName, XLOG_BLCKSZ}; use postgres_ffi::waldecoder::WalStreamDecoder; -use zenith_metrics::{ +use metrics::{ register_gauge_vec, register_histogram_vec, Gauge, GaugeVec, Histogram, HistogramVec, DISK_WRITE_SECONDS_BUCKETS, }; diff --git a/test_runner/batch_others/test_wal_restore.py b/test_runner/batch_others/test_wal_restore.py index 2dbde954fc..49421aa4e8 100644 --- a/test_runner/batch_others/test_wal_restore.py +++ b/test_runner/batch_others/test_wal_restore.py @@ -26,7 +26,7 @@ def test_wal_restore(zenith_env_builder: ZenithEnvBuilder, data_dir = os.path.join(test_output_dir, 'pgsql.restored') with VanillaPostgres(data_dir, PgBin(test_output_dir), port) as restored: pg_bin.run_capture([ - os.path.join(base_dir, 'zenith_utils/scripts/restore_from_wal.sh'), + os.path.join(base_dir, 'libs/utils/scripts/restore_from_wal.sh'), os.path.join(pg_distrib_dir, 'bin'), os.path.join(test_output_dir, 'repo/safekeepers/sk1/{}/*'.format(tenant_id)), data_dir, diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 84244b3363..f178b5b766 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -24,8 +24,8 @@ indexmap = { version = "1", default-features = false, features = ["std"] } libc = { version = "0.2", features = ["extra_traits", "std"] } log = { version = "0.4", default-features = false, features = ["serde", "std"] } memchr = { version = "2", features = ["std", "use_std"] } -num-integer = { version = "0.1", default-features = false, features = ["std"] } -num-traits = { version = "0.2", features = ["std"] } +num-integer = { version = "0.1", default-features = false, features = ["i128"] } +num-traits = { version = "0.2", features = ["i128", "std"] } prost = { version = "0.9", features = ["prost-derive", "std"] } rand = { version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha", "rand_hc", "small_rng", "std", "std_rng"] } regex = { version = "1", features = ["aho-corasick", "memchr", "perf", "perf-cache", "perf-dfa", "perf-inline", "perf-literal", "std", "unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] } @@ -39,7 +39,6 @@ tracing-core = { version = "0.1", features = ["lazy_static", "std"] } [build-dependencies] anyhow = { version = "1", features = ["backtrace", "std"] } bytes = { version = "1", features = ["serde", "std"] } -cc = { version = "1", default-features = false, features = ["jobserver", "parallel"] } clap = { version = "2", features = ["ansi_term", "atty", "color", "strsim", "suggestions", "vec_map"] } either = { version = "1", features = ["use_std"] } hashbrown = { version = "0.11", features = ["ahash", "inline-more", "raw"] } diff --git a/zenith/Cargo.toml b/zenith/Cargo.toml index 69283d3763..9692e97331 100644 --- a/zenith/Cargo.toml +++ b/zenith/Cargo.toml @@ -13,6 +13,6 @@ postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98 pageserver = { path = "../pageserver" } control_plane = { path = "../control_plane" } safekeeper = { path = "../safekeeper" } -postgres_ffi = { path = "../postgres_ffi" } -zenith_utils = { path = "../zenith_utils" } +postgres_ffi = { path = "../libs/postgres_ffi" } +utils = { path = "../libs/utils" } workspace_hack = { version = "0.1", path = "../workspace_hack" } diff --git a/zenith/src/main.rs b/zenith/src/main.rs index f248a5db5b..afbbbe395b 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -16,11 +16,13 @@ use safekeeper::defaults::{ use std::collections::{BTreeSet, HashMap}; use std::process::exit; use std::str::FromStr; -use zenith_utils::auth::{Claims, Scope}; -use zenith_utils::lsn::Lsn; -use zenith_utils::postgres_backend::AuthType; -use zenith_utils::zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}; -use zenith_utils::GIT_VERSION; +use utils::{ + auth::{Claims, Scope}, + lsn::Lsn, + postgres_backend::AuthType, + zid::{ZNodeId, ZTenantId, ZTenantTimelineId, ZTimelineId}, + GIT_VERSION, +}; use pageserver::timelines::TimelineInfo;