From 403d9779d93900b19a4e2e5a898472b04c103e68 Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Fri, 8 Oct 2021 18:55:41 +0300 Subject: [PATCH 01/46] safekeeper: add initial metrics and HTTP handler (#699, #541) * `wal_acceptor`: add HTTP handler, /metrics endpoint only, no authentication * Two gauges are currently reported: `flush_lsn` and `commit_lsn` * Add `DEFAULT_PG_LISTEN_PORT` and `DEFAULT_PG_LISTEN_PORT` consts for uniformity --- Cargo.lock | 2 + test_runner/fixtures/zenith_fixtures.py | 21 ++++++++--- walkeeper/Cargo.toml | 2 + walkeeper/src/bin/wal_acceptor.rs | 44 +++++++++++++++++++--- walkeeper/src/lib.rs | 13 ++++++- walkeeper/src/receive_wal.rs | 2 +- walkeeper/src/safekeeper.rs | 50 +++++++++++++++++++++++++ walkeeper/src/wal_service.rs | 6 +-- zenith_metrics/src/lib.rs | 2 + 9 files changed, 125 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c217dfbebb..e36b462d8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2339,6 +2339,7 @@ dependencies = [ "byteorder", "bytes", "clap", + "const_format", "crc32c", "daemonize", "fs2", @@ -2358,6 +2359,7 @@ dependencies = [ "tokio-stream", "walkdir", "workspace_hack", + "zenith_metrics", "zenith_utils", ] diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index d29d278cdd..c0ce57801d 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -794,12 +794,17 @@ def read_pid(path: Path): return int(path.read_text()) +@dataclass +class WalAcceptorPort: + pg: int + http: int + @dataclass class WalAcceptor: """ An object representing a running wal acceptor daemon. """ wa_bin_path: Path data_dir: Path - port: int + port: WalAcceptorPort num: int # identifier for logging pageserver_port: int auth_token: Optional[str] = None @@ -811,7 +816,8 @@ class WalAcceptor: cmd = [str(self.wa_bin_path)] cmd.extend(["-D", str(self.data_dir)]) - cmd.extend(["-l", f"localhost:{self.port}"]) + cmd.extend(["--listen-pg", f"localhost:{self.port.pg}"]) + cmd.extend(["--listen-http", f"localhost:{self.port.http}"]) cmd.append("--daemonize") cmd.append("--no-sync") # Tell page server it can receive WAL from this WAL safekeeper @@ -868,14 +874,14 @@ class WalAcceptor: # "replication=0" hacks psycopg not to send additional queries # on startup, see https://github.com/psycopg/psycopg2/pull/482 - connstr = f"host=localhost port={self.port} replication=0 options='-c ztimelineid={timeline_id} ztenantid={tenant_id}'" + connstr = f"host=localhost port={self.port.pg} replication=0 options='-c ztimelineid={timeline_id} ztenantid={tenant_id}'" with closing(psycopg2.connect(connstr)) as conn: # server doesn't support transactions conn.autocommit = True with conn.cursor() as cur: request_json = json.dumps(request) - print(f"JSON_CTRL request on port {self.port}: {request_json}") + print(f"JSON_CTRL request on port {self.port.pg}: {request_json}") cur.execute("JSON_CTRL " + request_json) all = cur.fetchall() print(f"JSON_CTRL response: {all[0][0]}") @@ -898,7 +904,10 @@ class WalAcceptorFactory: wa = WalAcceptor( wa_bin_path=self.wa_bin_path, data_dir=self.data_dir / "wal_acceptor_{}".format(wa_num), - port=self.port_distributor.get_port(), + port=WalAcceptorPort( + pg=self.port_distributor.get_port(), + http=self.port_distributor.get_port(), + ), num=wa_num, pageserver_port=self.pageserver_port, auth_token=auth_token, @@ -922,7 +931,7 @@ class WalAcceptorFactory: def get_connstrs(self) -> str: """ Get list of wal acceptor endpoints suitable for wal_acceptors GUC """ - return ','.join(["localhost:{}".format(wa.port) for wa in self.instances]) + return ','.join(["localhost:{}".format(wa.port.pg) for wa in self.instances]) @zenfixture diff --git a/walkeeper/Cargo.toml b/walkeeper/Cargo.toml index 16790ca214..f31126eb05 100644 --- a/walkeeper/Cargo.toml +++ b/walkeeper/Cargo.toml @@ -28,9 +28,11 @@ humantime = "2.1.0" walkdir = "2" serde = { version = "1.0", features = ["derive"] } hex = "0.4.3" +const_format = "0.2.21" # FIXME: 'pageserver' is needed for ZTimelineId. Refactor pageserver = { path = "../pageserver" } postgres_ffi = { path = "../postgres_ffi" } workspace_hack = { path = "../workspace_hack" } +zenith_metrics = { path = "../zenith_metrics" } zenith_utils = { path = "../zenith_utils" } diff --git a/walkeeper/src/bin/wal_acceptor.rs b/walkeeper/src/bin/wal_acceptor.rs index d8a0ab6737..5bd64cb8a6 100644 --- a/walkeeper/src/bin/wal_acceptor.rs +++ b/walkeeper/src/bin/wal_acceptor.rs @@ -3,18 +3,23 @@ // use anyhow::Result; use clap::{App, Arg}; +use const_format::formatcp; use daemonize::Daemonize; use log::*; use std::env; +use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::thread; +use zenith_utils::http::endpoint; use zenith_utils::logging; +use walkeeper::defaults::{DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_PG_LISTEN_ADDR}; use walkeeper::s3_offload; use walkeeper::wal_service; use walkeeper::WalAcceptorConf; fn main() -> Result<()> { + zenith_metrics::set_common_metrics_prefix("safekeeper"); let arg_matches = App::new("Zenith wal_acceptor") .about("Store WAL stream to local file system and push it to WAL receivers") .arg( @@ -25,11 +30,18 @@ fn main() -> Result<()> { .help("Path to the WAL acceptor data directory"), ) .arg( - Arg::with_name("listen") + Arg::with_name("listen-pg") .short("l") - .long("listen") + .long("listen-pg") + .alias("listen") // for compatibility .takes_value(true) - .help("listen for incoming connections on ip:port (default: 127.0.0.1:5454)"), + .help(formatcp!("listen for incoming WAL data connections on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")), + ) + .arg( + Arg::with_name("listen-http") + .long("listen-http") + .takes_value(true) + .help(formatcp!("http endpoint address for metrics on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")), ) .arg( Arg::with_name("pageserver") @@ -70,7 +82,8 @@ fn main() -> Result<()> { daemonize: false, no_sync: false, pageserver_addr: None, - listen_addr: "localhost:5454".to_string(), + listen_pg_addr: DEFAULT_PG_LISTEN_ADDR.to_string(), + listen_http_addr: DEFAULT_HTTP_LISTEN_ADDR.to_string(), ttl: None, recall_period: None, pageserver_auth_token: env::var("PAGESERVER_AUTH_TOKEN").ok(), @@ -91,8 +104,12 @@ fn main() -> Result<()> { conf.daemonize = true; } - if let Some(addr) = arg_matches.value_of("listen") { - conf.listen_addr = addr.to_owned(); + if let Some(addr) = arg_matches.value_of("listen-pg") { + conf.listen_pg_addr = addr.to_owned(); + } + + if let Some(addr) = arg_matches.value_of("listen-http") { + conf.listen_http_addr = addr.to_owned(); } if let Some(addr) = arg_matches.value_of("pageserver") { @@ -114,6 +131,11 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { let log_filename = conf.data_dir.join("wal_acceptor.log"); let (_scope_guard, log_file) = logging::init(log_filename, conf.daemonize)?; + let http_listener = TcpListener::bind(conf.listen_http_addr.clone()).map_err(|e| { + error!("failed to bind to address {}: {}", conf.listen_http_addr, e); + e + })?; + if conf.daemonize { info!("daemonizing..."); @@ -136,6 +158,16 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { let mut threads = Vec::new(); + let http_endpoint_thread = thread::Builder::new() + .name("http_endpoint_thread".into()) + .spawn(move || { + // No authentication at all: read-only metrics only, early stage. + let router = endpoint::make_router(); + endpoint::serve_thread_main(router, http_listener).unwrap(); + }) + .unwrap(); + threads.push(http_endpoint_thread); + if conf.ttl.is_some() { let s3_conf = conf.clone(); let s3_offload_thread = thread::Builder::new() diff --git a/walkeeper/src/lib.rs b/walkeeper/src/lib.rs index fb04459c47..6c1f70efa2 100644 --- a/walkeeper/src/lib.rs +++ b/walkeeper/src/lib.rs @@ -11,12 +11,23 @@ pub mod send_wal; pub mod timeline; pub mod wal_service; +pub mod defaults { + use const_format::formatcp; + + pub const DEFAULT_PG_LISTEN_PORT: u16 = 5454; + pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}"); + + pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 7676; + pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}"); +} + #[derive(Debug, Clone)] pub struct WalAcceptorConf { pub data_dir: PathBuf, pub daemonize: bool, pub no_sync: bool, - pub listen_addr: String, + pub listen_pg_addr: String, + pub listen_http_addr: String, pub pageserver_addr: Option, // TODO (create issue) this is temporary, until protocol between PG<->SK<->PS rework pub pageserver_auth_token: Option, diff --git a/walkeeper/src/receive_wal.rs b/walkeeper/src/receive_wal.rs index 4596344b76..527c8d891c 100644 --- a/walkeeper/src/receive_wal.rs +++ b/walkeeper/src/receive_wal.rs @@ -42,7 +42,7 @@ fn request_callback(conf: WalAcceptorConf, timelineid: ZTimelineId, tenantid: ZT ); // use Config parsing because SockAddr parsing doesnt allow to use host names instead of ip addresses - let me_connstr = format!("postgresql://no_user@{}/no_db", conf.listen_addr); + let me_connstr = format!("postgresql://no_user@{}/no_db", conf.listen_pg_addr); let me_conf: Config = me_connstr.parse().unwrap(); let (host, port) = connection_host_port(&me_conf); let callme = format!( diff --git a/walkeeper/src/safekeeper.rs b/walkeeper/src/safekeeper.rs index 8cb629774e..111cbbaf95 100644 --- a/walkeeper/src/safekeeper.rs +++ b/walkeeper/src/safekeeper.rs @@ -15,8 +15,11 @@ use std::cmp::min; use std::io; use std::io::Read; +use lazy_static::lazy_static; + use crate::replication::HotStandbyFeedback; use postgres_ffi::xlog_utils::MAX_SEND_SIZE; +use zenith_metrics::{register_gauge_vec, Gauge, GaugeVec}; use zenith_utils::bin_ser::LeSer; use zenith_utils::lsn::Lsn; use zenith_utils::pq_proto::SystemId; @@ -279,6 +282,38 @@ pub trait Storage { fn write_wal(&mut self, server: &ServerInfo, startpos: Lsn, buf: &[u8]) -> Result<()>; } +lazy_static! { + // The prometheus crate does not support u64 yet, i64 only (see `IntGauge`). + // i64 is faster than f64, so update to u64 when available. + static ref FLUSH_LSN_GAUGE: GaugeVec = register_gauge_vec!( + "safekeeper_flush_lsn", + "Current flush_lsn, grouped by timeline", + &["ztli"] + ) + .expect("Failed to register safekeeper_flush_lsn int gauge vec"); + static ref COMMIT_LSN_GAUGE: GaugeVec = register_gauge_vec!( + "safekeeper_commit_lsn", + "Current commit_lsn (not necessarily persisted to disk), grouped by timeline", + &["ztli"] + ) + .expect("Failed to register safekeeper_commit_lsn int gauge vec"); +} + +struct SafeKeeperMetrics { + flush_lsn: Gauge, + commit_lsn: Gauge, +} + +impl SafeKeeperMetrics { + fn new(ztli: ZTimelineId) -> SafeKeeperMetrics { + let ztli_str = format!("{}", ztli); + SafeKeeperMetrics { + flush_lsn: FLUSH_LSN_GAUGE.with_label_values(&[&ztli_str]), + commit_lsn: COMMIT_LSN_GAUGE.with_label_values(&[&ztli_str]), + } + } +} + /// SafeKeeper which consumes events (messages from compute) and provides /// replies. pub struct SafeKeeper { @@ -286,6 +321,8 @@ pub struct SafeKeeper { /// Established by reading wal. pub flush_lsn: Lsn, pub tli: u32, + // Cached metrics so we don't have to recompute labels on each update. + metrics: Option, /// not-yet-flushed pairs of same named fields in s.* pub commit_lsn: Lsn, pub truncate_lsn: Lsn, @@ -304,6 +341,7 @@ where SafeKeeper { flush_lsn, tli, + metrics: None, commit_lsn: state.commit_lsn, truncate_lsn: state.truncate_lsn, storage, @@ -355,6 +393,8 @@ where self.s.server.wal_seg_size = msg.wal_seg_size; self.storage.persist(&self.s, true)?; + self.metrics = Some(SafeKeeperMetrics::new(self.s.server.ztli)); + info!( "processed greeting from proposer {:?}, sending term {:?}", msg.proposer_id, self.s.acceptor_state.term @@ -478,6 +518,11 @@ where } if last_rec_lsn > self.flush_lsn { self.flush_lsn = last_rec_lsn; + self.metrics + .as_ref() + .unwrap() + .flush_lsn + .set(u64::from(self.flush_lsn) as f64); } // Advance commit_lsn taking into account what we have locally. xxx this @@ -495,6 +540,11 @@ where sync_control_file |= commit_lsn >= msg.h.epoch_start_lsn && self.s.commit_lsn < msg.h.epoch_start_lsn; self.commit_lsn = commit_lsn; + self.metrics + .as_ref() + .unwrap() + .commit_lsn + .set(u64::from(self.commit_lsn) as f64); } self.truncate_lsn = msg.h.truncate_lsn; diff --git a/walkeeper/src/wal_service.rs b/walkeeper/src/wal_service.rs index c77078560c..7730a058ed 100644 --- a/walkeeper/src/wal_service.rs +++ b/walkeeper/src/wal_service.rs @@ -13,9 +13,9 @@ use zenith_utils::postgres_backend::{AuthType, PostgresBackend}; /// Accept incoming TCP connections and spawn them into a background thread. pub fn thread_main(conf: WalAcceptorConf) -> Result<()> { - info!("Starting wal acceptor on {}", conf.listen_addr); - let listener = TcpListener::bind(conf.listen_addr.clone()).map_err(|e| { - error!("failed to bind to address {}: {}", conf.listen_addr, e); + info!("Starting wal acceptor on {}", conf.listen_pg_addr); + let listener = TcpListener::bind(conf.listen_pg_addr.clone()).map_err(|e| { + error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); e })?; diff --git a/zenith_metrics/src/lib.rs b/zenith_metrics/src/lib.rs index e3c3c81ee7..b01e80d873 100644 --- a/zenith_metrics/src/lib.rs +++ b/zenith_metrics/src/lib.rs @@ -5,6 +5,8 @@ use lazy_static::lazy_static; use once_cell::race::OnceBox; pub use prometheus::{exponential_buckets, linear_buckets}; +pub use prometheus::{register_gauge, Gauge}; +pub use prometheus::{register_gauge_vec, GaugeVec}; pub use prometheus::{register_histogram, Histogram}; pub use prometheus::{register_histogram_vec, HistogramVec}; pub use prometheus::{register_int_counter, IntCounter}; From 3b82e806f28d69c92a67b145ceb33055aea52508 Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Fri, 8 Oct 2021 00:08:03 -0700 Subject: [PATCH 02/46] pageserver - use VecMap for in-memory PageVersions --- pageserver/src/layered_repository.rs | 1 + .../src/layered_repository/delta_layer.rs | 6 +- .../src/layered_repository/inmemory_layer.rs | 62 +++-- .../src/layered_repository/page_versions.rs | 182 +++++++++++++ zenith_utils/src/lib.rs | 3 + zenith_utils/src/vec_map.rs | 239 ++++++++++++++++++ 6 files changed, 457 insertions(+), 36 deletions(-) create mode 100644 pageserver/src/layered_repository/page_versions.rs create mode 100644 zenith_utils/src/vec_map.rs diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index b420b381c5..0726baa86d 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -57,6 +57,7 @@ mod image_layer; mod inmemory_layer; mod interval_tree; mod layer_map; +mod page_versions; mod storage_layer; use delta_layer::DeltaLayer; diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index ad16a86030..4ea415c899 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -380,7 +380,7 @@ impl DeltaLayer { start_lsn: Lsn, end_lsn: Lsn, dropped: bool, - page_versions: impl Iterator, + page_versions: impl Iterator, relsizes: BTreeMap, ) -> Result { if seg.rel.is_blocky() { @@ -416,11 +416,11 @@ impl DeltaLayer { let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER); - for (key, page_version) in page_versions { + for (blknum, lsn, page_version) in page_versions { let buf = PageVersion::ser(page_version)?; let blob_range = page_version_writer.write_blob(&buf)?; - let old = inner.page_version_metas.insert(*key, blob_range); + let old = inner.page_version_metas.insert((blknum, lsn), blob_range); assert!(old.is_none()); } diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index f96b5e71d1..6e3b00350d 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -24,6 +24,8 @@ use std::sync::{Arc, RwLock}; use zenith_utils::accum::Accum; use zenith_utils::lsn::Lsn; +use super::page_versions::PageVersions; + pub struct InMemoryLayer { conf: &'static PageServerConf, tenantid: ZTenantId, @@ -58,7 +60,7 @@ pub struct InMemoryLayerInner { /// All versions of all pages in the layer are are kept here. /// Indexed by block number and LSN. /// - page_versions: BTreeMap<(u32, Lsn), PageVersion>, + page_versions: PageVersions, /// /// `segsizes` tracks the size of the segment at different points in time. @@ -172,13 +174,13 @@ impl Layer for InMemoryLayer { { let inner = self.inner.read().unwrap(); - // Scan the BTreeMap backwards, starting from reconstruct_data.lsn. - let minkey = (blknum, Lsn(0)); - let maxkey = (blknum, lsn); - let mut iter = inner + // Scan the page versions backwards, starting from `lsn`. + let iter = inner .page_versions - .range((Included(&minkey), Included(&maxkey))); - while let Some(((_blknum, _entry_lsn), entry)) = iter.next_back() { + .get_block_lsn_range(blknum, ..=lsn) + .iter() + .rev(); + for (_entry_lsn, entry) in iter { if let Some(img) = &entry.page_image { reconstruct_data.page_img = Some(img.clone()); need_image = false; @@ -279,13 +281,13 @@ impl Layer for InMemoryLayer { println!("segsizes {}: {}", k, v); } - for (k, v) in inner.page_versions.iter() { + for (blknum, lsn, pv) in inner.page_versions.ordered_page_version_iter(None) { println!( "blk {} at {}: {}/{}\n", - k.0, - k.1, - v.page_image.is_some(), - v.record.is_some() + blknum, + lsn, + pv.page_image.is_some(), + pv.record.is_some() ); } @@ -353,7 +355,7 @@ impl InMemoryLayer { incremental: false, inner: RwLock::new(InMemoryLayerInner { drop_lsn: None, - page_versions: BTreeMap::new(), + page_versions: PageVersions::default(), segsizes, writeable: true, }), @@ -403,7 +405,7 @@ impl InMemoryLayer { inner.check_writeable()?; - let old = inner.page_versions.insert((blknum, lsn), pv); + let old = inner.page_versions.append_or_update_last(blknum, lsn, pv); if old.is_some() { // We already had an entry for this LSN. That's odd.. @@ -448,7 +450,9 @@ impl InMemoryLayer { gapblknum, blknum ); - let old = inner.page_versions.insert((gapblknum, lsn), zeropv); + let old = inner + .page_versions + .append_or_update_last(gapblknum, lsn, zeropv); // We already had an entry for this LSN. That's odd.. if old.is_some() { @@ -550,7 +554,7 @@ impl InMemoryLayer { incremental: true, inner: RwLock::new(InMemoryLayerInner { drop_lsn: None, - page_versions: BTreeMap::new(), + page_versions: PageVersions::default(), segsizes, writeable: true, }), @@ -615,16 +619,9 @@ impl InMemoryLayer { } } - let mut before_page_versions = BTreeMap::new(); - let mut after_page_versions = BTreeMap::new(); - for ((blknum, lsn), pv) in inner.page_versions.iter() { - if *lsn > cutoff_lsn { - after_page_versions.insert((*blknum, *lsn), pv.clone()); - after_oldest_lsn.accum(min, *lsn); - } else { - before_page_versions.insert((*blknum, *lsn), pv.clone()); - } - } + let (before_page_versions, after_page_versions) = inner + .page_versions + .split_at(Lsn(cutoff_lsn.0 + 1), &mut after_oldest_lsn); let frozen = Arc::new(InMemoryLayer { conf: self.conf, @@ -654,7 +651,10 @@ impl InMemoryLayer { )?; let new_inner = new_open.inner.get_mut().unwrap(); - new_inner.page_versions.append(&mut after_page_versions); + // Ensure page_versions doesn't contain anything + // so we can just replace it + assert!(new_inner.page_versions.is_empty()); + new_inner.page_versions = after_page_versions; new_inner.segsizes.append(&mut after_segsizes); Some(Arc::new(new_open)) @@ -702,7 +702,7 @@ impl InMemoryLayer { self.start_lsn, drop_lsn, true, - inner.page_versions.iter(), + inner.page_versions.ordered_page_version_iter(None), inner.segsizes.clone(), )?; trace!( @@ -722,11 +722,7 @@ impl InMemoryLayer { before_segsizes.insert(*lsn, *size); } } - let mut before_page_versions = inner.page_versions.iter().filter(|tup| { - let ((_blknum, lsn), _pv) = tup; - - *lsn < end_lsn - }); + let mut before_page_versions = inner.page_versions.ordered_page_version_iter(Some(end_lsn)); let mut frozen_layers: Vec> = Vec::new(); diff --git a/pageserver/src/layered_repository/page_versions.rs b/pageserver/src/layered_repository/page_versions.rs new file mode 100644 index 0000000000..fe7df0caa9 --- /dev/null +++ b/pageserver/src/layered_repository/page_versions.rs @@ -0,0 +1,182 @@ +use std::{collections::HashMap, ops::RangeBounds, slice}; + +use zenith_utils::{accum::Accum, lsn::Lsn, vec_map::VecMap}; + +use super::storage_layer::PageVersion; + +const EMPTY_SLICE: &[(Lsn, PageVersion)] = &[]; + +#[derive(Debug, Default)] +pub struct PageVersions(HashMap>); + +impl PageVersions { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn append_or_update_last( + &mut self, + blknum: u32, + lsn: Lsn, + page_version: PageVersion, + ) -> Option { + let map = self.0.entry(blknum).or_insert_with(VecMap::default); + map.append_or_update_last(lsn, page_version).unwrap() + } + + /// Get all [`PageVersion`]s in a block + pub fn get_block_slice(&self, blknum: u32) -> &[(Lsn, PageVersion)] { + self.0 + .get(&blknum) + .map(VecMap::as_slice) + .unwrap_or(EMPTY_SLICE) + } + + /// Get a range of [`PageVersions`] in a block + pub fn get_block_lsn_range>( + &self, + blknum: u32, + range: R, + ) -> &[(Lsn, PageVersion)] { + self.0 + .get(&blknum) + .map(|vec_map| vec_map.slice_range(range)) + .unwrap_or(EMPTY_SLICE) + } + + /// Split the page version map into two. + /// + /// Left contains everything up to and not including [`cutoff_lsn`]. + /// Right contains [`cutoff_lsn`] and everything after. + pub fn split_at(&self, cutoff_lsn: Lsn, after_oldest_lsn: &mut Accum) -> (Self, Self) { + let mut before_blocks = HashMap::new(); + let mut after_blocks = HashMap::new(); + + for (blknum, vec_map) in self.0.iter() { + let (before_versions, after_versions) = vec_map.split_at(&cutoff_lsn); + + if !before_versions.is_empty() { + let old = before_blocks.insert(*blknum, before_versions); + assert!(old.is_none()); + } + + if !after_versions.is_empty() { + let (first_lsn, _first_pv) = &after_versions.as_slice()[0]; + after_oldest_lsn.accum(std::cmp::min, *first_lsn); + + let old = after_blocks.insert(*blknum, after_versions); + assert!(old.is_none()); + } + } + + (Self(before_blocks), Self(after_blocks)) + } + + /// Iterate through [`PageVersion`]s in (block, lsn) order. + /// If a [`cutoff_lsn`] is set, only show versions with `lsn < cutoff_lsn` + pub fn ordered_page_version_iter(&self, cutoff_lsn: Option) -> OrderedPageVersionIter<'_> { + let mut ordered_blocks: Vec = self.0.keys().cloned().collect(); + ordered_blocks.sort_unstable(); + + let slice = ordered_blocks + .first() + .map(|&blknum| self.get_block_slice(blknum)) + .unwrap_or(EMPTY_SLICE); + + OrderedPageVersionIter { + page_versions: self, + ordered_blocks, + cur_block_idx: 0, + cutoff_lsn, + cur_slice_iter: slice.iter(), + } + } +} + +pub struct OrderedPageVersionIter<'a> { + page_versions: &'a PageVersions, + + ordered_blocks: Vec, + cur_block_idx: usize, + + cutoff_lsn: Option, + + cur_slice_iter: slice::Iter<'a, (Lsn, PageVersion)>, +} + +impl OrderedPageVersionIter<'_> { + fn is_lsn_before_cutoff(&self, lsn: &Lsn) -> bool { + if let Some(cutoff_lsn) = self.cutoff_lsn.as_ref() { + lsn < cutoff_lsn + } else { + true + } + } +} + +impl<'a> Iterator for OrderedPageVersionIter<'a> { + type Item = (u32, Lsn, &'a PageVersion); + + fn next(&mut self) -> Option { + loop { + if let Some((lsn, page_version)) = self.cur_slice_iter.next() { + if self.is_lsn_before_cutoff(lsn) { + let blknum = self.ordered_blocks[self.cur_block_idx]; + return Some((blknum, *lsn, page_version)); + } + } + + let next_block_idx = self.cur_block_idx + 1; + let blknum: u32 = *self.ordered_blocks.get(next_block_idx)?; + self.cur_block_idx = next_block_idx; + self.cur_slice_iter = self.page_versions.get_block_slice(blknum).iter(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const EMPTY_PAGE_VERSION: PageVersion = PageVersion { + page_image: None, + record: None, + }; + + #[test] + fn test_ordered_iter() { + let mut page_versions = PageVersions::default(); + const BLOCKS: u32 = 1000; + const LSNS: u64 = 50; + + for blknum in 0..BLOCKS { + for lsn in 0..LSNS { + let old = page_versions.append_or_update_last(blknum, Lsn(lsn), EMPTY_PAGE_VERSION); + assert!(old.is_none()); + } + } + + let mut iter = page_versions.ordered_page_version_iter(None); + for blknum in 0..BLOCKS { + for lsn in 0..LSNS { + let (actual_blknum, actual_lsn, _pv) = iter.next().unwrap(); + assert_eq!(actual_blknum, blknum); + assert_eq!(Lsn(lsn), actual_lsn); + } + } + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); // should be robust against excessive next() calls + + const CUTOFF_LSN: Lsn = Lsn(30); + let mut iter = page_versions.ordered_page_version_iter(Some(CUTOFF_LSN)); + for blknum in 0..BLOCKS { + for lsn in 0..CUTOFF_LSN.0 { + let (actual_blknum, actual_lsn, _pv) = iter.next().unwrap(); + assert_eq!(actual_blknum, blknum); + assert_eq!(Lsn(lsn), actual_lsn); + } + } + assert!(iter.next().is_none()); + assert!(iter.next().is_none()); // should be robust against excessive next() calls + } +} diff --git a/zenith_utils/src/lib.rs b/zenith_utils/src/lib.rs index ca26be5df2..96b3cf5066 100644 --- a/zenith_utils/src/lib.rs +++ b/zenith_utils/src/lib.rs @@ -8,6 +8,9 @@ pub mod lsn; /// SeqWait allows waiting for a future sequence number to arrive pub mod seqwait; +/// append only ordered map implemented with a Vec +pub mod vec_map; + // Async version of SeqWait. Currently unused. // pub mod seqwait_async; diff --git a/zenith_utils/src/vec_map.rs b/zenith_utils/src/vec_map.rs new file mode 100644 index 0000000000..4753010188 --- /dev/null +++ b/zenith_utils/src/vec_map.rs @@ -0,0 +1,239 @@ +use std::{cmp::Ordering, ops::RangeBounds}; + +/// Ordered map datastructure implemented in a Vec. +/// Append only - can only add keys that are larger than the +/// current max key. +#[derive(Clone, Debug)] +pub struct VecMap(Vec<(K, V)>); + +impl Default for VecMap { + fn default() -> Self { + VecMap(Default::default()) + } +} + +#[derive(Debug)] +pub struct InvalidKey; + +impl VecMap { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn as_slice(&self) -> &[(K, V)] { + self.0.as_slice() + } + + /// This function may panic if given a range where the lower bound is + /// greater than the upper bound. + pub fn slice_range>(&self, range: R) -> &[(K, V)] { + use std::ops::Bound::*; + + let binary_search = |k: &K| self.0.binary_search_by_key(&k, extract_key); + + let start_idx = match range.start_bound() { + Unbounded => 0, + Included(k) => binary_search(k).unwrap_or_else(std::convert::identity), + Excluded(k) => match binary_search(k) { + Ok(idx) => idx + 1, + Err(idx) => idx, + }, + }; + + let end_idx = match range.end_bound() { + Unbounded => self.0.len(), + Included(k) => match binary_search(k) { + Ok(idx) => idx + 1, + Err(idx) => idx, + }, + Excluded(k) => binary_search(k).unwrap_or_else(std::convert::identity), + }; + + &self.0[start_idx..end_idx] + } + + /// Add a key value pair to the map. + /// If [`key`] is less than or equal to the current maximum key + /// the pair will not be added and InvalidKey error will be returned. + pub fn append(&mut self, key: K, value: V) -> Result<(), InvalidKey> { + if let Some((last_key, _last_value)) = self.0.last() { + if &key <= last_key { + return Err(InvalidKey); + } + } + + self.0.push((key, value)); + Ok(()) + } + + /// Update the maximum key value pair or add a new key value pair to the map. + /// If [`key`] is less than the current maximum key no updates or additions + /// will occur and InvalidKey error will be returned. + pub fn append_or_update_last(&mut self, key: K, mut value: V) -> Result, InvalidKey> { + if let Some((last_key, last_value)) = self.0.last_mut() { + match key.cmp(last_key) { + Ordering::Less => return Err(InvalidKey), + Ordering::Equal => { + std::mem::swap(last_value, &mut value); + return Ok(Some(value)); + } + Ordering::Greater => {} + } + } + + self.0.push((key, value)); + Ok(None) + } + + /// Split the map into two. + /// + /// The left map contains everything before [`cutoff`] (exclusive). + /// Right map contains [`cutoff`] and everything after (inclusive). + pub fn split_at(&self, cutoff: &K) -> (Self, Self) + where + K: Clone, + V: Clone, + { + let split_idx = self + .0 + .binary_search_by_key(&cutoff, extract_key) + .unwrap_or_else(std::convert::identity); + + ( + VecMap(self.0[..split_idx].to_vec()), + VecMap(self.0[split_idx..].to_vec()), + ) + } +} + +fn extract_key(entry: &(K, V)) -> &K { + &entry.0 +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, ops::Bound}; + + use super::VecMap; + + #[test] + fn unbounded_range() { + let mut vec = VecMap::default(); + vec.append(0, ()).unwrap(); + + assert_eq!(vec.slice_range(0..0), &[]); + } + + #[test] + #[should_panic] + fn invalid_ordering_range() { + let mut vec = VecMap::default(); + vec.append(0, ()).unwrap(); + + #[allow(clippy::reversed_empty_ranges)] + vec.slice_range(1..0); + } + + #[test] + fn range_tests() { + let mut vec = VecMap::default(); + vec.append(0, ()).unwrap(); + vec.append(2, ()).unwrap(); + vec.append(4, ()).unwrap(); + + assert_eq!(vec.slice_range(0..0), &[]); + assert_eq!(vec.slice_range(0..1), &[(0, ())]); + assert_eq!(vec.slice_range(0..2), &[(0, ())]); + assert_eq!(vec.slice_range(0..3), &[(0, ()), (2, ())]); + + assert_eq!(vec.slice_range(..0), &[]); + assert_eq!(vec.slice_range(..1), &[(0, ())]); + + assert_eq!(vec.slice_range(..3), &[(0, ()), (2, ())]); + assert_eq!(vec.slice_range(..3), &[(0, ()), (2, ())]); + + assert_eq!(vec.slice_range(0..=0), &[(0, ())]); + assert_eq!(vec.slice_range(0..=1), &[(0, ())]); + assert_eq!(vec.slice_range(0..=2), &[(0, ()), (2, ())]); + assert_eq!(vec.slice_range(0..=3), &[(0, ()), (2, ())]); + + assert_eq!(vec.slice_range(..=0), &[(0, ())]); + assert_eq!(vec.slice_range(..=1), &[(0, ())]); + assert_eq!(vec.slice_range(..=2), &[(0, ()), (2, ())]); + assert_eq!(vec.slice_range(..=3), &[(0, ()), (2, ())]); + } + + struct BoundIter { + min: i32, + max: i32, + + next: Option>, + } + + impl BoundIter { + fn new(min: i32, max: i32) -> Self { + Self { + min, + max, + + next: Some(Bound::Unbounded), + } + } + } + + impl Iterator for BoundIter { + type Item = Bound; + + fn next(&mut self) -> Option { + let cur = self.next?; + + self.next = match &cur { + Bound::Unbounded => Some(Bound::Included(self.min)), + Bound::Included(x) => { + if *x >= self.max { + Some(Bound::Excluded(self.min)) + } else { + Some(Bound::Included(x + 1)) + } + } + Bound::Excluded(x) => { + if *x >= self.max { + None + } else { + Some(Bound::Excluded(x + 1)) + } + } + }; + + Some(cur) + } + } + + #[test] + fn range_exhaustive() { + let map: BTreeMap = (1..=7).step_by(2).map(|x| (x, ())).collect(); + let mut vec = VecMap::default(); + for &key in map.keys() { + vec.append(key, ()).unwrap(); + } + + const RANGE_MIN: i32 = 0; + const RANGE_MAX: i32 = 8; + for lower_bound in BoundIter::new(RANGE_MIN, RANGE_MAX) { + let ub_min = match lower_bound { + Bound::Unbounded => RANGE_MIN, + Bound::Included(x) => x, + Bound::Excluded(x) => x + 1, + }; + for upper_bound in BoundIter::new(ub_min, RANGE_MAX) { + let map_range: Vec<(i32, ())> = map + .range((lower_bound, upper_bound)) + .map(|(&x, _)| (x, ())) + .collect(); + let vec_slice = vec.slice_range((lower_bound, upper_bound)); + + assert_eq!(map_range, vec_slice); + } + } + } +} From c4bb6d78d4652d2e3524634bfb6c679cad956e18 Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Fri, 8 Oct 2021 10:44:54 -0700 Subject: [PATCH 03/46] pageserver - use VecMap for in memory segsizes --- .../src/layered_repository/inmemory_layer.rs | 55 +++++++++---------- zenith_utils/src/vec_map.rs | 52 ++++++++++++++++++ 2 files changed, 78 insertions(+), 29 deletions(-) diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index 6e3b00350d..b492ecd252 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -17,9 +17,9 @@ use bytes::Bytes; use log::*; use std::cmp::min; use std::collections::BTreeMap; -use std::ops::Bound::Included; use std::path::PathBuf; use std::sync::{Arc, RwLock}; +use zenith_utils::vec_map::VecMap; use zenith_utils::accum::Accum; use zenith_utils::lsn::Lsn; @@ -69,7 +69,7 @@ pub struct InMemoryLayerInner { /// so that determining the size never depends on the predecessor layer. For /// a non-blocky rel, 'segsizes' is not used and is always empty. /// - segsizes: BTreeMap, + segsizes: VecMap, /// Writes are only allowed when true. /// Set to false when this layer is in the process of being replaced. @@ -87,10 +87,10 @@ impl InMemoryLayerInner { fn get_seg_size(&self, lsn: Lsn) -> u32 { // Scan the BTreeMap backwards, starting from the given entry. - let mut iter = self.segsizes.range((Included(&Lsn(0)), Included(&lsn))); + let slice = self.segsizes.slice_range(..=lsn); // We make sure there is always at least one entry - if let Some((_entry_lsn, entry)) = iter.next_back() { + if let Some((_entry_lsn, entry)) = slice.last() { *entry } else { panic!("could not find seg size in in-memory layer"); @@ -277,7 +277,7 @@ impl Layer for InMemoryLayer { self.timelineid, self.seg, self.start_lsn, end_str ); - for (k, v) in inner.segsizes.iter() { + for (k, v) in inner.segsizes.as_slice() { println!("segsizes {}: {}", k, v); } @@ -339,9 +339,9 @@ impl InMemoryLayer { ); // The segment is initially empty, so initialize 'segsizes' with 0. - let mut segsizes = BTreeMap::new(); + let mut segsizes = VecMap::default(); if seg.rel.is_blocky() { - segsizes.insert(start_lsn, 0); + segsizes.append(start_lsn, 0).unwrap(); } Ok(InMemoryLayer { @@ -463,7 +463,7 @@ impl InMemoryLayer { } } - inner.segsizes.insert(lsn, newsize); + inner.segsizes.append_or_update_last(lsn, newsize).unwrap(); return Ok(newsize - oldsize); } } @@ -485,7 +485,7 @@ impl InMemoryLayer { let oldsize = inner.get_seg_size(lsn); assert!(segsize < oldsize); - let old = inner.segsizes.insert(lsn, segsize); + let old = inner.segsizes.append_or_update_last(lsn, segsize).unwrap(); if old.is_some() { // We already had an entry for this LSN. That's odd.. @@ -537,10 +537,10 @@ impl InMemoryLayer { ); // Copy the segment size at the start LSN from the predecessor layer. - let mut segsizes = BTreeMap::new(); + let mut segsizes = VecMap::default(); if seg.rel.is_blocky() { let size = src.get_seg_size(start_lsn)?; - segsizes.insert(start_lsn, size); + segsizes.append(start_lsn, size).unwrap(); } Ok(InMemoryLayer { @@ -607,21 +607,18 @@ impl InMemoryLayer { // Divide all the page versions into old and new // at the 'cutoff_lsn' point. - let mut before_segsizes = BTreeMap::new(); - let mut after_segsizes = BTreeMap::new(); let mut after_oldest_lsn: Accum = Accum(None); - for (lsn, size) in inner.segsizes.iter() { - if *lsn > cutoff_lsn { - after_segsizes.insert(*lsn, *size); - after_oldest_lsn.accum(min, *lsn); - } else { - before_segsizes.insert(*lsn, *size); - } + + let cutoff_lsn_exclusive = Lsn(cutoff_lsn.0 + 1); + + let (before_segsizes, mut after_segsizes) = inner.segsizes.split_at(&cutoff_lsn_exclusive); + if let Some((lsn, _size)) = after_segsizes.as_slice().first() { + after_oldest_lsn.accum(min, *lsn); } let (before_page_versions, after_page_versions) = inner .page_versions - .split_at(Lsn(cutoff_lsn.0 + 1), &mut after_oldest_lsn); + .split_at(cutoff_lsn_exclusive, &mut after_oldest_lsn); let frozen = Arc::new(InMemoryLayer { conf: self.conf, @@ -655,7 +652,7 @@ impl InMemoryLayer { // so we can just replace it assert!(new_inner.page_versions.is_empty()); new_inner.page_versions = after_page_versions; - new_inner.segsizes.append(&mut after_segsizes); + new_inner.segsizes.extend(&mut after_segsizes).unwrap(); Some(Arc::new(new_open)) } else { @@ -694,6 +691,8 @@ impl InMemoryLayer { assert!(!inner.writeable); if let Some(drop_lsn) = inner.drop_lsn { + let segsizes_map: BTreeMap = + inner.segsizes.as_slice().iter().cloned().collect(); let delta_layer = DeltaLayer::create( self.conf, self.timelineid, @@ -703,7 +702,7 @@ impl InMemoryLayer { drop_lsn, true, inner.page_versions.ordered_page_version_iter(None), - inner.segsizes.clone(), + segsizes_map, )?; trace!( "freeze: created delta layer for dropped segment {} {}-{}", @@ -716,12 +715,10 @@ impl InMemoryLayer { let end_lsn = self.end_lsn.unwrap(); - let mut before_segsizes = BTreeMap::new(); - for (lsn, size) in inner.segsizes.iter() { - if *lsn <= end_lsn { - before_segsizes.insert(*lsn, *size); - } - } + let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1)); + let before_segsizes: BTreeMap = + before_segsizes.as_slice().iter().cloned().collect(); + let mut before_page_versions = inner.page_versions.ordered_page_version_iter(Some(end_lsn)); let mut frozen_layers: Vec> = Vec::new(); diff --git a/zenith_utils/src/vec_map.rs b/zenith_utils/src/vec_map.rs index 4753010188..5d14245de2 100644 --- a/zenith_utils/src/vec_map.rs +++ b/zenith_utils/src/vec_map.rs @@ -104,6 +104,24 @@ impl VecMap { VecMap(self.0[split_idx..].to_vec()), ) } + + /// Move items from [`other`] to the end of [`self`], leaving [`other`] empty. + /// If any keys in [`other`] is less than or equal to any key in [`self`], + /// [`InvalidKey`] error will be returned and no mutation will occur. + pub fn extend(&mut self, other: &mut Self) -> Result<(), InvalidKey> { + let self_last_opt = self.0.last().map(extract_key); + let other_first_opt = other.0.last().map(extract_key); + + if let (Some(self_last), Some(other_first)) = (self_last_opt, other_first_opt) { + if self_last >= other_first { + return Err(InvalidKey); + } + } + + self.0.append(&mut other.0); + + Ok(()) + } } fn extract_key(entry: &(K, V)) -> &K { @@ -236,4 +254,38 @@ mod tests { } } } + + #[test] + fn extend() { + let mut left = VecMap::default(); + left.append(0, ()).unwrap(); + assert_eq!(left.as_slice(), &[(0, ())]); + + let mut empty = VecMap::default(); + left.extend(&mut empty).unwrap(); + assert_eq!(left.as_slice(), &[(0, ())]); + assert_eq!(empty.as_slice(), &[]); + + let mut right = VecMap::default(); + right.append(1, ()).unwrap(); + + left.extend(&mut right).unwrap(); + + assert_eq!(left.as_slice(), &[(0, ()), (1, ())]); + assert_eq!(right.as_slice(), &[]); + + let mut zero_map = VecMap::default(); + zero_map.append(0, ()).unwrap(); + + left.extend(&mut zero_map).unwrap_err(); + assert_eq!(left.as_slice(), &[(0, ()), (1, ())]); + assert_eq!(zero_map.as_slice(), &[(0, ())]); + + let mut one_map = VecMap::default(); + one_map.append(1, ()).unwrap(); + + left.extend(&mut one_map).unwrap_err(); + assert_eq!(left.as_slice(), &[(0, ()), (1, ())]); + assert_eq!(one_map.as_slice(), &[(1, ())]); + } } From c356030660fbe24e69c3590786a3431b7a9760fc Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Fri, 8 Oct 2021 11:28:00 -0700 Subject: [PATCH 04/46] pageserver - use VecMap for delta metadata & sizes --- .../src/layered_repository/delta_layer.rs | 61 ++++++++++--------- .../src/layered_repository/inmemory_layer.rs | 11 +--- zenith_utils/src/vec_map.rs | 4 +- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index 4ea415c899..b96ea88920 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -48,7 +48,7 @@ use crate::{ZTenantId, ZTimelineId}; use anyhow::{bail, ensure, Result}; use log::*; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use zenith_utils::vec_map::VecMap; // avoid binding to Write (conflicts with std::io::Write) // while being able to use std::fmt::Write's methods use std::fmt::Write as _; @@ -141,10 +141,10 @@ pub struct DeltaLayerInner { /// All versions of all pages in the file are are kept here. /// Indexed by block number and LSN. - page_version_metas: BTreeMap<(u32, Lsn), BlobRange>, + page_version_metas: VecMap<(u32, Lsn), BlobRange>, /// `relsizes` tracks the size of the relation at different points in time. - relsizes: BTreeMap, + relsizes: VecMap, } impl Layer for DeltaLayer { @@ -215,10 +215,12 @@ impl Layer for DeltaLayer { // Scan the metadata BTreeMap backwards, starting from the given entry. let minkey = (blknum, Lsn(0)); let maxkey = (blknum, lsn); - let mut iter = inner + let iter = inner .page_version_metas - .range((Included(&minkey), Included(&maxkey))); - while let Some(((_blknum, _entry_lsn), blob_range)) = iter.next_back() { + .slice_range((Included(&minkey), Included(&maxkey))) + .iter() + .rev(); + for ((_blknum, _lsn), blob_range) in iter { let pv = PageVersion::des(&read_blob(&page_version_reader, blob_range)?)?; if let Some(img) = pv.page_image { @@ -262,15 +264,15 @@ impl Layer for DeltaLayer { // Scan the BTreeMap backwards, starting from the given entry. let inner = self.load()?; - let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn))); + let slice = inner + .relsizes + .slice_range((Included(&Lsn(0)), Included(&lsn))); - let result; - if let Some((_entry_lsn, entry)) = iter.next_back() { - result = *entry; + if let Some((_entry_lsn, entry)) = slice.last() { + Ok(*entry) } else { - bail!("could not find seg size in delta layer"); + Err(anyhow::anyhow!("could not find seg size in delta layer")) } - Ok(result) } /// Does this segment exist at given LSN? @@ -290,8 +292,8 @@ impl Layer for DeltaLayer { /// fn unload(&self) -> Result<()> { let mut inner = self.inner.lock().unwrap(); - inner.page_version_metas = BTreeMap::new(); - inner.relsizes = BTreeMap::new(); + inner.page_version_metas = VecMap::default(); + inner.relsizes = VecMap::default(); inner.loaded = false; Ok(()) } @@ -317,13 +319,13 @@ impl Layer for DeltaLayer { println!("--- relsizes ---"); let inner = self.load()?; - for (k, v) in inner.relsizes.iter() { + for (k, v) in inner.relsizes.as_slice() { println!(" {}: {}", k, v); } println!("--- page versions ---"); let (_path, book) = self.open_book()?; let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?; - for ((blk, lsn), blob_range) in inner.page_version_metas.iter() { + for ((blk, lsn), blob_range) in inner.page_version_metas.as_slice() { let mut desc = String::new(); let buf = read_blob(&chapter, blob_range)?; @@ -381,7 +383,7 @@ impl DeltaLayer { end_lsn: Lsn, dropped: bool, page_versions: impl Iterator, - relsizes: BTreeMap, + relsizes: VecMap, ) -> Result { if seg.rel.is_blocky() { assert!(!relsizes.is_empty()); @@ -397,7 +399,7 @@ impl DeltaLayer { dropped, inner: Mutex::new(DeltaLayerInner { loaded: true, - page_version_metas: BTreeMap::new(), + page_version_metas: VecMap::default(), relsizes, }), }; @@ -420,22 +422,23 @@ impl DeltaLayer { let buf = PageVersion::ser(page_version)?; let blob_range = page_version_writer.write_blob(&buf)?; - let old = inner.page_version_metas.insert((blknum, lsn), blob_range); - - assert!(old.is_none()); + inner + .page_version_metas + .append((blknum, lsn), blob_range) + .unwrap(); } let book = page_version_writer.close()?; // Write out page versions let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER); - let buf = BTreeMap::ser(&inner.page_version_metas)?; + let buf = VecMap::ser(&inner.page_version_metas)?; chapter.write_all(&buf)?; let book = chapter.close()?; // and relsizes to separate chapter let mut chapter = book.new_chapter(REL_SIZES_CHAPTER); - let buf = BTreeMap::ser(&inner.relsizes)?; + let buf = VecMap::ser(&inner.relsizes)?; chapter.write_all(&buf)?; let book = chapter.close()?; @@ -522,10 +525,10 @@ impl DeltaLayer { } let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?; - let page_version_metas = BTreeMap::des(&chapter)?; + let page_version_metas = VecMap::des(&chapter)?; let chapter = book.read_chapter(REL_SIZES_CHAPTER)?; - let relsizes = BTreeMap::des(&chapter)?; + let relsizes = VecMap::des(&chapter)?; debug!("loaded from {}", &path.display()); @@ -555,8 +558,8 @@ impl DeltaLayer { dropped: filename.dropped, inner: Mutex::new(DeltaLayerInner { loaded: false, - page_version_metas: BTreeMap::new(), - relsizes: BTreeMap::new(), + page_version_metas: VecMap::default(), + relsizes: VecMap::default(), }), } } @@ -578,8 +581,8 @@ impl DeltaLayer { dropped: summary.dropped, inner: Mutex::new(DeltaLayerInner { loaded: false, - page_version_metas: BTreeMap::new(), - relsizes: BTreeMap::new(), + page_version_metas: VecMap::default(), + relsizes: VecMap::default(), }), }) } diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index b492ecd252..5f5307fec0 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -16,7 +16,6 @@ use anyhow::{bail, ensure, Result}; use bytes::Bytes; use log::*; use std::cmp::min; -use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::{Arc, RwLock}; use zenith_utils::vec_map::VecMap; @@ -691,8 +690,6 @@ impl InMemoryLayer { assert!(!inner.writeable); if let Some(drop_lsn) = inner.drop_lsn { - let segsizes_map: BTreeMap = - inner.segsizes.as_slice().iter().cloned().collect(); let delta_layer = DeltaLayer::create( self.conf, self.timelineid, @@ -702,7 +699,7 @@ impl InMemoryLayer { drop_lsn, true, inner.page_versions.ordered_page_version_iter(None), - segsizes_map, + inner.segsizes.clone(), )?; trace!( "freeze: created delta layer for dropped segment {} {}-{}", @@ -715,15 +712,13 @@ impl InMemoryLayer { let end_lsn = self.end_lsn.unwrap(); - let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1)); - let before_segsizes: BTreeMap = - before_segsizes.as_slice().iter().cloned().collect(); - let mut before_page_versions = inner.page_versions.ordered_page_version_iter(Some(end_lsn)); let mut frozen_layers: Vec> = Vec::new(); if self.start_lsn != end_lsn { + let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1)); + // Write the page versions before the cutoff to disk. let delta_layer = DeltaLayer::create( self.conf, diff --git a/zenith_utils/src/vec_map.rs b/zenith_utils/src/vec_map.rs index 5d14245de2..0fd33bf489 100644 --- a/zenith_utils/src/vec_map.rs +++ b/zenith_utils/src/vec_map.rs @@ -1,9 +1,11 @@ use std::{cmp::Ordering, ops::RangeBounds}; +use serde::{Deserialize, Serialize}; + /// Ordered map datastructure implemented in a Vec. /// Append only - can only add keys that are larger than the /// current max key. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct VecMap(Vec<(K, V)>); impl Default for VecMap { From 0baf4bc796a11ad1001c18eda71b19c51e8b2360 Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Sat, 9 Oct 2021 01:34:03 -0700 Subject: [PATCH 05/46] fix `cargo doc` complaints --- zenith_metrics/src/lib.rs | 2 +- zenith_utils/src/vec_map.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/zenith_metrics/src/lib.rs b/zenith_metrics/src/lib.rs index b01e80d873..59a8a31c9e 100644 --- a/zenith_metrics/src/lib.rs +++ b/zenith_metrics/src/lib.rs @@ -46,7 +46,7 @@ pub fn set_common_metrics_prefix(prefix: &'static str) { } /// Prepends a prefix to a common metric name so they are distinguished between -/// different services, see https://github.com/zenithdb/zenith/pull/681 +/// different services, see /// A call to set_common_metrics_prefix() is necessary prior to calling this. pub fn new_common_metric_name(unprefixed_metric_name: &str) -> String { // Not unwrap() because metrics may be initialized after multiple threads have been started. diff --git a/zenith_utils/src/vec_map.rs b/zenith_utils/src/vec_map.rs index 0fd33bf489..4e2c827b47 100644 --- a/zenith_utils/src/vec_map.rs +++ b/zenith_utils/src/vec_map.rs @@ -55,7 +55,7 @@ impl VecMap { } /// Add a key value pair to the map. - /// If [`key`] is less than or equal to the current maximum key + /// If `key` is less than or equal to the current maximum key /// the pair will not be added and InvalidKey error will be returned. pub fn append(&mut self, key: K, value: V) -> Result<(), InvalidKey> { if let Some((last_key, _last_value)) = self.0.last() { @@ -69,7 +69,7 @@ impl VecMap { } /// Update the maximum key value pair or add a new key value pair to the map. - /// If [`key`] is less than the current maximum key no updates or additions + /// If `key` is less than the current maximum key no updates or additions /// will occur and InvalidKey error will be returned. pub fn append_or_update_last(&mut self, key: K, mut value: V) -> Result, InvalidKey> { if let Some((last_key, last_value)) = self.0.last_mut() { @@ -89,8 +89,8 @@ impl VecMap { /// Split the map into two. /// - /// The left map contains everything before [`cutoff`] (exclusive). - /// Right map contains [`cutoff`] and everything after (inclusive). + /// The left map contains everything before `cutoff` (exclusive). + /// Right map contains `cutoff` and everything after (inclusive). pub fn split_at(&self, cutoff: &K) -> (Self, Self) where K: Clone, @@ -107,9 +107,9 @@ impl VecMap { ) } - /// Move items from [`other`] to the end of [`self`], leaving [`other`] empty. - /// If any keys in [`other`] is less than or equal to any key in [`self`], - /// [`InvalidKey`] error will be returned and no mutation will occur. + /// Move items from `other` to the end of `self`, leaving `other` empty. + /// If any keys in `other` is less than or equal to any key in `self`, + /// `InvalidKey` error will be returned and no mutation will occur. pub fn extend(&mut self, other: &mut Self) -> Result<(), InvalidKey> { let self_last_opt = self.0.last().map(extract_key); let other_first_opt = other.0.last().map(extract_key); From 3f0ebc6a404631865f3f03ad33f6ccf10ae4b07a Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Fri, 8 Oct 2021 15:50:27 -0700 Subject: [PATCH 06/46] pageserver - move early File::open call --- pageserver/src/layered_repository.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 0726baa86d..a86565ac59 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -1329,8 +1329,6 @@ impl LayeredTimeline { last_record_lsn ); - let timeline_dir = File::open(self.conf.timeline_path(&self.timelineid, &self.tenantid))?; - // Take the in-memory layer with the oldest WAL record. If it's older // than the threshold, write it out to disk as a new image and delta file. // Repeat until all remaining in-memory layers are within the threshold. @@ -1422,6 +1420,8 @@ impl LayeredTimeline { if created_historics { // We must fsync the timeline dir to ensure the directory entries for // new layer files are durable + let timeline_dir = + File::open(self.conf.timeline_path(&self.timelineid, &self.tenantid))?; timeline_dir.sync_all()?; } From bf58f7f649b3eb4758155e43dadbefd60c62b119 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sat, 9 Oct 2021 19:23:57 +0300 Subject: [PATCH 07/46] Expose certain layered repository structs to reuse in relish storage (#688) --- pageserver/src/layered_repository.rs | 79 +++++++++++++------ .../src/layered_repository/delta_layer.rs | 58 ++++++-------- pageserver/src/layered_repository/filename.rs | 12 +-- .../src/layered_repository/image_layer.rs | 57 ++++++------- .../src/layered_repository/inmemory_layer.rs | 36 ++++++--- .../src/layered_repository/storage_layer.rs | 4 - pageserver/src/relish_storage.rs | 6 +- pageserver/src/relish_storage/rust_s3.rs | 7 +- pageserver/src/repository.rs | 4 +- 9 files changed, 138 insertions(+), 125 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index a86565ac59..bb9d8af112 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -111,6 +111,9 @@ lazy_static! { .expect("failed to define a metric"); } +/// The name of the metadata file pageserver creates per timeline. +pub const METADATA_FILE_NAME: &str = "metadata"; + /// /// Repository consists of multiple timelines. Keep them in a hash table. /// @@ -252,7 +255,16 @@ impl LayeredRepository { )?; // List the layers on disk, and load them into the layer map - timeline.load_layer_map(disk_consistent_lsn)?; + let _loaded_layers = timeline.load_layer_map(disk_consistent_lsn)?; + if self.upload_relishes { + schedule_timeline_upload(()); + // schedule_timeline_upload( + // self.tenantid, + // timelineid, + // loaded_layers, + // disk_consistent_lsn, + // ); + } // needs to be after load_layer_map timeline.init_current_logical_size()?; @@ -351,9 +363,8 @@ impl LayeredRepository { tenantid: ZTenantId, data: &TimelineMetadata, first_save: bool, - ) -> Result { - let timeline_path = conf.timeline_path(&timelineid, &tenantid); - let path = timeline_path.join("metadata"); + ) -> Result<()> { + let path = metadata_path(conf, timelineid, tenantid); // use OpenOptions to ensure file presence is consistent with first_save let mut file = OpenOptions::new() .write(true) @@ -377,11 +388,15 @@ impl LayeredRepository { // fsync the parent directory to ensure the directory entry is durable if first_save { - let timeline_dir = File::open(&timeline_path)?; + let timeline_dir = File::open( + &path + .parent() + .expect("Metadata should always have a parent dir"), + )?; timeline_dir.sync_all()?; } - Ok(path) + Ok(()) } fn load_metadata( @@ -389,7 +404,7 @@ impl LayeredRepository { timelineid: ZTimelineId, tenantid: ZTenantId, ) -> Result { - let path = conf.timeline_path(&timelineid, &tenantid).join("metadata"); + let path = metadata_path(conf, timelineid, tenantid); let metadata_bytes = std::fs::read(&path)?; ensure!(metadata_bytes.len() == METADATA_MAX_SAFE_SIZE); @@ -469,7 +484,7 @@ impl LayeredRepository { let timeline = self.get_timeline_locked(*timelineid, &mut *timelines)?; if let Some(ancestor_timeline) = &timeline.ancestor_timeline { - // If target_timeline is specified, we only need to know branchpoints of its childs + // If target_timeline is specified, we only need to know branchpoints of its children if let Some(timelineid) = target_timelineid { if ancestor_timeline.timelineid == timelineid { all_branchpoints @@ -1023,9 +1038,10 @@ impl LayeredTimeline { } /// - /// Scan the timeline directory to populate the layer map + /// Scan the timeline directory to populate the layer map. + /// Returns all timeline-related files that were found and loaded. /// - fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> { + fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result> { info!( "loading layer map for timeline {} into memory", self.timelineid @@ -1035,9 +1051,9 @@ impl LayeredTimeline { filename::list_files(self.conf, self.timelineid, self.tenantid)?; let timeline_path = self.conf.timeline_path(&self.timelineid, &self.tenantid); - + let mut local_layers = Vec::with_capacity(imgfilenames.len() + deltafilenames.len()); // First create ImageLayer structs for each image file. - for filename in imgfilenames.iter() { + for filename in &imgfilenames { if filename.lsn > disk_consistent_lsn { warn!( "found future image layer {} on timeline {}", @@ -1056,11 +1072,11 @@ impl LayeredTimeline { layer.get_start_lsn(), self.timelineid ); + local_layers.push(layer.path()); layers.insert_historic(Arc::new(layer)); } - // Then for the Delta files. - for filename in deltafilenames.iter() { + for filename in &deltafilenames { ensure!(filename.start_lsn < filename.end_lsn); if filename.end_lsn > disk_consistent_lsn { warn!( @@ -1079,10 +1095,11 @@ impl LayeredTimeline { layer.filename().display(), self.timelineid, ); + local_layers.push(layer.path()); layers.insert_historic(Arc::new(layer)); } - Ok(()) + Ok(local_layers) } /// @@ -1341,7 +1358,7 @@ impl LayeredTimeline { let mut disk_consistent_lsn = last_record_lsn; let mut created_historics = false; - + let mut layer_uploads = Vec::new(); while let Some((oldest_layer, oldest_generation)) = layers.peek_oldest_open() { let oldest_pending_lsn = oldest_layer.get_oldest_pending_lsn(); @@ -1403,8 +1420,13 @@ impl LayeredTimeline { layers.remove_historic(frozen.clone()); // Add the historics to the LayerMap - for n in new_historics { - layers.insert_historic(n); + for delta_layer in new_historics.delta_layers { + layer_uploads.push(delta_layer.path()); + layers.insert_historic(Arc::new(delta_layer)); + } + for image_layer in new_historics.image_layers { + layer_uploads.push(image_layer.path()); + layers.insert_historic(Arc::new(image_layer)); } } @@ -1449,7 +1471,7 @@ impl LayeredTimeline { ancestor_timeline: ancestor_timelineid, ancestor_lsn: self.ancestor_lsn, }; - let _metadata_path = LayeredRepository::save_metadata( + LayeredRepository::save_metadata( self.conf, self.timelineid, self.tenantid, @@ -1458,12 +1480,10 @@ impl LayeredTimeline { )?; if self.upload_relishes { schedule_timeline_upload(()) - // schedule_timeline_upload(LocalTimeline { - // tenant_id: self.tenantid, - // timeline_id: self.timelineid, - // metadata_path, - // image_layers: image_layer_uploads, - // delta_layers: delta_layer_uploads, + // schedule_timeline_upload( + // self.tenantid, + // self.timelineid, + // layer_uploads, // disk_consistent_lsn, // }); } @@ -1896,6 +1916,15 @@ pub fn dump_layerfile_from_path(path: &Path) -> Result<()> { Ok(()) } +fn metadata_path( + conf: &'static PageServerConf, + timelineid: ZTimelineId, + tenantid: ZTenantId, +) -> PathBuf { + conf.timeline_path(&timelineid, &tenantid) + .join(METADATA_FILE_NAME) +} + /// Add a suffix to a layer file's name: .{num}.old /// Uses the first available num (starts at 0) fn rename_to_backup(path: PathBuf) -> anyhow::Result<()> { diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index b96ea88920..e93eddb7e6 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -169,29 +169,7 @@ impl Layer for DeltaLayer { } fn filename(&self) -> PathBuf { - PathBuf::from( - DeltaFileName { - seg: self.seg, - start_lsn: self.start_lsn, - end_lsn: self.end_lsn, - dropped: self.dropped, - } - .to_string(), - ) - } - - fn path(&self) -> Option { - Some(Self::path_for( - &self.path_or_conf, - self.timelineid, - self.tenantid, - &DeltaFileName { - seg: self.seg, - start_lsn: self.start_lsn, - end_lsn: self.end_lsn, - dropped: self.dropped, - }, - )) + PathBuf::from(self.layer_name().to_string()) } /// Look up given page in the cache. @@ -300,9 +278,7 @@ impl Layer for DeltaLayer { fn delete(&self) -> Result<()> { // delete underlying file - if let Some(path) = self.path() { - fs::remove_file(path)?; - } + fs::remove_file(self.path())?; Ok(()) } @@ -406,9 +382,7 @@ impl DeltaLayer { let mut inner = delta_layer.inner.lock().unwrap(); // Write the in-memory btreemaps into a file - let path = delta_layer - .path() - .expect("DeltaLayer is supposed to have a layer path on disk"); + let path = delta_layer.path(); // Note: This overwrites any existing file. There shouldn't be any. // FIXME: throw an error instead? @@ -472,12 +446,7 @@ impl DeltaLayer { &self.path_or_conf, self.timelineid, self.tenantid, - &DeltaFileName { - seg: self.seg, - start_lsn: self.start_lsn, - end_lsn: self.end_lsn, - dropped: self.dropped, - }, + &self.layer_name(), ); let file = File::open(&path)?; @@ -586,4 +555,23 @@ impl DeltaLayer { }), }) } + + fn layer_name(&self) -> DeltaFileName { + DeltaFileName { + seg: self.seg, + start_lsn: self.start_lsn, + end_lsn: self.end_lsn, + dropped: self.dropped, + } + } + + /// Path to the layer file in pageserver workdir. + pub fn path(&self) -> PathBuf { + Self::path_for( + &self.path_or_conf, + self.timelineid, + self.tenantid, + &self.layer_name(), + ) + } } diff --git a/pageserver/src/layered_repository/filename.rs b/pageserver/src/layered_repository/filename.rs index 50bfe2977e..afa106f939 100644 --- a/pageserver/src/layered_repository/filename.rs +++ b/pageserver/src/layered_repository/filename.rs @@ -13,6 +13,8 @@ use anyhow::Result; use log::*; use zenith_utils::lsn::Lsn; +use super::METADATA_FILE_NAME; + // Note: LayeredTimeline::load_layer_map() relies on this sort order #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] pub struct DeltaFileName { @@ -35,7 +37,7 @@ impl DeltaFileName { /// Parse a string as a delta file name. Returns None if the filename does not /// match the expected pattern. /// - pub fn from_str(fname: &str) -> Option { + pub fn parse_str(fname: &str) -> Option { let rel; let mut parts; if let Some(rest) = fname.strip_prefix("rel_") { @@ -168,7 +170,7 @@ impl ImageFileName { /// Parse a string as an image file name. Returns None if the filename does not /// match the expected pattern. /// - pub fn from_str(fname: &str) -> Option { + pub fn parse_str(fname: &str) -> Option { let rel; let mut parts; if let Some(rest) = fname.strip_prefix("rel_") { @@ -286,11 +288,11 @@ pub fn list_files( let fname = direntry?.file_name(); let fname = fname.to_str().unwrap(); - if let Some(deltafilename) = DeltaFileName::from_str(fname) { + if let Some(deltafilename) = DeltaFileName::parse_str(fname) { deltafiles.push(deltafilename); - } else if let Some(imgfilename) = ImageFileName::from_str(fname) { + } else if let Some(imgfilename) = ImageFileName::parse_str(fname) { imgfiles.push(imgfilename); - } else if fname == "metadata" || fname == "ancestor" || fname.ends_with(".old") { + } else if fname == METADATA_FILE_NAME || fname == "ancestor" || fname.ends_with(".old") { // ignore these } else { warn!("unrecognized filename in timeline dir: {}", fname); diff --git a/pageserver/src/layered_repository/image_layer.rs b/pageserver/src/layered_repository/image_layer.rs index a9487a02d4..744f793558 100644 --- a/pageserver/src/layered_repository/image_layer.rs +++ b/pageserver/src/layered_repository/image_layer.rs @@ -114,25 +114,7 @@ pub struct ImageLayerInner { impl Layer for ImageLayer { fn filename(&self) -> PathBuf { - PathBuf::from( - ImageFileName { - seg: self.seg, - lsn: self.lsn, - } - .to_string(), - ) - } - - fn path(&self) -> Option { - Some(Self::path_for( - &self.path_or_conf, - self.timelineid, - self.tenantid, - &ImageFileName { - seg: self.seg, - lsn: self.lsn, - }, - )) + PathBuf::from(self.layer_name().to_string()) } fn get_timeline_id(&self) -> ZTimelineId { @@ -222,9 +204,7 @@ impl Layer for ImageLayer { fn delete(&self) -> Result<()> { // delete underlying file - if let Some(path) = self.path() { - fs::remove_file(path)?; - } + fs::remove_file(self.path())?; Ok(()) } @@ -300,9 +280,7 @@ impl ImageLayer { let inner = layer.inner.lock().unwrap(); // Write the images into a file - let path = layer - .path() - .expect("ImageLayer is supposed to have a layer path on disk"); + let path = layer.path(); // Note: This overwrites any existing file. There shouldn't be any. // FIXME: throw an error instead? let file = File::create(&path)?; @@ -340,7 +318,7 @@ impl ImageLayer { let writer = book.close()?; writer.get_ref().sync_all()?; - trace!("saved {}", &path.display()); + trace!("saved {}", path.display()); drop(inner); @@ -445,15 +423,7 @@ impl ImageLayer { } fn open_book(&self) -> Result<(PathBuf, Book)> { - let path = Self::path_for( - &self.path_or_conf, - self.timelineid, - self.tenantid, - &ImageFileName { - seg: self.seg, - lsn: self.lsn, - }, - ); + let path = self.path(); let file = File::open(&path)?; let book = Book::new(file)?; @@ -500,4 +470,21 @@ impl ImageLayer { }), }) } + + fn layer_name(&self) -> ImageFileName { + ImageFileName { + seg: self.seg, + lsn: self.lsn, + } + } + + /// Path to the layer file in pageserver workdir. + pub fn path(&self) -> PathBuf { + Self::path_for( + &self.path_or_conf, + self.timelineid, + self.tenantid, + &self.layer_name(), + ) + } } diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index 5f5307fec0..214c392fb6 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -124,10 +124,6 @@ impl Layer for InMemoryLayer { PathBuf::from(format!("inmem-{}", delta_filename)) } - fn path(&self) -> Option { - None - } - fn get_timeline_id(&self) -> ZTimelineId { self.timelineid } @@ -309,6 +305,18 @@ pub struct FreezeLayers { pub open: Option>, } +/// A result of an inmemory layer data being written to disk. +pub struct LayersOnDisk { + pub delta_layers: Vec, + pub image_layers: Vec, +} + +impl LayersOnDisk { + pub fn is_empty(&self) -> bool { + self.delta_layers.is_empty() && self.image_layers.is_empty() + } +} + impl InMemoryLayer { fn assert_not_frozen(&self) { assert!(self.end_lsn.is_none()); @@ -669,7 +677,7 @@ impl InMemoryLayer { /// WAL records between start and end LSN. (The delta layer is not needed /// when a new relish is created with a single LSN, so that the start and /// end LSN are the same.) - pub fn write_to_disk(&self, timeline: &LayeredTimeline) -> Result>> { + pub fn write_to_disk(&self, timeline: &LayeredTimeline) -> Result { trace!( "write_to_disk {} end_lsn is {} get_end_lsn is {}", self.filename().display(), @@ -678,7 +686,7 @@ impl InMemoryLayer { ); // Grab the lock in read-mode. We hold it over the I/O, but because this - // layer is not writeable anymore, no one should be trying to aquire the + // layer is not writeable anymore, no one should be trying to acquire the // write lock on it, so we shouldn't block anyone. There's one exception // though: another thread might have grabbed a reference to this layer // in `get_layer_for_write' just before the checkpointer called @@ -707,15 +715,17 @@ impl InMemoryLayer { self.start_lsn, drop_lsn ); - return Ok(vec![Arc::new(delta_layer)]); + return Ok(LayersOnDisk { + delta_layers: vec![delta_layer], + image_layers: Vec::new(), + }); } let end_lsn = self.end_lsn.unwrap(); let mut before_page_versions = inner.page_versions.ordered_page_version_iter(Some(end_lsn)); - let mut frozen_layers: Vec> = Vec::new(); - + let mut delta_layers = Vec::new(); if self.start_lsn != end_lsn { let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1)); @@ -731,7 +741,7 @@ impl InMemoryLayer { before_page_versions, before_segsizes, )?; - frozen_layers.push(Arc::new(delta_layer)); + delta_layers.push(delta_layer); trace!( "freeze: created delta layer {} {}-{}", self.seg, @@ -746,9 +756,11 @@ impl InMemoryLayer { // Write a new base image layer at the cutoff point let image_layer = ImageLayer::create_from_src(self.conf, timeline, self, end_lsn)?; - frozen_layers.push(Arc::new(image_layer)); trace!("freeze: created image layer {} at {}", self.seg, end_lsn); - Ok(frozen_layers) + Ok(LayersOnDisk { + delta_layers, + image_layers: vec![image_layer], + }) } } diff --git a/pageserver/src/layered_repository/storage_layer.rs b/pageserver/src/layered_repository/storage_layer.rs index a107d63b40..c49fbbdd99 100644 --- a/pageserver/src/layered_repository/storage_layer.rs +++ b/pageserver/src/layered_repository/storage_layer.rs @@ -123,10 +123,6 @@ pub trait Layer: Send + Sync { /// Is the segment represented by this layer dropped by PostgreSQL? fn is_dropped(&self) -> bool; - /// Gets the physical location of the layer on disk. - /// Some layers, such as in-memory, might not have the location. - fn path(&self) -> Option; - /// Filename used to store this layer on disk. (Even in-memory layers /// implement this, to print a handy unique identifier for the layer for /// log messages, even though they're never not on disk.) diff --git a/pageserver/src/relish_storage.rs b/pageserver/src/relish_storage.rs index a687abe489..70d75c34bf 100644 --- a/pageserver/src/relish_storage.rs +++ b/pageserver/src/relish_storage.rs @@ -12,14 +12,12 @@ mod rust_s3; /// local page server layer files with external storage. mod synced_storage; -use std::path::Path; -use std::thread; +use std::{path::Path, thread}; use anyhow::Context; -use self::local_fs::LocalFs; pub use self::synced_storage::schedule_timeline_upload; -use crate::relish_storage::rust_s3::RustS3; +use self::{local_fs::LocalFs, rust_s3::RustS3}; use crate::{PageServerConf, RelishStorageKind}; pub fn run_storage_sync_thread( diff --git a/pageserver/src/relish_storage/rust_s3.rs b/pageserver/src/relish_storage/rust_s3.rs index e98bf8949f..dc29752e99 100644 --- a/pageserver/src/relish_storage/rust_s3.rs +++ b/pageserver/src/relish_storage/rust_s3.rs @@ -5,9 +5,10 @@ use std::path::Path; use anyhow::Context; use s3::{bucket::Bucket, creds::Credentials, region::Region}; -use crate::{relish_storage::strip_workspace_prefix, S3Config}; - -use super::RelishStorage; +use crate::{ + relish_storage::{strip_workspace_prefix, RelishStorage}, + S3Config, +}; const S3_FILE_SEPARATOR: char = '/'; diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index e4412aef6f..e8a7952d77 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -209,7 +209,7 @@ impl WALRecord { #[cfg(test)] mod tests { use super::*; - use crate::layered_repository::LayeredRepository; + use crate::layered_repository::{LayeredRepository, METADATA_FILE_NAME}; use crate::walredo::{WalRedoError, WalRedoManager}; use crate::PageServerConf; use hex_literal::hex; @@ -728,7 +728,7 @@ mod tests { repo.create_empty_timeline(TIMELINE_ID)?; drop(repo); - let metadata_path = harness.timeline_path(&TIMELINE_ID).join("metadata"); + let metadata_path = harness.timeline_path(&TIMELINE_ID).join(METADATA_FILE_NAME); assert!(metadata_path.is_file()); From 7216f2260909595e2e5f87b074b71c2ff7276ace Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 11 Oct 2021 08:59:06 +0300 Subject: [PATCH 08/46] Use tracing crate to have more context in log messages. Whenever we start processing a request, we now enter a tracing "span" that includes context information like the tenant and timeline ID, and the operation we're performing. That context information gets attached to every log message we create within the span. That way, we don't need to include basic context information like that in every log message, and it also becomes easier to filter the logs programmatically. This removes the eplicit timeline and tenant IDs from most log messages, as you get that information from the enclosing span now. Also improve log messages in general, dialing down the level of some messages that are not very useful, and adding information to others. We now obey the RUST_LOG env variable, if it's set. The 'tracing' crate allows for different log formatters, like JSON or bunyan output. The one we use now is human-readable multi-line format, which is nice when reading the log directly, but hard for post-processing. For production, we'll probably want JSON output and some tools for working with it, but that's left as a TODO. The log format is easy to change. --- Cargo.lock | 233 ++++++++---------- pageserver/Cargo.toml | 1 + pageserver/src/basebackup.rs | 3 +- pageserver/src/bin/pageserver.rs | 6 +- pageserver/src/branches.rs | 13 +- pageserver/src/http/routes.rs | 20 +- pageserver/src/layered_repository.rs | 84 +++---- .../src/layered_repository/inmemory_layer.rs | 3 +- pageserver/src/page_service.rs | 31 ++- pageserver/src/restore_local_repo.rs | 17 +- pageserver/src/walreceiver.rs | 14 +- walkeeper/src/bin/wal_acceptor.rs | 2 +- zenith_utils/Cargo.toml | 9 +- zenith_utils/src/logging.rs | 59 +++-- 14 files changed, 234 insertions(+), 261 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e36b462d8e..f7ce2bf543 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,18 +26,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486" -[[package]] -name = "arc-swap" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820" - [[package]] name = "async-trait" version = "0.1.50" @@ -298,7 +301,7 @@ version = "2.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" dependencies = [ - "ansi_term", + "ansi_term 0.11.0", "atty", "bitflags", "strsim", @@ -387,26 +390,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" -dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" -dependencies = [ - "cfg-if 1.0.0", - "lazy_static", -] - [[package]] name = "crypto-mac" version = "0.10.0" @@ -445,16 +428,6 @@ dependencies = [ "dirs-sys", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if 1.0.0", - "dirs-sys-next", -] - [[package]] name = "dirs-sys" version = "0.3.6" @@ -466,17 +439,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "dlv-list" version = "0.2.3" @@ -956,6 +918,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -1224,6 +1195,7 @@ dependencies = [ "thiserror", "tokio", "toml", + "tracing", "workspace_hack", "zenith_metrics", "zenith_utils", @@ -1531,6 +1503,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.25" @@ -1689,12 +1670,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustversion" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" - [[package]] name = "ryu" version = "1.0.5" @@ -1852,6 +1827,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "sharded-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.0.0" @@ -1890,59 +1874,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" - -[[package]] -name = "slog-async" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60813879f820c85dbc4eabf3269befe374591289019775898d56a81a804fbdc" -dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", -] - -[[package]] -name = "slog-scope" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" -dependencies = [ - "arc-swap", - "lazy_static", - "slog", -] - -[[package]] -name = "slog-stdlog" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8228ab7302adbf4fcb37e66f3cda78003feb521e7fd9e3847ec117a7784d0f5a" -dependencies = [ - "log", - "slog", - "slog-scope", -] - -[[package]] -name = "slog-term" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c1e7e5aab61ced6006149ea772770b84a0d16ce0f7885def313e4829946d76" -dependencies = [ - "atty", - "chrono", - "slog", - "term", - "thread_local", -] - [[package]] name = "smallvec" version = "1.6.1" @@ -1998,12 +1929,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - [[package]] name = "tap" version = "1.0.1" @@ -2035,17 +1960,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - [[package]] name = "termcolor" version = "1.1.2" @@ -2223,24 +2137,79 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.26" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", + "tracing-attributes", "tracing-core", ] [[package]] -name = "tracing-core" +name = "tracing-attributes" version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-log" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" +dependencies = [ + "ansi_term 0.12.1", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + [[package]] name = "try-lock" version = "0.2.3" @@ -2605,14 +2574,12 @@ dependencies = [ "rustls-split", "serde", "serde_json", - "slog", - "slog-async", - "slog-scope", - "slog-stdlog", - "slog-term", "tempfile", "thiserror", "tokio", + "tracing", + "tracing-log", + "tracing-subscriber", "webpki", "workspace_hack", "zenith_metrics", diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 0e5a82df88..abbade4355 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -35,6 +35,7 @@ scopeguard = "1.1.0" rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] } async-trait = "0.1" const_format = "0.2.21" +tracing = "0.1.27" postgres_ffi = { path = "../postgres_ffi" } zenith_metrics = { path = "../zenith_metrics" } diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index 5389d609a5..a4ee89918c 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -31,7 +31,7 @@ use zenith_utils::lsn::Lsn; pub struct Basebackup<'a> { ar: Builder<&'a mut dyn Write>, timeline: &'a Arc, - lsn: Lsn, + pub lsn: Lsn, prev_record_lsn: Lsn, } @@ -97,7 +97,6 @@ impl<'a> Basebackup<'a> { pub fn send_tarball(&mut self) -> anyhow::Result<()> { // Create pgdata subdirs structure for dir in pg_constants::PGDATA_SUBDIRS.iter() { - info!("send subdir {:?}", *dir); let header = new_tar_header_dir(*dir)?; self.ar.append(&header, &mut io::empty())?; } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index c763f98a7f..d6cf86ba24 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -2,7 +2,6 @@ // Main entry point for the Page Server executable // -use log::*; use pageserver::defaults::*; use serde::{Deserialize, Serialize}; use std::{ @@ -12,6 +11,7 @@ use std::{ str::FromStr, thread, }; +use tracing::*; use zenith_utils::{auth::JwtAuth, logging, postgres_backend::AuthType}; use anyhow::{bail, ensure, Context, Result}; @@ -447,7 +447,7 @@ fn main() -> Result<()> { fn start_pageserver(conf: &'static PageServerConf) -> Result<()> { // Initialize logger - let (_scope_guard, log_file) = logging::init(LOG_FILE_NAME, conf.daemonize)?; + let log_file = logging::init(LOG_FILE_NAME, conf.daemonize)?; // TODO: Check that it looks like a valid repository before going further @@ -480,7 +480,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> { match daemonize.start() { Ok(_) => info!("Success, daemonized"), - Err(e) => error!("could not daemonize: {:#}", e), + Err(err) => error!(%err, "could not daemonize"), } } diff --git a/pageserver/src/branches.rs b/pageserver/src/branches.rs index 57adf479ca..4a30e760b5 100644 --- a/pageserver/src/branches.rs +++ b/pageserver/src/branches.rs @@ -14,12 +14,12 @@ use std::{ str::FromStr, sync::Arc, }; -use zenith_utils::zid::{ZTenantId, ZTimelineId}; +use tracing::*; -use log::*; use zenith_utils::crashsafe_dir; use zenith_utils::logging; use zenith_utils::lsn::Lsn; +use zenith_utils::zid::{ZTenantId, ZTimelineId}; use crate::tenant_mgr; use crate::walredo::WalRedoManager; @@ -100,7 +100,7 @@ pub struct PointInTime { pub fn init_pageserver(conf: &'static PageServerConf, create_tenant: Option<&str>) -> Result<()> { // Initialize logger // use true as daemonize parameter because otherwise we pollute zenith cli output with a few pages long output of info messages - let (_scope_guard, _log_file) = logging::init(LOG_FILE_NAME, true)?; + let _log_file = logging::init(LOG_FILE_NAME, true)?; // We don't use the real WAL redo manager, because we don't want to spawn the WAL redo // process during repository initialization. @@ -176,7 +176,7 @@ fn get_lsn_from_controlfile(path: &Path) -> Result { // to get bootstrap data for timeline initialization. // fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> { - info!("running initdb... "); + info!("running initdb in {}... ", initdbpath.display()); let initdb_path = conf.pg_bin_dir().join("initdb"); let initdb_output = Command::new(initdb_path) @@ -195,7 +195,6 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> { String::from_utf8_lossy(&initdb_output.stderr) ); } - info!("initdb succeeded"); Ok(()) } @@ -210,6 +209,8 @@ fn bootstrap_timeline( tli: ZTimelineId, repo: &dyn Repository, ) -> Result<()> { + let _enter = info_span!("bootstrapping", timeline = %tli, tenant = %tenantid).entered(); + let initdb_path = conf.tenant_path(&tenantid).join("tmp"); // Init temporarily repo to get bootstrap data @@ -218,8 +219,6 @@ fn bootstrap_timeline( let lsn = get_lsn_from_controlfile(&pgdata_path)?.align(); - info!("bootstrap_timeline {:?} at lsn {}", pgdata_path, lsn); - // Import the contents of the data directory at the initial checkpoint // LSN, and any WAL after that. let timeline = repo.create_empty_timeline(tli)?; diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index cd6b84b22f..8b15e438da 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -6,6 +6,7 @@ use hyper::header; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use routerify::{ext::RequestExt, RouterBuilder}; +use tracing::*; use zenith_utils::auth::JwtAuth; use zenith_utils::http::endpoint::attach_openapi_ui; use zenith_utils::http::endpoint::auth_middleware; @@ -98,6 +99,7 @@ async fn branch_create_handler(mut request: Request) -> Result) -> Result, A check_permission(&request, Some(tenantid))?; let response_data = tokio::task::spawn_blocking(move || { + let _enter = info_span!("branch_list", tenant = %tenantid).entered(); crate::branches::get_branches(get_config(&request), &tenantid) }) .await @@ -126,11 +129,12 @@ async fn branch_list_handler(request: Request) -> Result, A // TODO add to swagger async fn branch_detail_handler(request: Request) -> Result, ApiError> { let tenantid: ZTenantId = parse_request_param(&request, "tenant_id")?; - let branch_name: &str = get_request_param(&request, "branch_name")?; + let branch_name: String = get_request_param(&request, "branch_name")?.to_string(); let conf = get_state(&request).conf; - let path = conf.branch_path(branch_name, &tenantid); + let path = conf.branch_path(&branch_name, &tenantid); let response_data = tokio::task::spawn_blocking(move || { + let _enter = info_span!("branch_detail", tenant = %tenantid, branch=%branch_name).entered(); let repo = tenant_mgr::get_repository_for_tenant(tenantid)?; BranchInfo::from_path(path, conf, &tenantid, &repo) }) @@ -144,10 +148,13 @@ async fn tenant_list_handler(request: Request) -> Result, A // check for management permission check_permission(&request, None)?; - let response_data = - tokio::task::spawn_blocking(move || crate::branches::get_tenants(get_config(&request))) - .await - .map_err(ApiError::from_err)??; + let response_data = tokio::task::spawn_blocking(move || { + let _enter = info_span!("tenant_list").entered(); + crate::branches::get_tenants(get_config(&request)) + }) + .await + .map_err(ApiError::from_err)??; + Ok(json_response(StatusCode::OK, response_data)?) } @@ -158,6 +165,7 @@ async fn tenant_create_handler(mut request: Request) -> Result Result<()> { + let _enter = info_span!("saving metadata").entered(); let path = metadata_path(conf, timelineid, tenantid); // use OpenOptions to ensure file presence is consistent with first_save let mut file = OpenOptions::new() @@ -371,8 +380,6 @@ impl LayeredRepository { .create_new(first_save) .open(&path)?; - info!("saving metadata {}", path.display()); - let mut metadata_bytes = TimelineMetadata::ser(data)?; assert!(metadata_bytes.len() <= METADATA_MAX_DATA_SIZE); @@ -686,13 +693,7 @@ impl Timeline for LayeredTimeline { let segsize; if let Some((layer, lsn)) = self.get_layer_for_read(seg, lsn)? { segsize = layer.get_seg_size(lsn)?; - trace!( - "get_seg_size: {} at {}/{} -> {}", - seg, - self.timelineid, - lsn, - segsize - ); + trace!("get_seg_size: {} at {} -> {}", seg, lsn, segsize); } else { if segno == 0 { return Ok(None); @@ -794,7 +795,7 @@ impl Timeline for LayeredTimeline { result.insert(new_relish); trace!("List object {}", new_relish); } else { - trace!("Filter out droped object {}", new_relish); + trace!("Filtered out dropped object {}", new_relish); } } @@ -972,6 +973,8 @@ impl Timeline for LayeredTimeline { fn get_current_logical_size_non_incremental(&self, lsn: Lsn) -> Result { let mut total_blocks: usize = 0; + let _enter = info_span!("calc logical size", %lsn).entered(); + // list of all relations in this timeline, including ancestor timelines let all_rels = self.list_rels(0, 0, lsn)?; @@ -1042,11 +1045,8 @@ impl LayeredTimeline { /// Returns all timeline-related files that were found and loaded. /// fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result> { - info!( - "loading layer map for timeline {} into memory", - self.timelineid - ); let mut layers = self.layers.lock().unwrap(); + let mut num_layers = 0; let (imgfilenames, deltafilenames) = filename::list_files(self.conf, self.timelineid, self.tenantid)?; @@ -1066,14 +1066,10 @@ impl LayeredTimeline { let layer = ImageLayer::new(self.conf, self.timelineid, self.tenantid, filename); - info!( - "found layer {} {} on timeline {}", - layer.get_seg_tag(), - layer.get_start_lsn(), - self.timelineid - ); + trace!("found layer {}", layer.filename().display()); local_layers.push(layer.path()); layers.insert_historic(Arc::new(layer)); + num_layers += 1; } for filename in &deltafilenames { @@ -1090,14 +1086,12 @@ impl LayeredTimeline { let layer = DeltaLayer::new(self.conf, self.timelineid, self.tenantid, filename); - info!( - "found layer {} on timeline {}", - layer.filename().display(), - self.timelineid, - ); + trace!("found layer {}", layer.filename().display()); local_layers.push(layer.path()); layers.insert_historic(Arc::new(layer)); + num_layers += 1; } + info!("loaded layer map with {} layers", num_layers); Ok(local_layers) } @@ -1148,12 +1142,7 @@ impl LayeredTimeline { lsn: Lsn, self_layers: &MutexGuard, ) -> Result, Lsn)>> { - trace!( - "get_layer_for_read called for {} at {}/{}", - seg, - self.timelineid, - lsn - ); + trace!("get_layer_for_read called for {} at {}", seg, lsn); // If you requested a page at an older LSN, before the branch point, dig into // the right ancestor timeline. This can only happen if you launch a read-only @@ -1271,17 +1260,15 @@ impl LayeredTimeline { // First modification on this timeline start_lsn = self.ancestor_lsn + 1; trace!( - "creating layer for write for {} at branch point {}/{}", + "creating layer for write for {} at branch point {}", seg, - self.timelineid, start_lsn ); } else { start_lsn = prev_layer.get_end_lsn(); trace!( - "creating layer for write for {} after previous layer {}/{}", + "creating layer for write for {} after previous layer {}", seg, - self.timelineid, start_lsn ); } @@ -1340,11 +1327,7 @@ impl LayeredTimeline { prev: prev_record_lsn, } = self.last_record_lsn.load(); - trace!( - "checkpointing timeline {} at {}", - self.timelineid, - last_record_lsn - ); + trace!("checkpoint starting at {}", last_record_lsn); // Take the in-memory layer with the oldest WAL record. If it's older // than the threshold, write it out to disk as a new image and delta file. @@ -1520,11 +1503,11 @@ impl LayeredTimeline { let now = Instant::now(); let mut result: GcResult = Default::default(); - info!( - "running GC on timeline {}, cutoff {}", - self.timelineid, cutoff - ); - info!("retain_lsns: {:?}", retain_lsns); + let _enter = info_span!("garbage collection", timeline = %self.timelineid, tenant = %self.tenantid, cutoff = %cutoff).entered(); + + info!("GC starting"); + + debug!("retain_lsns: {:?}", retain_lsns); let mut layers_to_remove: Vec> = Vec::new(); @@ -1786,10 +1769,9 @@ impl LayeredTimeline { if data.records.is_empty() { if let Some(img) = &data.page_img { trace!( - "found page image for blk {} in {} at {}/{}, no WAL redo required", + "found page image for blk {} in {} at {}, no WAL redo required", blknum, rel, - self.timelineid, request_lsn ); Ok(img.clone()) @@ -1815,9 +1797,9 @@ impl LayeredTimeline { Ok(ZERO_PAGE.clone()) } else { if data.page_img.is_some() { - trace!("found {} WAL records and a base image for blk {} in {} at {}/{}, performing WAL redo", data.records.len(), blknum, rel, self.timelineid, request_lsn); + trace!("found {} WAL records and a base image for blk {} in {} at {}, performing WAL redo", data.records.len(), blknum, rel, request_lsn); } else { - trace!("found {} WAL records that will init the page for blk {} in {} at {}/{}, performing WAL redo", data.records.len(), blknum, rel, self.timelineid, request_lsn); + trace!("found {} WAL records that will init the page for blk {} in {} at {}, performing WAL redo", data.records.len(), blknum, rel, request_lsn); } let img = self.walredo_mgr.request_redo( rel, diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index 214c392fb6..8c44ac6296 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -579,9 +579,8 @@ impl InMemoryLayer { /// After completion, self is non-writeable, but not frozen. pub fn freeze(self: Arc, cutoff_lsn: Lsn) -> Result { info!( - "freezing in memory layer {} on timeline {} at {} (oldest {})", + "freezing in-memory layer {} at {} (oldest {})", self.filename().display(), - self.timelineid, cutoff_lsn, self.oldest_pending_lsn ); diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index d592a83993..32b8746f09 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -13,7 +13,6 @@ use anyhow::{anyhow, bail, ensure, Result}; use bytes::{Buf, BufMut, Bytes, BytesMut}; use lazy_static::lazy_static; -use log::*; use regex::Regex; use std::net::TcpListener; use std::str; @@ -21,6 +20,7 @@ use std::str::FromStr; use std::sync::Arc; use std::thread; use std::{io, net::TcpStream}; +use tracing::*; use zenith_metrics::{register_histogram_vec, HistogramVec}; use zenith_utils::auth::{self, JwtAuth}; use zenith_utils::auth::{Claims, Scope}; @@ -194,7 +194,7 @@ pub fn thread_main( let local_auth = auth.clone(); thread::spawn(move || { if let Err(err) = page_service_conn_main(conf, local_auth, socket, auth_type) { - error!("page server thread exiting with error: {:#}", err); + error!(%err, "page server thread exited with error"); } }); } @@ -260,6 +260,8 @@ impl PageServerHandler { timelineid: ZTimelineId, tenantid: ZTenantId, ) -> anyhow::Result<()> { + let _enter = info_span!("pagestream", timeline = %timelineid, tenant = %tenantid).entered(); + // Check that the timeline exists let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?; @@ -267,7 +269,7 @@ impl PageServerHandler { pgb.write_message(&BeMessage::CopyBothResponse)?; while let Some(message) = pgb.read_message()? { - trace!("query({:?}): {:?}", timelineid, message); + trace!("query: {:?}", message); let copy_data_bytes = match message { FeMessage::CopyData(bytes) => bytes, @@ -363,6 +365,8 @@ impl PageServerHandler { timeline: &dyn Timeline, req: &PagestreamExistsRequest, ) -> Result { + let _enter = info_span!("get_rel_exists", rel = %req.rel, req_lsn = %req.lsn).entered(); + let tag = RelishTag::Relation(req.rel); let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?; @@ -378,6 +382,7 @@ impl PageServerHandler { timeline: &dyn Timeline, req: &PagestreamNblocksRequest, ) -> Result { + let _enter = info_span!("get_nblocks", rel = %req.rel, req_lsn = %req.lsn).entered(); let tag = RelishTag::Relation(req.rel); let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?; @@ -397,6 +402,8 @@ impl PageServerHandler { timeline: &dyn Timeline, req: &PagestreamGetPageRequest, ) -> Result { + let _enter = info_span!("get_page", rel = %req.rel, blkno = &req.blkno, req_lsn = %req.lsn) + .entered(); let tag = RelishTag::Relation(req.rel); let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest)?; @@ -414,17 +421,20 @@ impl PageServerHandler { lsn: Option, tenantid: ZTenantId, ) -> anyhow::Result<()> { + let span = info_span!("basebackup", timeline = %timelineid, tenant = %tenantid, lsn = field::Empty); + let _enter = span.enter(); + // check that the timeline exists let timeline = tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?; - /* switch client to COPYOUT */ + // switch client to COPYOUT pgb.write_message(&BeMessage::CopyOutResponse)?; - info!("sent CopyOut"); /* Send a tarball of the latest layer on the timeline */ { let mut writer = CopyDataSink { pgb }; let mut basebackup = basebackup::Basebackup::new(&mut writer, &timeline, lsn)?; + span.record("lsn", &basebackup.lsn.to_string().as_str()); basebackup.send_tarball()?; } pgb.write_message(&BeMessage::CopyDone)?; @@ -529,11 +539,6 @@ impl postgres_backend::Handler for PageServerHandler { None }; - info!( - "got basebackup command. tenantid=\"{}\" timelineid=\"{}\" lsn=\"{:#?}\"", - tenantid, timelineid, lsn - ); - // Check that the timeline exists self.handle_basebackup_request(pgb, timelineid, lsn, tenantid)?; pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; @@ -551,6 +556,9 @@ impl postgres_backend::Handler for PageServerHandler { self.check_permission(Some(tenantid))?; + let _enter = + info_span!("callmemaybe", timeline = %timelineid, tenant = %tenantid).entered(); + // Check that the timeline exists tenant_mgr::get_timeline_for_tenant(tenantid, timelineid)?; @@ -573,6 +581,9 @@ impl postgres_backend::Handler for PageServerHandler { self.check_permission(Some(tenantid))?; + let _enter = + info_span!("branch_create", name = %branchname, tenant = %tenantid).entered(); + let branch = branches::create_branch(self.conf, &branchname, &startpoint_str, &tenantid)?; let branch = serde_json::to_vec(&branch)?; diff --git a/pageserver/src/restore_local_repo.rs b/pageserver/src/restore_local_repo.rs index dfe3edd7ac..ec6ee61d52 100644 --- a/pageserver/src/restore_local_repo.rs +++ b/pageserver/src/restore_local_repo.rs @@ -2,7 +2,6 @@ //! Import data and WAL from a PostgreSQL data directory and WAL segments into //! zenith Timeline. //! -use log::*; use postgres_ffi::nonrelfile_utils::clogpage_precedes; use postgres_ffi::nonrelfile_utils::slru_may_delete_clogsegment; use std::cmp::min; @@ -13,6 +12,7 @@ use std::path::Path; use anyhow::{bail, Result}; use bytes::{Buf, Bytes}; +use tracing::*; use crate::relish::*; use crate::repository::*; @@ -139,6 +139,7 @@ fn import_relfile( dboid: Oid, ) -> Result<()> { // Does it look like a relation file? + trace!("importing rel file {}", path.display()); let p = parse_relfilename(path.file_name().unwrap().to_str().unwrap()); if let Err(e) = p { @@ -166,14 +167,14 @@ fn import_relfile( } // TODO: UnexpectedEof is expected - Err(e) => match e.kind() { + Err(err) => match err.kind() { std::io::ErrorKind::UnexpectedEof => { // reached EOF. That's expected. // FIXME: maybe check that we read the full length of the file? break; } _ => { - bail!("error reading file {}: {:#}", path.display(), e); + bail!("error reading file {}: {:#}", path.display(), err); } }, }; @@ -200,7 +201,7 @@ fn import_nonrel_file( // read the whole file file.read_to_end(&mut buffer)?; - info!("importing non-rel file {}", path.display()); + trace!("importing non-rel file {}", path.display()); timeline.put_page_image(tag, 0, lsn, Bytes::copy_from_slice(&buffer[..]))?; Ok(()) @@ -217,7 +218,7 @@ fn import_control_file(timeline: &dyn Timeline, lsn: Lsn, path: &Path) -> Result // read the whole file file.read_to_end(&mut buffer)?; - info!("importing control file {}", path.display()); + trace!("importing control file {}", path.display()); // Import it as ControlFile timeline.put_page_image( @@ -244,7 +245,7 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa let mut buf: [u8; 8192] = [0u8; 8192]; let segno = u32::from_str_radix(path.file_name().unwrap().to_str().unwrap(), 16)?; - info!("importing slru file {}", path.display()); + trace!("importing slru file {}", path.display()); let mut rpageno = 0; loop { @@ -260,14 +261,14 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa } // TODO: UnexpectedEof is expected - Err(e) => match e.kind() { + Err(err) => match err.kind() { std::io::ErrorKind::UnexpectedEof => { // reached EOF. That's expected. // FIXME: maybe check that we read the full length of the file? break; } _ => { - bail!("error reading file {}: {:#}", path.display(), e); + bail!("error reading file {}: {:#}", path.display(), err); } }, }; diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index db0f2806ae..0fd9dfdf27 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -12,7 +12,6 @@ use crate::waldecoder::*; use crate::PageServerConf; use anyhow::{bail, Error, Result}; use lazy_static::lazy_static; -use log::*; use postgres::fallible_iterator::FallibleIterator; use postgres::replication::ReplicationIter; use postgres::{Client, NoTls, SimpleQueryMessage, SimpleQueryRow}; @@ -27,6 +26,7 @@ use std::thread; use std::thread::sleep; use std::thread_local; use std::time::{Duration, SystemTime}; +use tracing::*; use zenith_utils::lsn::Lsn; use zenith_utils::zid::ZTenantId; use zenith_utils::zid::ZTimelineId; @@ -96,10 +96,8 @@ fn get_wal_producer_connstr(timelineid: ZTimelineId) -> String { // This is the entry point for the WAL receiver thread. // fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: ZTenantId) { - info!( - "WAL receiver thread started for timeline : '{}'", - timelineid - ); + let _enter = info_span!("WAL receiver", timeline = %timelineid, tenant = %tenantid).entered(); + info!("WAL receiver thread started"); // // Make a connection to the WAL safekeeper, or directly to the primary PostgreSQL server, @@ -169,8 +167,8 @@ fn walreceiver_main( startpoint += startpoint.calc_padding(8u32); info!( - "last_record_lsn {} starting replication from {} for timeline {}, server is at {}...", - last_rec_lsn, startpoint, timelineid, end_of_wal + "last_record_lsn {} starting replication from {}, server is at {}...", + last_rec_lsn, startpoint, end_of_wal ); let query = format!("START_REPLICATION PHYSICAL {}", startpoint); @@ -198,6 +196,8 @@ fn walreceiver_main( waldecoder.feed_bytes(data); while let Some((lsn, recdata)) = waldecoder.poll_decode()? { + let _enter = info_span!("processing record", lsn = %lsn).entered(); + // Save old checkpoint value to compare with it after decoding WAL record let old_checkpoint_bytes = checkpoint.encode(); let decoded = decode_wal_record(recdata.clone()); diff --git a/walkeeper/src/bin/wal_acceptor.rs b/walkeeper/src/bin/wal_acceptor.rs index 5bd64cb8a6..bfb09db5cb 100644 --- a/walkeeper/src/bin/wal_acceptor.rs +++ b/walkeeper/src/bin/wal_acceptor.rs @@ -129,7 +129,7 @@ fn main() -> Result<()> { fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { let log_filename = conf.data_dir.join("wal_acceptor.log"); - let (_scope_guard, log_file) = logging::init(log_filename, conf.daemonize)?; + let log_file = logging::init(log_filename, conf.daemonize)?; let http_listener = TcpListener::bind(conf.listen_http_addr.clone()).map_err(|e| { error!("failed to bind to address {}: {}", conf.listen_http_addr, e); diff --git a/zenith_utils/Cargo.toml b/zenith_utils/Cargo.toml index 22c1c9bab6..6571fae042 100644 --- a/zenith_utils/Cargo.toml +++ b/zenith_utils/Cargo.toml @@ -18,12 +18,9 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1" thiserror = "1.0" tokio = "1.11" - -slog-async = "2.6.0" -slog-stdlog = "4.1.0" -slog-scope = "4.4.0" -slog-term = "2.8.0" -slog = "2.7.0" +tracing = "0.1" +tracing-log = "0.1" +tracing-subscriber = "0.2" zenith_metrics = { path = "../zenith_metrics" } workspace_hack = { path = "../workspace_hack" } diff --git a/zenith_utils/src/logging.rs b/zenith_utils/src/logging.rs index c6ed35cbf4..53dbfc305d 100644 --- a/zenith_utils/src/logging.rs +++ b/zenith_utils/src/logging.rs @@ -1,4 +1,3 @@ -use slog::{Drain, Level}; use std::{ fs::{File, OpenOptions}, path::Path, @@ -6,10 +5,12 @@ use std::{ use anyhow::{Context, Result}; -pub fn init( - log_filename: impl AsRef, - daemonize: bool, -) -> Result<(slog_scope::GlobalLoggerGuard, File)> { +use tracing::subscriber::set_global_default; +use tracing_log::LogTracer; +use tracing_subscriber::fmt; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; + +pub fn init(log_filename: impl AsRef, daemonize: bool) -> Result { // Don't open the same file for output multiple times; // the different fds could overwrite each other's output. let log_file = OpenOptions::new() @@ -18,30 +19,38 @@ pub fn init( .open(&log_filename) .with_context(|| format!("failed to open {:?}", log_filename.as_ref()))?; + let default_filter_str = "info"; + + // We fall back to printing all spans at info-level or above if + // the RUST_LOG environment variable is not set. + let env_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_filter_str)); + // we are cloning and returning log file in order to allow redirecting daemonized stdout and stderr to it // if we do not use daemonization (e.g. in docker) it is better to log to stdout directly // for example to be in line with docker log command which expects logs comimg from stdout - let guard = if daemonize { - let decorator = slog_term::PlainSyncDecorator::new(log_file.try_clone()?); - let drain = slog_term::FullFormat::new(decorator) - .build() - .filter_level(Level::Info) - .fuse(); - let logger = slog::Logger::root(drain, slog::o!()); - slog_scope::set_global_logger(logger) + // + // TODO: perhaps use a more human-readable format when !daemonize + if daemonize { + let x = log_file.try_clone().unwrap(); + + let fmt_layer = fmt::layer() + .pretty() + .with_target(false) // don't include event targets + .with_ansi(false) // don't use colors in log file + .with_writer(move || x.try_clone().unwrap()); + let subscriber = Registry::default().with(env_filter).with(fmt_layer); + + set_global_default(subscriber).expect("Failed to set subscriber"); } else { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::FullFormat::new(decorator) - .build() - .filter_level(Level::Info) - .fuse(); - let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse(); - let logger = slog::Logger::root(drain, slog::o!()); - slog_scope::set_global_logger(logger) - }; + let fmt_layer = fmt::layer().with_target(false); // don't include event targets + let subscriber = Registry::default().with(env_filter).with(fmt_layer); - // initialise forwarding of std log calls - slog_stdlog::init()?; + set_global_default(subscriber).expect("Failed to set subscriber"); + } - Ok((guard, log_file)) + // Redirect all `log`'s events to our subscriber + LogTracer::init().expect("Failed to set logger"); + + Ok(log_file) } From b9119f11bfc4d208ded1a7c8ea802fa66b010dd8 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 11 Oct 2021 11:10:58 +0300 Subject: [PATCH 09/46] Add perf test case for buffering GiST build. When a WAL record affects multiple pages, we currently duplicate the record for each affected page. That's a bit wasteful, but not too bad for b-tree splits and non-hot heap updates that affect two pages. But buffering GiST index build WAL-logs the whole relation in 32 page chunks, with one giant WAL record for each 32-page chunk. Currently we duplicate that giant record for each of the 32 pages, which is really wasteful. Github issue https://github.com/zenithdb/zenith/issues/720 tracks the problem. This commit adds a test case for it to demonstrate it. --- test_runner/performance/test_gist_build.py | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 test_runner/performance/test_gist_build.py diff --git a/test_runner/performance/test_gist_build.py b/test_runner/performance/test_gist_build.py new file mode 100644 index 0000000000..040adb1242 --- /dev/null +++ b/test_runner/performance/test_gist_build.py @@ -0,0 +1,48 @@ +import os +from contextlib import closing +from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver + +pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") + +# +# Test buffering GisT build. It WAL-logs the whole relation, in 32-page chunks. +# As of this writing, we're duplicate those giant WAL records for each page, +# which makes the delta layer about 32x larger than it needs to be. +# +def test_gist_buffering_build(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str): + # Create a branch for us + zenith_cli.run(["branch", "test_gist_buffering_build", "empty"]) + + pg = postgres.create_start('test_gist_buffering_build') + print("postgres is running on 'test_gist_buffering_build' branch") + + # Open a connection directly to the page server that we'll use to force + # flushing the layers to disk + psconn = pageserver.connect(); + pscur = psconn.cursor() + + # Get the timeline ID of our branch. We need it for the 'do_gc' command + with closing(pg.connect()) as conn: + with conn.cursor() as cur: + cur.execute("SHOW zenith.zenith_timeline") + timeline = cur.fetchone()[0] + + # Create test table. + cur.execute("create table gist_point_tbl(id int4, p point)"); + cur.execute("insert into gist_point_tbl select g, point(g, g) from generate_series(1, 1000000) g;"); + + # Build the index. + with zenbenchmark.record_pageserver_writes(pageserver, 'pageserver_writes'): + with zenbenchmark.record_duration('build'): + cur.execute("create index gist_pointidx2 on gist_point_tbl using gist(p) with (buffering = on)"); + + # Flush the layers from memory to disk. This is included in the reported + # time and I/O + pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 1000000") + + # Record peak memory usage + zenbenchmark.record("peak_mem", zenbenchmark.get_peak_mem(pageserver) / 1024, 'MB') + + # Report disk space used by the repository + timeline_size = zenbenchmark.get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) + zenbenchmark.record('size', timeline_size / (1024*1024), 'MB') From d7c9dd06f42c17a5c0d1ae2a075032d3e63671e8 Mon Sep 17 00:00:00 2001 From: anastasia Date: Mon, 27 Sep 2021 15:28:03 +0300 Subject: [PATCH 10/46] Implement graceful shutdown at 'pageserver stop': - perform checkpoint for each tenant repository. - wait for the completion of all threads. Add new option 'immediate' to 'pageserver stop' command to terminate the pageserver immediately. --- Cargo.lock | 12 ++ control_plane/src/storage.rs | 47 ++++++-- pageserver/Cargo.toml | 3 +- pageserver/src/bin/pageserver.rs | 62 +++++++++- pageserver/src/layered_repository.rs | 71 +++++++++--- pageserver/src/page_service.rs | 108 +++++++++++------- .../src/relish_storage/synced_storage.rs | 33 +++--- pageserver/src/repository.rs | 2 + pageserver/src/tenant_mgr.rs | 71 +++++++++++- pageserver/src/walreceiver.rs | 37 ++++-- proxy/src/mgmt.rs | 2 +- proxy/src/proxy.rs | 1 + test_runner/fixtures/zenith_fixtures.py | 13 ++- walkeeper/src/wal_service.rs | 2 +- zenith/src/main.rs | 16 ++- zenith_utils/src/http/endpoint.rs | 27 ++++- zenith_utils/src/postgres_backend.rs | 52 ++++++++- zenith_utils/tests/ssl_test.rs | 6 +- 18 files changed, 448 insertions(+), 117 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7ce2bf543..6ea1fd5593 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1191,6 +1191,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", + "signal-hook", "tar", "thiserror", "tokio", @@ -1842,6 +1843,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42a568c8f2cd051a4d283bd6eb0343ac214c1b0f1ac19f93e1175b2dee38c73d" +[[package]] +name = "signal-hook" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c98891d737e271a2954825ef19e46bd16bdb98e2746f2eec4f7a4ef7946efd1" +dependencies = [ + "cc", + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 9d762c360f..3d331ca2a7 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -199,23 +199,45 @@ impl PageServerNode { bail!("pageserver failed to start in {} seconds", RETRIES); } - pub fn stop(&self) -> anyhow::Result<()> { + pub fn stop(&self, immediate: bool) -> anyhow::Result<()> { let pid = read_pidfile(&self.pid_file())?; let pid = Pid::from_raw(pid); - if kill(pid, Signal::SIGTERM).is_err() { - bail!("Failed to kill pageserver with pid {}", pid); + if immediate { + println!("Stop pageserver immediately"); + if kill(pid, Signal::SIGQUIT).is_err() { + bail!("Failed to kill pageserver with pid {}", pid); + } + } else { + println!("Stop pageserver gracefully"); + if kill(pid, Signal::SIGTERM).is_err() { + bail!("Failed to stop pageserver with pid {}", pid); + } } - // wait for pageserver stop let address = connection_address(&self.pg_connection_config); - for _ in 0..5 { - let stream = TcpStream::connect(&address); - thread::sleep(Duration::from_secs(1)); - if let Err(_e) = stream { - println!("Pageserver stopped"); - return Ok(()); + + // TODO Remove this "timeout" and handle it on caller side instead. + // Shutting down may take a long time, + // if pageserver checkpoints a lot of data + for _ in 0..100 { + if let Err(_e) = TcpStream::connect(&address) { + println!("Pageserver stopped receiving connections"); + + //Now check status + match self.check_status() { + Ok(_) => { + println!("Pageserver status is OK. Wait a bit."); + thread::sleep(Duration::from_secs(1)); + } + Err(err) => { + println!("Pageserver status is: {}", err); + return Ok(()); + } + } + } else { + println!("Pageserver still receives connections"); + thread::sleep(Duration::from_secs(1)); } - println!("Stopping pageserver on {}", address); } bail!("Failed to stop pageserver with pid {}", pid); @@ -313,8 +335,9 @@ impl PageServerNode { impl Drop for PageServerNode { fn drop(&mut self) { + // TODO Looks like this flag is never set if self.kill_on_exit { - let _ = self.stop(); + let _ = self.stop(true); } } } diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index abbade4355..33c911c840 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -17,7 +17,7 @@ lazy_static = "1.4.0" log = "0.4.14" clap = "2.33.0" daemonize = "0.4.1" -tokio = { version = "1.11", features = ["process", "macros", "fs"] } +tokio = { version = "1.11", features = ["process", "macros", "fs", "rt"] } postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" } postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" } postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" } @@ -36,6 +36,7 @@ rust-s3 = { version = "0.27.0-rc4", features = ["no-verify-ssl"] } async-trait = "0.1" const_format = "0.2.21" tracing = "0.1.27" +signal-hook = {version = "0.3.10", features = ["extended-siginfo"] } postgres_ffi = { path = "../postgres_ffi" } zenith_metrics = { path = "../zenith_metrics" } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index d6cf86ba24..4e2c02a83a 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -15,6 +15,15 @@ use tracing::*; use zenith_utils::{auth::JwtAuth, logging, postgres_backend::AuthType}; use anyhow::{bail, ensure, Context, Result}; +use signal_hook::consts::signal::*; +use signal_hook::consts::TERM_SIGNALS; +use signal_hook::flag; +use signal_hook::iterator::exfiltrator::WithOrigin; +use signal_hook::iterator::SignalsInfo; +use std::process::exit; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; + use clap::{App, Arg, ArgMatches}; use daemonize::Daemonize; @@ -28,6 +37,7 @@ use pageserver::{ RelishStorageKind, S3Config, LOG_FILE_NAME, }; use zenith_utils::http::endpoint; +use zenith_utils::postgres_backend; use const_format::formatcp; @@ -449,6 +459,17 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> { // Initialize logger let log_file = logging::init(LOG_FILE_NAME, conf.daemonize)?; + let term_now = Arc::new(AtomicBool::new(false)); + for sig in TERM_SIGNALS { + // When terminated by a second term signal, exit with exit code 1. + // This will do nothing the first time (because term_now is false). + flag::register_conditional_shutdown(*sig, 1, Arc::clone(&term_now))?; + // But this will "arm" the above for the second time, by setting it to true. + // The order of registering these is important, if you put this one first, it will + // first arm and then terminate ‒ all in the first round. + flag::register(*sig, Arc::clone(&term_now))?; + } + // TODO: Check that it looks like a valid repository before going further // bind sockets before daemonizing so we report errors early and do not return until we are listening @@ -525,13 +546,42 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> { page_service::thread_main(conf, auth, pageserver_listener, conf.auth_type) })?; - join_handles.push(page_service_thread); + for info in SignalsInfo::::new(TERM_SIGNALS)?.into_iter() { + match info.signal { + SIGQUIT => { + info!("Got SIGQUIT. Terminate pageserver in immediate shutdown mode"); + exit(111); + } + SIGTERM => { + info!("Got SIGINT/SIGTERM. Terminate gracefully in fast shutdown mode"); + // Terminate postgres backends + postgres_backend::set_pgbackend_shutdown_requested(); + // Stop all tenants and flush their data + tenant_mgr::shutdown_all_tenants()?; + // Wait for pageservice thread to complete the job + page_service_thread + .join() + .expect("thread panicked") + .expect("thread exited with an error"); - for handle in join_handles.into_iter() { - handle - .join() - .expect("thread panicked") - .expect("thread exited with an error") + // Shut down http router + endpoint::shutdown(); + + // Wait for all threads + for handle in join_handles.into_iter() { + handle + .join() + .expect("thread panicked") + .expect("thread exited with an error"); + } + info!("Pageserver shut down successfully completed"); + exit(0); + } + _ => { + debug!("Unknown signal."); + } + } } + Ok(()) } diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 4c9e03999e..b3164f6cc1 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -30,12 +30,15 @@ use std::ops::Bound::Included; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; use std::time::{Duration, Instant}; use crate::layered_repository::inmemory_layer::FreezeLayers; use crate::relish::*; use crate::relish_storage::schedule_timeline_upload; use crate::repository::{GcResult, Repository, Timeline, WALRecord}; +use crate::tenant_mgr; +use crate::walreceiver; use crate::walreceiver::IS_WAL_RECEIVER; use crate::walredo::WalRedoManager; use crate::PageServerConf; @@ -215,6 +218,23 @@ impl Repository for LayeredRepository { self.gc_iteration_internal(target_timelineid, horizon, checkpoint_before_gc) }) } + + // Wait for all threads to complete and persist repository data before pageserver shutdown. + fn shutdown(&self) -> Result<()> { + trace!("LayeredRepository shutdown for tenant {}", self.tenantid); + + let timelines = self.timelines.lock().unwrap(); + for (timelineid, timeline) in timelines.iter() { + walreceiver::stop_wal_receiver(*timelineid); + // Wait for syncing data to disk + trace!("repo shutdown. checkpoint timeline {}", timelineid); + timeline.checkpoint()?; + + //TODO Wait for walredo process to shutdown too + } + + Ok(()) + } } /// Private functions @@ -298,21 +318,24 @@ impl LayeredRepository { /// /// Launch the checkpointer thread in given repository. /// - pub fn launch_checkpointer_thread(conf: &'static PageServerConf, rc: Arc) { - let _thread = std::thread::Builder::new() + pub fn launch_checkpointer_thread( + conf: &'static PageServerConf, + rc: Arc, + ) -> JoinHandle<()> { + std::thread::Builder::new() .name("Checkpointer thread".into()) .spawn(move || { // FIXME: relaunch it? Panic is not good. rc.checkpoint_loop(conf).expect("Checkpointer thread died"); }) - .unwrap(); + .unwrap() } /// /// Checkpointer thread's main loop /// fn checkpoint_loop(&self, conf: &'static PageServerConf) -> Result<()> { - loop { + while !tenant_mgr::shutdown_requested() { std::thread::sleep(conf.checkpoint_period); info!("checkpointer thread for tenant {} waking up", self.tenantid); @@ -328,40 +351,52 @@ impl LayeredRepository { STORAGE_TIME .with_label_values(&["checkpoint_timed"]) .observe_closure_duration(|| { - timeline.checkpoint_internal(conf.checkpoint_distance) + timeline.checkpoint_internal(conf.checkpoint_distance, false) })? } // release lock on 'timelines' } } + trace!("Checkpointer thread shut down"); + Ok(()) } /// /// Launch the GC thread in given repository. /// - pub fn launch_gc_thread(conf: &'static PageServerConf, rc: Arc) { - let _thread = std::thread::Builder::new() + pub fn launch_gc_thread( + conf: &'static PageServerConf, + rc: Arc, + ) -> JoinHandle<()> { + std::thread::Builder::new() .name("GC thread".into()) .spawn(move || { // FIXME: relaunch it? Panic is not good. rc.gc_loop(conf).expect("GC thread died"); }) - .unwrap(); + .unwrap() } /// /// GC thread's main loop /// fn gc_loop(&self, conf: &'static PageServerConf) -> Result<()> { - loop { - std::thread::sleep(conf.gc_period); - info!("gc thread for tenant {} waking up", self.tenantid); - + while !tenant_mgr::shutdown_requested() { // Garbage collect old files that are not needed for PITR anymore if conf.gc_horizon > 0 { self.gc_iteration(None, conf.gc_horizon, false).unwrap(); } + + // TODO Write it in more adequate way using + // condvar.wait_timeout() or something + let mut sleep_time = conf.gc_period.as_secs(); + while sleep_time > 0 && !tenant_mgr::shutdown_requested() { + sleep_time -= 1; + std::thread::sleep(Duration::from_secs(1)); + } + info!("gc thread for tenant {} waking up", self.tenantid); } + Ok(()) } /// Save timeline metadata to file @@ -508,6 +543,10 @@ impl LayeredRepository { // Ok, we now know all the branch points. // Perform GC for each timeline. for timelineid in timelineids { + if tenant_mgr::shutdown_requested() { + return Ok(totals); + } + // We have already loaded all timelines above // so this operation is just a quick map lookup. let timeline = self.get_timeline_locked(timelineid, &mut *timelines)?; @@ -934,7 +973,7 @@ impl Timeline for LayeredTimeline { STORAGE_TIME .with_label_values(&["checkpoint_force"]) //pass checkpoint_distance=0 to force checkpoint - .observe_closure_duration(|| self.checkpoint_internal(0)) + .observe_closure_duration(|| self.checkpoint_internal(0, true)) } /// @@ -1309,7 +1348,7 @@ impl LayeredTimeline { /// Flush to disk all data that was written with the put_* functions /// /// NOTE: This has nothing to do with checkpoint in PostgreSQL. - fn checkpoint_internal(&self, checkpoint_distance: u64) -> Result<()> { + fn checkpoint_internal(&self, checkpoint_distance: u64, forced: bool) -> Result<()> { // Grab lock on the layer map. // // TODO: We hold it locked throughout the checkpoint operation. That's bad, @@ -1345,6 +1384,10 @@ impl LayeredTimeline { while let Some((oldest_layer, oldest_generation)) = layers.peek_oldest_open() { let oldest_pending_lsn = oldest_layer.get_oldest_pending_lsn(); + if tenant_mgr::shutdown_requested() && !forced { + return Ok(()); + } + // Does this layer need freezing? // // Write out all in-memory layers that contain WAL older than CHECKPOINT_DISTANCE. diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 32b8746f09..be849ce35f 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -25,6 +25,7 @@ use zenith_metrics::{register_histogram_vec, HistogramVec}; use zenith_utils::auth::{self, JwtAuth}; use zenith_utils::auth::{Claims, Scope}; use zenith_utils::lsn::Lsn; +use zenith_utils::postgres_backend::is_socket_read_timed_out; use zenith_utils::postgres_backend::PostgresBackend; use zenith_utils::postgres_backend::{self, AuthType}; use zenith_utils::pq_proto::{ @@ -187,17 +188,32 @@ pub fn thread_main( listener: TcpListener, auth_type: AuthType, ) -> anyhow::Result<()> { - loop { + let mut join_handles = Vec::new(); + + while !tenant_mgr::shutdown_requested() { let (socket, peer_addr) = listener.accept()?; debug!("accepted connection from {}", peer_addr); socket.set_nodelay(true).unwrap(); let local_auth = auth.clone(); - thread::spawn(move || { - if let Err(err) = page_service_conn_main(conf, local_auth, socket, auth_type) { - error!(%err, "page server thread exited with error"); - } - }); + + let handle = thread::Builder::new() + .name("serving Page Service thread".into()) + .spawn(move || { + if let Err(err) = page_service_conn_main(conf, local_auth, socket, auth_type) { + error!(%err, "page server thread exited with error"); + } + }) + .unwrap(); + + join_handles.push(handle); } + + debug!("page_service loop terminated. wait for connections to cancel"); + for handle in join_handles.into_iter() { + handle.join().unwrap(); + } + + Ok(()) } fn page_service_conn_main( @@ -216,7 +232,7 @@ fn page_service_conn_main( } let mut conn_handler = PageServerHandler::new(conf, auth); - let pgbackend = PostgresBackend::new(socket, auth_type, None)?; + let pgbackend = PostgresBackend::new(socket, auth_type, None, true)?; pgbackend.run(&mut conn_handler) } @@ -268,44 +284,58 @@ impl PageServerHandler { /* switch client to COPYBOTH */ pgb.write_message(&BeMessage::CopyBothResponse)?; - while let Some(message) = pgb.read_message()? { - trace!("query: {:?}", message); + while !tenant_mgr::shutdown_requested() { + match pgb.read_message() { + Ok(message) => { + if let Some(message) = message { + trace!("query: {:?}", message); - let copy_data_bytes = match message { - FeMessage::CopyData(bytes) => bytes, - _ => continue, - }; + let copy_data_bytes = match message { + FeMessage::CopyData(bytes) => bytes, + _ => continue, + }; - let zenith_fe_msg = PagestreamFeMessage::parse(copy_data_bytes)?; + let zenith_fe_msg = PagestreamFeMessage::parse(copy_data_bytes)?; - let response = match zenith_fe_msg { - PagestreamFeMessage::Exists(req) => SMGR_QUERY_TIME - .with_label_values(&["get_rel_exists"]) - .observe_closure_duration(|| { - self.handle_get_rel_exists_request(&*timeline, &req) - }), - PagestreamFeMessage::Nblocks(req) => SMGR_QUERY_TIME - .with_label_values(&["get_rel_size"]) - .observe_closure_duration(|| self.handle_get_nblocks_request(&*timeline, &req)), - PagestreamFeMessage::GetPage(req) => SMGR_QUERY_TIME - .with_label_values(&["get_page_at_lsn"]) - .observe_closure_duration(|| { - self.handle_get_page_at_lsn_request(&*timeline, &req) - }), - }; + let response = match zenith_fe_msg { + PagestreamFeMessage::Exists(req) => SMGR_QUERY_TIME + .with_label_values(&["get_rel_exists"]) + .observe_closure_duration(|| { + self.handle_get_rel_exists_request(&*timeline, &req) + }), + PagestreamFeMessage::Nblocks(req) => SMGR_QUERY_TIME + .with_label_values(&["get_rel_size"]) + .observe_closure_duration(|| { + self.handle_get_nblocks_request(&*timeline, &req) + }), + PagestreamFeMessage::GetPage(req) => SMGR_QUERY_TIME + .with_label_values(&["get_page_at_lsn"]) + .observe_closure_duration(|| { + self.handle_get_page_at_lsn_request(&*timeline, &req) + }), + }; - let response = response.unwrap_or_else(|e| { - // print the all details to the log with {:#}, but for the client the - // error message is enough - error!("error reading relation or page version: {:#}", e); - PagestreamBeMessage::Error(PagestreamErrorResponse { - message: e.to_string(), - }) - }); + let response = response.unwrap_or_else(|e| { + // print the all details to the log with {:#}, but for the client the + // error message is enough + error!("error reading relation or page version: {:#}", e); + PagestreamBeMessage::Error(PagestreamErrorResponse { + message: e.to_string(), + }) + }); - pgb.write_message(&BeMessage::CopyData(&response.serialize()))?; + pgb.write_message(&BeMessage::CopyData(&response.serialize()))?; + } else { + break; + } + } + Err(e) => { + if !is_socket_read_timed_out(&e) { + return Err(e); + } + } + } } - Ok(()) } diff --git a/pageserver/src/relish_storage/synced_storage.rs b/pageserver/src/relish_storage/synced_storage.rs index f51e976a83..e9ac20ff8c 100644 --- a/pageserver/src/relish_storage/synced_storage.rs +++ b/pageserver/src/relish_storage/synced_storage.rs @@ -1,6 +1,7 @@ use std::time::Duration; use std::{collections::BinaryHeap, sync::Mutex, thread}; +use crate::tenant_mgr; use crate::{relish_storage::RelishStorage, PageServerConf}; lazy_static::lazy_static! { @@ -31,22 +32,26 @@ pub fn run_storage_sync_thread< let handle = thread::Builder::new() .name("Queue based relish storage sync".to_string()) - .spawn(move || loop { - let mut queue_accessor = UPLOAD_QUEUE.lock().unwrap(); - log::debug!("Upload queue length: {}", queue_accessor.len()); - let next_task = queue_accessor.pop(); - drop(queue_accessor); - match next_task { - Some(task) => runtime.block_on(async { - // suppress warnings - let _ = (config, task, &relish_storage, max_concurrent_sync); - todo!("omitted for brevity") - }), - None => { - thread::sleep(Duration::from_secs(1)); - continue; + .spawn(move || { + while !tenant_mgr::shutdown_requested() { + let mut queue_accessor = UPLOAD_QUEUE.lock().unwrap(); + log::debug!("Upload queue length: {}", queue_accessor.len()); + let next_task = queue_accessor.pop(); + drop(queue_accessor); + match next_task { + Some(task) => runtime.block_on(async { + // suppress warnings + let _ = (config, task, &relish_storage, max_concurrent_sync); + todo!("omitted for brevity") + }), + None => { + thread::sleep(Duration::from_secs(1)); + continue; + } } } + log::debug!("Queue based relish storage sync thread shut down"); + Ok(()) })?; Ok(Some(handle)) } diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index e8a7952d77..f1082513bc 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -13,6 +13,8 @@ use zenith_utils::zid::ZTimelineId; /// A repository corresponds to one .zenith directory. One repository holds multiple /// timelines, forked off from the same initial call to 'initdb'. pub trait Repository: Send + Sync { + fn shutdown(&self) -> Result<()>; + /// Get Timeline handle for given zenith timeline ID. fn get_timeline(&self, timelineid: ZTimelineId) -> Result>; diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 4eb46ba71a..1712cf1b8a 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -8,12 +8,14 @@ use crate::walredo::PostgresRedoManager; use crate::PageServerConf; use anyhow::{anyhow, bail, Context, Result}; use lazy_static::lazy_static; -use log::info; +use log::{debug, info}; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::fs; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; use zenith_utils::zid::{ZTenantId, ZTimelineId}; lazy_static! { @@ -24,6 +26,19 @@ lazy_static! { fn access_repository() -> MutexGuard<'static, HashMap>> { REPOSITORY.lock().unwrap() } +struct TenantHandleEntry { + checkpointer_handle: Option>, + gc_handle: Option>, +} + +// Logically these handles belong to Repository, +// but it's just simpler to store them separately +lazy_static! { + static ref TENANT_HANDLES: Mutex> = + Mutex::new(HashMap::new()); +} + +static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false); pub fn init(conf: &'static PageServerConf) { let mut m = access_repository(); @@ -47,8 +62,18 @@ fn init_repo(conf: &'static PageServerConf, tenant_id: ZTenantId) -> Arc bool { + SHUTDOWN_REQUESTED.load(Ordering::Relaxed) +} + +pub fn stop_tenant_threads(tenantid: ZTenantId) { + let mut handles = TENANT_HANDLES.lock().unwrap(); + if let Some(h) = handles.get_mut(&tenantid) { + h.checkpointer_handle.take().map(JoinHandle::join); + debug!("checkpointer for tenant {} has stopped", tenantid); + h.gc_handle.take().map(JoinHandle::join); + debug!("gc for tenant {} has stopped", tenantid); + } +} + +pub fn shutdown_all_tenants() -> Result<()> { + SHUTDOWN_REQUESTED.swap(true, Ordering::Relaxed); + + let tenants = list_tenants()?; + for tenantid in tenants { + stop_tenant_threads(tenantid); + let repo = get_repository_for_tenant(tenantid)?; + debug!("shutdown tenant {}", tenantid); + repo.shutdown()?; + } + + Ok(()) +} + pub fn create_repository_for_tenant( conf: &'static PageServerConf, tenantid: ZTenantId, @@ -115,3 +169,14 @@ pub fn get_timeline_for_tenant( .get_timeline(timelineid) .with_context(|| format!("cannot fetch timeline {}", timelineid)) } + +fn list_tenants() -> Result> { + let o = &mut REPOSITORY.lock().unwrap(); + + o.iter() + .map(|tenant| { + let (tenantid, _) = tenant; + Ok(*tenantid) + }) + .collect() +} diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 0fd9dfdf27..5602d8e1db 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -24,6 +24,7 @@ use std::str::FromStr; use std::sync::Mutex; use std::thread; use std::thread::sleep; +use std::thread::JoinHandle; use std::thread_local; use std::time::{Duration, SystemTime}; use tracing::*; @@ -36,6 +37,7 @@ use zenith_utils::zid::ZTimelineId; // struct WalReceiverEntry { wal_producer_connstr: String, + wal_receiver_handle: Option>, } lazy_static! { @@ -50,6 +52,19 @@ thread_local! { pub(crate) static IS_WAL_RECEIVER: Cell = Cell::new(false); } +// Wait for walreceiver to stop +// Now it stops when pageserver shutdown is requested. +// In future we can make this more granular and send shutdown signals +// per tenant/timeline to cancel inactive walreceivers. +// TODO deal with blocking pg connections +pub fn stop_wal_receiver(timelineid: ZTimelineId) { + let mut receivers = WAL_RECEIVERS.lock().unwrap(); + if let Some(r) = receivers.get_mut(&timelineid) { + r.wal_receiver_handle.take(); + // r.wal_receiver_handle.take().map(JoinHandle::join); + } +} + // Launch a new WAL receiver, or tell one that's running about change in connection string pub fn launch_wal_receiver( conf: &'static PageServerConf, @@ -64,19 +79,19 @@ pub fn launch_wal_receiver( receiver.wal_producer_connstr = wal_producer_connstr.into(); } None => { - let receiver = WalReceiverEntry { - wal_producer_connstr: wal_producer_connstr.into(), - }; - receivers.insert(timelineid, receiver); - - // Also launch a new thread to handle this connection - let _walreceiver_thread = thread::Builder::new() + let wal_receiver_handle = thread::Builder::new() .name("WAL receiver thread".into()) .spawn(move || { IS_WAL_RECEIVER.with(|c| c.set(true)); thread_main(conf, timelineid, tenantid); }) .unwrap(); + + let receiver = WalReceiverEntry { + wal_producer_connstr: wal_producer_connstr.into(), + wal_receiver_handle: Some(wal_receiver_handle), + }; + receivers.insert(timelineid, receiver); } }; } @@ -103,7 +118,7 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: // Make a connection to the WAL safekeeper, or directly to the primary PostgreSQL server, // and start streaming WAL from it. If the connection is lost, keep retrying. // - loop { + while !tenant_mgr::shutdown_requested() { // Look up the current WAL producer address let wal_producer_connstr = get_wal_producer_connstr(timelineid); @@ -117,6 +132,7 @@ fn thread_main(conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: sleep(Duration::from_secs(1)); } } + debug!("WAL streaming shut down"); } fn walreceiver_main( @@ -273,6 +289,11 @@ fn walreceiver_main( physical_stream.standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY)?; } + + if tenant_mgr::shutdown_requested() { + debug!("stop walreceiver because pageserver shutdown is requested"); + break; + } } Ok(()) } diff --git a/proxy/src/mgmt.rs b/proxy/src/mgmt.rs index 2b3259f8ec..1f33b68a1c 100644 --- a/proxy/src/mgmt.rs +++ b/proxy/src/mgmt.rs @@ -34,7 +34,7 @@ pub fn thread_main(state: &'static ProxyState, listener: TcpListener) -> anyhow: pub fn mgmt_conn_main(state: &'static ProxyState, socket: TcpStream) -> anyhow::Result<()> { let mut conn_handler = MgmtHandler { state }; - let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None)?; + let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None, true)?; pgbackend.run(&mut conn_handler) } diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index f246d4470a..61a742cf38 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -64,6 +64,7 @@ pub fn proxy_conn_main( socket, postgres_backend::AuthType::MD5, state.conf.ssl_config.clone(), + false, )?, md5_salt: [0u8; 4], psql_session_id: "".into(), diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index c0ce57801d..597c39db0b 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -375,6 +375,7 @@ class ZenithPageserver(PgProtocol): Start the page server. Returns self. """ + assert self.running == False self.zenith_cli.run(['start']) self.running = True @@ -382,14 +383,18 @@ class ZenithPageserver(PgProtocol): self.initial_tenant = self.zenith_cli.run(['tenant', 'list']).stdout.strip() return self - def stop(self) -> 'ZenithPageserver': + def stop(self, immediate=False) -> 'ZenithPageserver': """ Stop the page server. Returns self. """ + cmd = ['stop'] + if immediate: + cmd.append('immediate') + print(cmd) if self.running: - self.zenith_cli.run(['stop']) + self.zenith_cli.run(cmd) self.running = False return self @@ -398,7 +403,7 @@ class ZenithPageserver(PgProtocol): return self def __exit__(self, exc_type, exc, tb): - self.stop() + self.stop(True) @cached_property def auth_keys(self) -> AuthKeys: @@ -444,7 +449,7 @@ def pageserver(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: Pageserver # After the yield comes any cleanup code we need. print('Starting pageserver cleanup') - ps.stop() + ps.stop(True) class PgBin: """ A helper class for executing postgres binaries """ diff --git a/walkeeper/src/wal_service.rs b/walkeeper/src/wal_service.rs index 7730a058ed..a2c3f8d02b 100644 --- a/walkeeper/src/wal_service.rs +++ b/walkeeper/src/wal_service.rs @@ -41,7 +41,7 @@ fn handle_socket(socket: TcpStream, conf: WalAcceptorConf) -> Result<()> { socket.set_nodelay(true)?; let mut conn_handler = SendWalHandler::new(conf); - let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None)?; + let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None, false)?; // libpq replication protocol between wal_acceptor and replicas/pagers pgbackend.run(&mut conn_handler)?; diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 1c04e803e6..e86ce10041 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -88,7 +88,12 @@ fn main() -> Result<()> { ) .subcommand(SubCommand::with_name("status")) .subcommand(SubCommand::with_name("start").about("Start local pageserver")) - .subcommand(SubCommand::with_name("stop").about("Stop local pageserver")) + .subcommand(SubCommand::with_name("stop").about("Stop local pageserver") + .arg(Arg::with_name("immediate") + .help("Don't flush repository data at shutdown") + .required(false) + ) + ) .subcommand(SubCommand::with_name("restart").about("Restart local pageserver")) .subcommand( SubCommand::with_name("pg") @@ -196,10 +201,12 @@ fn main() -> Result<()> { } } - ("stop", Some(_sub_m)) => { + ("stop", Some(stop_match)) => { let pageserver = PageServerNode::from_env(&env); - if let Err(e) = pageserver.stop() { + let immediate = stop_match.is_present("immediate"); + + if let Err(e) = pageserver.stop(immediate) { eprintln!("pageserver stop failed: {}", e); exit(1); } @@ -208,7 +215,8 @@ fn main() -> Result<()> { ("restart", Some(_sub_m)) => { let pageserver = PageServerNode::from_env(&env); - if let Err(e) = pageserver.stop() { + //TODO what shutdown strategy should we use here? + if let Err(e) = pageserver.stop(false) { eprintln!("pageserver stop failed: {}", e); exit(1); } diff --git a/zenith_utils/src/http/endpoint.rs b/zenith_utils/src/http/endpoint.rs index 3c5b53b77a..30e7bfc921 100644 --- a/zenith_utils/src/http/endpoint.rs +++ b/zenith_utils/src/http/endpoint.rs @@ -12,8 +12,17 @@ use std::net::TcpListener; use zenith_metrics::{new_common_metric_name, register_int_counter, IntCounter}; use zenith_metrics::{Encoder, TextEncoder}; +use std::sync::Mutex; +use tokio::sync::oneshot::Sender; + use super::error::ApiError; +lazy_static! { + /// Channel used to send shutdown signal - wrapped in an Option to allow + /// it to be taken by value (since oneshot channels consume themselves on send) + static ref SHUTDOWN_SENDER: Mutex>> = Mutex::new(None); +} + lazy_static! { static ref SERVE_METRICS_COUNT: IntCounter = register_int_counter!( new_common_metric_name("serve_metrics_count"), @@ -143,11 +152,18 @@ pub fn check_permission(req: &Request, tenantid: Option) -> Res } } +// Send shutdown signal +pub fn shutdown() { + if let Some(tx) = SHUTDOWN_SENDER.lock().unwrap().take() { + let _ = tx.send(()); + } +} + pub fn serve_thread_main( router_builder: RouterBuilder, listener: TcpListener, ) -> anyhow::Result<()> { - log::info!("Starting a http endoint at {}", listener.local_addr()?); + log::info!("Starting a http endpoint at {}", listener.local_addr()?); // Create a Service from the router above to handle incoming requests. let service = RouterService::new(router_builder.build().map_err(|err| anyhow!(err))?).unwrap(); @@ -159,7 +175,14 @@ pub fn serve_thread_main( let _guard = runtime.enter(); - let server = Server::from_tcp(listener)?.serve(service); + let (send, recv) = tokio::sync::oneshot::channel::<()>(); + *SHUTDOWN_SENDER.lock().unwrap() = Some(send); + + let server = Server::from_tcp(listener)? + .serve(service) + .with_graceful_shutdown(async { + recv.await.ok(); + }); runtime.block_on(server)?; diff --git a/zenith_utils/src/postgres_backend.rs b/zenith_utils/src/postgres_backend.rs index b2e0a1a525..02eb330f3b 100644 --- a/zenith_utils/src/postgres_backend.rs +++ b/zenith_utils/src/postgres_backend.rs @@ -13,7 +13,11 @@ use serde::{Deserialize, Serialize}; use std::io::{self, Write}; use std::net::{Shutdown, SocketAddr, TcpStream}; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::time::Duration; + +static PGBACKEND_SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false); pub trait Handler { /// Handle single query. @@ -135,13 +139,32 @@ pub fn query_from_cstring(query_string: Bytes) -> Vec { query_string } +// Helper function for socket read loops +pub fn is_socket_read_timed_out(error: &anyhow::Error) -> bool { + for cause in error.chain() { + if let Some(io_error) = cause.downcast_ref::() { + if io_error.kind() == std::io::ErrorKind::WouldBlock { + return true; + } + } + } + false +} + impl PostgresBackend { pub fn new( socket: TcpStream, auth_type: AuthType, tls_config: Option>, + set_read_timeout: bool, ) -> io::Result { let peer_addr = socket.peer_addr()?; + if set_read_timeout { + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + } + Ok(Self { stream: Some(Stream::Bidirectional(BidiStream::from_tcp(socket))), buf_out: BytesMut::with_capacity(10 * 1024), @@ -229,12 +252,26 @@ impl PostgresBackend { let mut unnamed_query_string = Bytes::new(); - while let Some(msg) = self.read_message()? { - trace!("got message {:?}", msg); + while !PGBACKEND_SHUTDOWN_REQUESTED.load(Ordering::Relaxed) { + match self.read_message() { + Ok(message) => { + if let Some(msg) = message { + trace!("got message {:?}", msg); - match self.process_message(handler, msg, &mut unnamed_query_string)? { - ProcessMsgResult::Continue => continue, - ProcessMsgResult::Break => break, + match self.process_message(handler, msg, &mut unnamed_query_string)? { + ProcessMsgResult::Continue => continue, + ProcessMsgResult::Break => break, + } + } else { + break; + } + } + Err(e) => { + // If it is a timeout error, continue the loop + if !is_socket_read_timed_out(&e) { + return Err(e); + } + } } } @@ -427,3 +464,8 @@ impl PostgresBackend { Ok(ProcessMsgResult::Continue) } } + +// Set the flag to inform connections to cancel +pub fn set_pgbackend_shutdown_requested() { + PGBACKEND_SHUTDOWN_REQUESTED.swap(true, Ordering::Relaxed); +} diff --git a/zenith_utils/tests/ssl_test.rs b/zenith_utils/tests/ssl_test.rs index ba0f63d6ec..2a597700ae 100644 --- a/zenith_utils/tests/ssl_test.rs +++ b/zenith_utils/tests/ssl_test.rs @@ -110,7 +110,7 @@ fn ssl() { .unwrap(); let tls_config = Some(Arc::new(cfg)); - let pgb = PostgresBackend::new(server_sock, AuthType::Trust, tls_config).unwrap(); + let pgb = PostgresBackend::new(server_sock, AuthType::Trust, tls_config, true).unwrap(); pgb.run(&mut handler).unwrap(); assert!(handler.got_query); @@ -150,7 +150,7 @@ fn no_ssl() { let mut handler = TestHandler; - let pgb = PostgresBackend::new(server_sock, AuthType::Trust, None).unwrap(); + let pgb = PostgresBackend::new(server_sock, AuthType::Trust, None, true).unwrap(); pgb.run(&mut handler).unwrap(); client_jh.join().unwrap(); @@ -214,7 +214,7 @@ fn server_forces_ssl() { .unwrap(); let tls_config = Some(Arc::new(cfg)); - let pgb = PostgresBackend::new(server_sock, AuthType::Trust, tls_config).unwrap(); + let pgb = PostgresBackend::new(server_sock, AuthType::Trust, tls_config, true).unwrap(); let res = pgb.run(&mut handler).unwrap_err(); assert_eq!("client did not connect with TLS", format!("{}", res)); From 8c61c3e54e2bf9def6da8ae21951acf28c3344a6 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Mon, 11 Oct 2021 16:31:44 +0300 Subject: [PATCH 11/46] Minor safekeeper readme fix. --- walkeeper/README | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/walkeeper/README b/walkeeper/README index 6c5a69e926..05325bafd9 100644 --- a/walkeeper/README +++ b/walkeeper/README @@ -89,12 +89,12 @@ A: Page Server is a single server which can be lost. As our primary Q: What if the compute node evicts a page, needs it back, but the page is yet to reach the Page Server? -A: If the compute node has evicted a page, all changes from that page are - already committed, i.e. they are saved on majority of WAL safekeepers. These - WAL records will eventually reach the Page Server. The Page Server notes - that the compute note requests pages with a very recent LSN and will not - respond to the compute node until it a corresponding WAL is received from WAL - safekeepers. +A: If the compute node has evicted a page, changes to it have been WAL-logged + (that's why it is called Write Ahead logging; there are some exceptions like + index builds, but these are exceptions). These WAL records will eventually + reach the Page Server. The Page Server notes that the compute note requests + pages with a very recent LSN and will not respond to the compute node until a + corresponding WAL is received from WAL safekeepers. Q: How long may Page Server wait for? A: Not too long, hopefully. If a page is evicted, it probably was not used for From 1cd7900790c5ebc418903c3ffa2d1ee126d0b0c9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 11 Oct 2021 16:00:52 +0300 Subject: [PATCH 12/46] [Makefile] Make build type detection more precise Previously, typos like `BUILD_TYPE=rlease` would silently lead to building debug binaries. The current approach is also more future-proof, since we might add `profile`, `valgrind` as well as other build types. --- Makefile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 2edf2a6b4a..bdfa59da37 100644 --- a/Makefile +++ b/Makefile @@ -12,15 +12,18 @@ endif # ifeq ($(BUILD_TYPE),release) PG_CONFIGURE_OPTS = --enable-debug - PG_CFLAGS = -O2 -g3 ${CFLAGS} -else + PG_CFLAGS = -O2 -g3 $(CFLAGS) +else ifeq ($(BUILD_TYPE),debug) PG_CONFIGURE_OPTS = --enable-debug --enable-cassert --enable-depend - PG_CFLAGS = -O0 -g3 ${CFLAGS} + PG_CFLAGS = -O0 -g3 $(CFLAGS) +else +$(error Bad build type `$(BUILD_TYPE)', see Makefile for options) endif # # Top level Makefile to build Zenith and PostgreSQL # +.PHONY: all all: zenith postgres # We don't want to run 'cargo build' in parallel with the postgres build, @@ -68,15 +71,18 @@ postgres: postgres-configure +@echo "Compiling contrib/zenith_test_utils" $(MAKE) -C tmp_install/build/contrib/zenith_test_utils install +.PHONY: postgres-clean postgres-clean: $(MAKE) -C tmp_install/build MAKELEVEL=0 clean # This doesn't remove the effects of 'configure'. +.PHONY: clean clean: - cd tmp_install/build && ${MAKE} clean + cd tmp_install/build && $(MAKE) clean cargo clean # This removes everything +.PHONY: distclean distclean: rm -rf tmp_install cargo clean From bb239b4f699c4b84d1e69038bd5ccd96efb46ec2 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 11 Oct 2021 17:08:31 +0300 Subject: [PATCH 13/46] [Makefile] Set default build type to debug --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index bdfa59da37..8716372b25 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ endif # We differentiate between release / debug build types using the BUILD_TYPE # environment variable. # +BUILD_TYPE ?= debug ifeq ($(BUILD_TYPE),release) PG_CONFIGURE_OPTS = --enable-debug PG_CFLAGS = -O2 -g3 $(CFLAGS) From 934fb8592fabb2d130142bf6b9d42c04bfba64f9 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 12 Oct 2021 09:09:10 +0300 Subject: [PATCH 14/46] Detect when a checkpoint is modified in a smarter way. Previously, the WAL receiver we would make a decoded copy of the current Checkpoint before each WAL record, and compare it with the Checkpoint after the record has been processed. If it has changed, the checkpoint relish is updated in the repository. That's somewhat expensive, the Checkpoint::encode() function is visible in 'perf' profile. Change that so that we set a flag whenever the Checkpoint struct is modified, so that we dont need to compare the whole struct anymore. --- pageserver/src/restore_local_repo.rs | 35 +++++++++++++++++++++++----- pageserver/src/walreceiver.rs | 13 ++++++----- postgres_ffi/src/xlog_utils.rs | 20 ++++++++++------ 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/pageserver/src/restore_local_repo.rs b/pageserver/src/restore_local_repo.rs index ec6ee61d52..7f3e35e4b5 100644 --- a/pageserver/src/restore_local_repo.rs +++ b/pageserver/src/restore_local_repo.rs @@ -286,12 +286,15 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa /// pub fn save_decoded_record( checkpoint: &mut CheckPoint, + checkpoint_modified: &mut bool, timeline: &dyn Timeline, decoded: &DecodedWALRecord, recdata: Bytes, lsn: Lsn, ) -> Result<()> { - checkpoint.update_next_xid(decoded.xl_xid); + if checkpoint.update_next_xid(decoded.xl_xid) { + *checkpoint_modified = true; + } // Iterate through all the blocks that the record modifies, and // "put" a separate copy of the record for each block. @@ -375,7 +378,7 @@ pub fn save_decoded_record( } else { assert!(info == pg_constants::CLOG_TRUNCATE); let xlrec = XlClogTruncate::decode(&mut buf); - save_clog_truncate_record(checkpoint, timeline, lsn, &xlrec)?; + save_clog_truncate_record(checkpoint, checkpoint_modified, timeline, lsn, &xlrec)?; } } else if decoded.xl_rmid == pg_constants::RM_XACT_ID { let info = decoded.xl_info & pg_constants::XLOG_XACT_OPMASK; @@ -444,10 +447,17 @@ pub fn save_decoded_record( )?; } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID { let xlrec = XlMultiXactCreate::decode(&mut buf); - save_multixact_create_record(checkpoint, timeline, lsn, &xlrec, decoded)?; + save_multixact_create_record( + checkpoint, + checkpoint_modified, + timeline, + lsn, + &xlrec, + decoded, + )?; } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID { let xlrec = XlMultiXactTruncate::decode(&mut buf); - save_multixact_truncate_record(checkpoint, timeline, lsn, &xlrec)?; + save_multixact_truncate_record(checkpoint, checkpoint_modified, timeline, lsn, &xlrec)?; } } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID { let xlrec = XlRelmapUpdate::decode(&mut buf); @@ -456,7 +466,10 @@ pub fn save_decoded_record( let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; if info == pg_constants::XLOG_NEXTOID { let next_oid = buf.get_u32_le(); - checkpoint.nextOid = next_oid; + if checkpoint.nextOid != next_oid { + checkpoint.nextOid = next_oid; + *checkpoint_modified = true; + } } else if info == pg_constants::XLOG_CHECKPOINT_ONLINE || info == pg_constants::XLOG_CHECKPOINT_SHUTDOWN { @@ -472,6 +485,7 @@ pub fn save_decoded_record( ); if (checkpoint.oldestXid.wrapping_sub(xlog_checkpoint.oldestXid) as i32) < 0 { checkpoint.oldestXid = xlog_checkpoint.oldestXid; + *checkpoint_modified = true; } } } @@ -675,6 +689,7 @@ fn save_xact_record( fn save_clog_truncate_record( checkpoint: &mut CheckPoint, + checkpoint_modified: &mut bool, timeline: &dyn Timeline, lsn: Lsn, xlrec: &XlClogTruncate, @@ -693,6 +708,7 @@ fn save_clog_truncate_record( // TODO Figure out if there will be any issues with replica. checkpoint.oldestXid = xlrec.oldest_xid; checkpoint.oldestXidDB = xlrec.oldest_xid_db; + *checkpoint_modified = true; // TODO Treat AdvanceOldestClogXid() or write a comment why we don't need it @@ -735,6 +751,7 @@ fn save_clog_truncate_record( fn save_multixact_create_record( checkpoint: &mut CheckPoint, + checkpoint_modified: &mut bool, timeline: &dyn Timeline, lsn: Lsn, xlrec: &XlMultiXactCreate, @@ -791,9 +808,11 @@ fn save_multixact_create_record( } if xlrec.mid >= checkpoint.nextMulti { checkpoint.nextMulti = xlrec.mid + 1; + *checkpoint_modified = true; } if xlrec.moff + xlrec.nmembers > checkpoint.nextMultiOffset { checkpoint.nextMultiOffset = xlrec.moff + xlrec.nmembers; + *checkpoint_modified = true; } let max_mbr_xid = xlrec.members.iter().fold(0u32, |acc, mbr| { if mbr.xid.wrapping_sub(acc) as i32 > 0 { @@ -803,18 +822,22 @@ fn save_multixact_create_record( } }); - checkpoint.update_next_xid(max_mbr_xid); + if checkpoint.update_next_xid(max_mbr_xid) { + *checkpoint_modified = true; + } Ok(()) } fn save_multixact_truncate_record( checkpoint: &mut CheckPoint, + checkpoint_modified: &mut bool, timeline: &dyn Timeline, lsn: Lsn, xlrec: &XlMultiXactTruncate, ) -> Result<()> { checkpoint.oldestMulti = xlrec.end_trunc_off; checkpoint.oldestMultiDB = xlrec.oldest_multi_db; + *checkpoint_modified = true; // PerformMembersTruncation let maxsegment: i32 = mx_offset_to_member_segment(pg_constants::MAX_MULTIXACT_OFFSET); diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 5602d8e1db..afb8c8c4b0 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -214,26 +214,27 @@ fn walreceiver_main( while let Some((lsn, recdata)) = waldecoder.poll_decode()? { let _enter = info_span!("processing record", lsn = %lsn).entered(); - // Save old checkpoint value to compare with it after decoding WAL record - let old_checkpoint_bytes = checkpoint.encode(); - let decoded = decode_wal_record(recdata.clone()); - // It is important to deal with the aligned records as lsn in getPage@LSN is // aligned and can be several bytes bigger. Without this alignment we are // at risk of hittind a deadlock. assert!(lsn.is_aligned()); + let mut checkpoint_modified = false; + + let decoded = decode_wal_record(recdata.clone()); restore_local_repo::save_decoded_record( &mut checkpoint, + &mut checkpoint_modified, &*timeline, &decoded, recdata, lsn, )?; - let new_checkpoint_bytes = checkpoint.encode(); // Check if checkpoint data was updated by save_decoded_record - if new_checkpoint_bytes != old_checkpoint_bytes { + if checkpoint_modified { + let new_checkpoint_bytes = checkpoint.encode(); + timeline.put_page_image( RelishTag::Checkpoint, 0, diff --git a/postgres_ffi/src/xlog_utils.rs b/postgres_ffi/src/xlog_utils.rs index c4caa18b32..5cdaec4a2a 100644 --- a/postgres_ffi/src/xlog_utils.rs +++ b/postgres_ffi/src/xlog_utils.rs @@ -377,10 +377,12 @@ impl CheckPoint { Ok(CheckPoint::des(buf)?) } - // Update next XID based on provided new_xid and stored epoch. - // Next XID should be greater than new_xid. - // Also take in account 32-bit wrap-around. - pub fn update_next_xid(&mut self, xid: u32) { + /// Update next XID based on provided new_xid and stored epoch. + /// Next XID should be greater than new_xid. This handles 32-bit + /// XID wraparound correctly. + /// + /// Returns 'true' if the XID was updated. + pub fn update_next_xid(&mut self, xid: u32) -> bool { let xid = xid.wrapping_add(XID_CHECKPOINT_INTERVAL - 1) & !(XID_CHECKPOINT_INTERVAL - 1); let full_xid = self.nextXid.value; let new_xid = std::cmp::max(xid + 1, pg_constants::FIRST_NORMAL_TRANSACTION_ID); @@ -391,10 +393,14 @@ impl CheckPoint { // wrap-around epoch += 1; } - self.nextXid = FullTransactionId { - value: (epoch << 32) | new_xid as u64, - }; + let nextXid = (epoch << 32) | new_xid as u64; + + if nextXid != self.nextXid.value { + self.nextXid = FullTransactionId { value: nextXid }; + return true; + } } + false } } From 95a85312f549691a5acc448672955427df2aff1e Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 11 Oct 2021 20:02:47 +0300 Subject: [PATCH 15/46] Simplify code to build walredo messages. No need to use BytesMut in these functions. Plain Vec is simpler. And should be marginally faster too; I saw BytesMut functions previously in 'perf' profile, consuming around 5% of the overall pageserver CPU time. That's gone with this patch, although I don't see any discernible difference in the overall performance test results. --- pageserver/src/walredo.rs | 61 ++++++++++++--------------------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index f233fceb3e..e9382c4da5 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -565,22 +565,16 @@ impl PostgresRedoProcess { stdin.write_all(&build_begin_redo_for_block_msg(tag)), ) .await??; - if base_img.is_some() { - timeout( - TIMEOUT, - stdin.write_all(&build_push_page_msg(tag, base_img.unwrap())), - ) - .await??; + if let Some(img) = base_img { + timeout(TIMEOUT, stdin.write_all(&build_push_page_msg(tag, &img))).await??; } // Send WAL records. for rec in records.iter() { - let r = rec.clone(); - WAL_REDO_RECORD_COUNTER.inc(); stdin - .write_all(&build_apply_record_msg(r.lsn, r.rec)) + .write_all(&build_apply_record_msg(rec.lsn, &rec.rec)) .await?; //debug!("sent WAL record to wal redo postgres process ({:X}/{:X}", @@ -617,58 +611,41 @@ impl PostgresRedoProcess { // process. See vendor/postgres/src/backend/tcop/zenith_wal_redo.c for // explanation of the protocol. -fn build_begin_redo_for_block_msg(tag: BufferTag) -> Bytes { +fn build_begin_redo_for_block_msg(tag: BufferTag) -> Vec { let len = 4 + 1 + 4 * 4; - let mut buf = BytesMut::with_capacity(1 + len); + let mut buf = Vec::with_capacity(1 + len); buf.put_u8(b'B'); buf.put_u32(len as u32); - // FIXME: this is a temporary hack that should go away when we refactor - // the postgres protocol serialization + handlers. - // - // BytesMut is a dynamic growable buffer, used a lot in tokio code but - // not in the std library. To write to a BytesMut from a serde serializer, - // we need to either: - // - pre-allocate the required buffer space. This is annoying because we - // shouldn't care what the exact serialized size is-- that's the - // serializer's job. - // - Or, we need to create a temporary "writer" (which implements the - // `Write` trait). It's a bit awkward, because the writer consumes the - // underlying BytesMut, and we need to extract it later with - // `into_inner`. - let mut writer = buf.writer(); - tag.ser_into(&mut writer) + tag.ser_into(&mut buf) .expect("serialize BufferTag should always succeed"); - let buf = writer.into_inner(); debug_assert!(buf.len() == 1 + len); - buf.freeze() + buf } -fn build_push_page_msg(tag: BufferTag, base_img: Bytes) -> Bytes { +fn build_push_page_msg(tag: BufferTag, base_img: &[u8]) -> Vec { assert!(base_img.len() == 8192); let len = 4 + 1 + 4 * 4 + base_img.len(); - let mut buf = BytesMut::with_capacity(1 + len); + let mut buf = Vec::with_capacity(1 + len); buf.put_u8(b'P'); buf.put_u32(len as u32); - let mut writer = buf.writer(); - tag.ser_into(&mut writer) + tag.ser_into(&mut buf) .expect("serialize BufferTag should always succeed"); - let mut buf = writer.into_inner(); buf.put(base_img); debug_assert!(buf.len() == 1 + len); - buf.freeze() + buf } -fn build_apply_record_msg(endlsn: Lsn, rec: Bytes) -> Bytes { +fn build_apply_record_msg(endlsn: Lsn, rec: &[u8]) -> Vec { let len = 4 + 8 + rec.len(); - let mut buf = BytesMut::with_capacity(1 + len); + let mut buf: Vec = Vec::with_capacity(1 + len); buf.put_u8(b'A'); buf.put_u32(len as u32); @@ -677,21 +654,19 @@ fn build_apply_record_msg(endlsn: Lsn, rec: Bytes) -> Bytes { debug_assert!(buf.len() == 1 + len); - buf.freeze() + buf } -fn build_get_page_msg(tag: BufferTag) -> Bytes { +fn build_get_page_msg(tag: BufferTag) -> Vec { let len = 4 + 1 + 4 * 4; - let mut buf = BytesMut::with_capacity(1 + len); + let mut buf = Vec::with_capacity(1 + len); buf.put_u8(b'G'); buf.put_u32(len as u32); - let mut writer = buf.writer(); - tag.ser_into(&mut writer) + tag.ser_into(&mut buf) .expect("serialize BufferTag should always succeed"); - let buf = writer.into_inner(); debug_assert!(buf.len() == 1 + len); - buf.freeze() + buf } From f3445949d19093d7d9b998e44944be437cdd159d Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Tue, 12 Oct 2021 16:51:28 +0300 Subject: [PATCH 16/46] Wal acceptor: report socket bind errors better when daemonizing (#738) Fixes #664 --- walkeeper/src/bin/wal_acceptor.rs | 10 ++++++++-- walkeeper/src/wal_service.rs | 8 +------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/walkeeper/src/bin/wal_acceptor.rs b/walkeeper/src/bin/wal_acceptor.rs index bfb09db5cb..f961c784d7 100644 --- a/walkeeper/src/bin/wal_acceptor.rs +++ b/walkeeper/src/bin/wal_acceptor.rs @@ -136,6 +136,12 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { e })?; + info!("Starting wal acceptor on {}", conf.listen_pg_addr); + let pg_listener = TcpListener::bind(conf.listen_pg_addr.clone()).map_err(|e| { + error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); + e + })?; + if conf.daemonize { info!("daemonizing..."); @@ -160,7 +166,7 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { let http_endpoint_thread = thread::Builder::new() .name("http_endpoint_thread".into()) - .spawn(move || { + .spawn(|| { // No authentication at all: read-only metrics only, early stage. let router = endpoint::make_router(); endpoint::serve_thread_main(router, http_listener).unwrap(); @@ -184,7 +190,7 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { .name("WAL acceptor thread".into()) .spawn(|| { // thread code - let thread_result = wal_service::thread_main(conf); + let thread_result = wal_service::thread_main(conf, pg_listener); if let Err(e) = thread_result { info!("wal_service thread terminated: {}", e); } diff --git a/walkeeper/src/wal_service.rs b/walkeeper/src/wal_service.rs index a2c3f8d02b..747c06a116 100644 --- a/walkeeper/src/wal_service.rs +++ b/walkeeper/src/wal_service.rs @@ -12,13 +12,7 @@ use crate::WalAcceptorConf; use zenith_utils::postgres_backend::{AuthType, PostgresBackend}; /// Accept incoming TCP connections and spawn them into a background thread. -pub fn thread_main(conf: WalAcceptorConf) -> Result<()> { - info!("Starting wal acceptor on {}", conf.listen_pg_addr); - let listener = TcpListener::bind(conf.listen_pg_addr.clone()).map_err(|e| { - error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); - e - })?; - +pub fn thread_main(conf: WalAcceptorConf, listener: TcpListener) -> Result<()> { loop { match listener.accept() { Ok((socket, peer_addr)) => { From 6e5ca5dc5c593334c64e47dfe6bbcc950f41cd69 Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Fri, 8 Oct 2021 16:24:49 -0700 Subject: [PATCH 17/46] pageserver - create TimelineWriter --- pageserver/src/branches.rs | 2 +- pageserver/src/layered_repository.rs | 296 +++++++++++++++------------ pageserver/src/repository.rs | 158 +++++++------- pageserver/src/restore_local_repo.rs | 59 +++--- pageserver/src/walreceiver.rs | 8 +- 5 files changed, 291 insertions(+), 232 deletions(-) diff --git a/pageserver/src/branches.rs b/pageserver/src/branches.rs index 4a30e760b5..00fd50a2a2 100644 --- a/pageserver/src/branches.rs +++ b/pageserver/src/branches.rs @@ -224,7 +224,7 @@ fn bootstrap_timeline( let timeline = repo.create_empty_timeline(tli)?; restore_local_repo::import_timeline_from_postgres_datadir( &pgdata_path, - timeline.as_ref(), + timeline.writer().as_ref(), lsn, )?; timeline.checkpoint()?; diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index b3164f6cc1..c6cd9094f6 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -26,7 +26,7 @@ use std::convert::TryInto; use std::fs; use std::fs::{File, OpenOptions}; use std::io::Write; -use std::ops::Bound::Included; +use std::ops::{Bound::Included, Deref}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; @@ -36,7 +36,7 @@ use std::time::{Duration, Instant}; use crate::layered_repository::inmemory_layer::FreezeLayers; use crate::relish::*; use crate::relish_storage::schedule_timeline_upload; -use crate::repository::{GcResult, Repository, Timeline, WALRecord}; +use crate::repository::{GcResult, Repository, Timeline, TimelineWriter, WALRecord}; use crate::tenant_mgr; use crate::walreceiver; use crate::walreceiver::IS_WAL_RECEIVER; @@ -841,131 +841,6 @@ impl Timeline for LayeredTimeline { Ok(result) } - fn put_wal_record(&self, rel: RelishTag, blknum: u32, rec: WALRecord) -> Result<()> { - if !rel.is_blocky() && blknum != 0 { - bail!( - "invalid request for block {} for non-blocky relish {}", - blknum, - rel - ); - } - ensure!(rec.lsn.is_aligned(), "unaligned record LSN"); - - let seg = SegmentTag::from_blknum(rel, blknum); - let delta_size = self.perform_write_op(seg, rec.lsn, |layer| { - layer.put_wal_record(blknum, rec.clone()) - })?; - self.increase_current_logical_size(delta_size * BLCKSZ as u32); - Ok(()) - } - - fn put_truncation(&self, rel: RelishTag, lsn: Lsn, relsize: u32) -> anyhow::Result<()> { - if !rel.is_blocky() { - bail!("invalid truncation for non-blocky relish {}", rel); - } - ensure!(lsn.is_aligned(), "unaligned record LSN"); - - debug!("put_truncation: {} to {} blocks at {}", rel, relsize, lsn); - - let oldsize = self - .get_relish_size(rel, self.get_last_record_lsn())? - .ok_or_else(|| { - anyhow!( - "attempted to truncate non-existent relish {} at {}", - rel, - lsn - ) - })?; - - if oldsize <= relsize { - return Ok(()); - } - let old_last_seg = (oldsize - 1) / RELISH_SEG_SIZE; - - let last_remain_seg = if relsize == 0 { - 0 - } else { - (relsize - 1) / RELISH_SEG_SIZE - }; - - // Drop segments beyond the last remaining segment. - for remove_segno in (last_remain_seg + 1)..=old_last_seg { - let seg = SegmentTag { - rel, - segno: remove_segno, - }; - self.perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; - } - - // Truncate the last remaining segment to the specified size - if relsize == 0 || relsize % RELISH_SEG_SIZE != 0 { - let seg = SegmentTag { - rel, - segno: last_remain_seg, - }; - self.perform_write_op(seg, lsn, |layer| { - layer.put_truncation(lsn, relsize % RELISH_SEG_SIZE) - })?; - } - self.decrease_current_logical_size((oldsize - relsize) * BLCKSZ as u32); - Ok(()) - } - - fn drop_relish(&self, rel: RelishTag, lsn: Lsn) -> Result<()> { - trace!("drop_segment: {} at {}", rel, lsn); - - if rel.is_blocky() { - if let Some(oldsize) = self.get_relish_size(rel, self.get_last_record_lsn())? { - let old_last_seg = if oldsize == 0 { - 0 - } else { - (oldsize - 1) / RELISH_SEG_SIZE - }; - - // Drop all segments of the relish - for remove_segno in 0..=old_last_seg { - let seg = SegmentTag { - rel, - segno: remove_segno, - }; - self.perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; - } - self.decrease_current_logical_size(oldsize * BLCKSZ as u32); - } else { - warn!( - "drop_segment called on non-existent relish {} at {}", - rel, lsn - ); - } - } else { - // TODO handle TwoPhase relishes - let seg = SegmentTag::from_blknum(rel, 0); - self.perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; - } - - Ok(()) - } - - fn put_page_image(&self, rel: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> { - if !rel.is_blocky() && blknum != 0 { - bail!( - "invalid request for block {} for non-blocky relish {}", - blknum, - rel - ); - } - ensure!(lsn.is_aligned(), "unaligned record LSN"); - - let seg = SegmentTag::from_blknum(rel, blknum); - - let delta_size = self.perform_write_op(seg, lsn, |layer| { - layer.put_page_image(blknum, lsn, img.clone()) - })?; - - self.increase_current_logical_size(delta_size * BLCKSZ as u32); - Ok(()) - } - /// Public entry point for checkpoint(). All the logic is in the private /// checkpoint_internal function, this public facade just wraps it for /// metrics collection. @@ -976,15 +851,6 @@ impl Timeline for LayeredTimeline { .observe_closure_duration(|| self.checkpoint_internal(0, true)) } - /// - /// Remember the (end of) last valid WAL record remembered in the timeline. - /// - fn advance_last_record_lsn(&self, new_lsn: Lsn) { - assert!(new_lsn.is_aligned()); - - self.last_record_lsn.advance(new_lsn); - } - fn get_last_record_lsn(&self) -> Lsn { self.last_record_lsn.load().last } @@ -1035,6 +901,10 @@ impl Timeline for LayeredTimeline { Ok(total_blocks * BLCKSZ as usize) } + + fn writer<'a>(&'a self) -> Box { + Box::new(LayeredTimelineWriter(self)) + } } impl LayeredTimeline { @@ -1923,6 +1793,160 @@ impl LayeredTimeline { } } +struct LayeredTimelineWriter<'a>(&'a LayeredTimeline); + +impl Deref for LayeredTimelineWriter<'_> { + type Target = dyn Timeline; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { + fn put_wal_record(&self, rel: RelishTag, blknum: u32, rec: WALRecord) -> Result<()> { + if !rel.is_blocky() && blknum != 0 { + bail!( + "invalid request for block {} for non-blocky relish {}", + blknum, + rel + ); + } + ensure!(rec.lsn.is_aligned(), "unaligned record LSN"); + + let seg = SegmentTag::from_blknum(rel, blknum); + let delta_size = self.0.perform_write_op(seg, rec.lsn, |layer| { + layer.put_wal_record(blknum, rec.clone()) + })?; + self.0 + .increase_current_logical_size(delta_size * BLCKSZ as u32); + Ok(()) + } + + fn put_page_image(&self, rel: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()> { + if !rel.is_blocky() && blknum != 0 { + bail!( + "invalid request for block {} for non-blocky relish {}", + blknum, + rel + ); + } + ensure!(lsn.is_aligned(), "unaligned record LSN"); + + let seg = SegmentTag::from_blknum(rel, blknum); + + let delta_size = self.0.perform_write_op(seg, lsn, |layer| { + layer.put_page_image(blknum, lsn, img.clone()) + })?; + + self.0 + .increase_current_logical_size(delta_size * BLCKSZ as u32); + Ok(()) + } + + fn put_truncation(&self, rel: RelishTag, lsn: Lsn, relsize: u32) -> Result<()> { + if !rel.is_blocky() { + bail!("invalid truncation for non-blocky relish {}", rel); + } + ensure!(lsn.is_aligned(), "unaligned record LSN"); + + debug!("put_truncation: {} to {} blocks at {}", rel, relsize, lsn); + + let oldsize = self + .0 + .get_relish_size(rel, self.0.get_last_record_lsn())? + .ok_or_else(|| { + anyhow!( + "attempted to truncate non-existent relish {} at {}", + rel, + lsn + ) + })?; + + if oldsize <= relsize { + return Ok(()); + } + let old_last_seg = (oldsize - 1) / RELISH_SEG_SIZE; + + let last_remain_seg = if relsize == 0 { + 0 + } else { + (relsize - 1) / RELISH_SEG_SIZE + }; + + // Drop segments beyond the last remaining segment. + for remove_segno in (last_remain_seg + 1)..=old_last_seg { + let seg = SegmentTag { + rel, + segno: remove_segno, + }; + self.0 + .perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; + } + + // Truncate the last remaining segment to the specified size + if relsize == 0 || relsize % RELISH_SEG_SIZE != 0 { + let seg = SegmentTag { + rel, + segno: last_remain_seg, + }; + self.0.perform_write_op(seg, lsn, |layer| { + layer.put_truncation(lsn, relsize % RELISH_SEG_SIZE) + })?; + } + self.0 + .decrease_current_logical_size((oldsize - relsize) * BLCKSZ as u32); + Ok(()) + } + + fn drop_relish(&self, rel: RelishTag, lsn: Lsn) -> Result<()> { + trace!("drop_segment: {} at {}", rel, lsn); + + if rel.is_blocky() { + if let Some(oldsize) = self.0.get_relish_size(rel, self.0.get_last_record_lsn())? { + let old_last_seg = if oldsize == 0 { + 0 + } else { + (oldsize - 1) / RELISH_SEG_SIZE + }; + + // Drop all segments of the relish + for remove_segno in 0..=old_last_seg { + let seg = SegmentTag { + rel, + segno: remove_segno, + }; + self.0 + .perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; + } + self.0 + .decrease_current_logical_size(oldsize * BLCKSZ as u32); + } else { + warn!( + "drop_segment called on non-existent relish {} at {}", + rel, lsn + ); + } + } else { + // TODO handle TwoPhase relishes + let seg = SegmentTag::from_blknum(rel, 0); + self.0 + .perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; + } + + Ok(()) + } + + /// + /// Remember the (end of) last valid WAL record remembered in the timeline. + /// + fn advance_last_record_lsn(&self, new_lsn: Lsn) { + assert!(new_lsn.is_aligned()); + + self.0.last_record_lsn.advance(new_lsn); + } +} + /// Dump contents of a layer file to stdout. pub fn dump_layerfile_from_path(path: &Path) -> Result<()> { let file = File::open(path)?; diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index f1082513bc..f033c74c31 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -3,7 +3,7 @@ use anyhow::Result; use bytes::{Buf, BufMut, Bytes, BytesMut}; use serde::{Deserialize, Serialize}; use std::collections::HashSet; -use std::ops::AddAssign; +use std::ops::{AddAssign, Deref}; use std::sync::Arc; use std::time::Duration; use zenith_utils::lsn::{Lsn, RecordLsn}; @@ -125,6 +125,39 @@ pub trait Timeline: Send + Sync { // These are called by the WAL receiver to digest WAL records. //------------------------------------------------------------------------------ + /// Atomically get both last and prev. + fn get_last_record_rlsn(&self) -> RecordLsn; + /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev. + fn get_last_record_lsn(&self) -> Lsn; + fn get_prev_record_lsn(&self) -> Lsn; + fn get_start_lsn(&self) -> Lsn; + + /// Mutate the timeline with a [`TimelineWriter`]. + fn writer<'a>(&'a self) -> Box; + + /// + /// Flush to disk all data that was written with the put_* functions + /// + /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't + /// know anything about them here in the repository. + fn checkpoint(&self) -> Result<()>; + + /// Retrieve current logical size of the timeline + /// + /// NOTE: counted incrementally, includes ancestors, + /// doesnt support TwoPhase relishes yet + fn get_current_logical_size(&self) -> usize; + + /// Does the same as get_current_logical_size but counted on demand. + /// Used in tests to ensure thet incremental and non incremental variants match. + fn get_current_logical_size_non_incremental(&self, lsn: Lsn) -> Result; +} + +/// Various functions to mutate the timeline. +// TODO Currently, Deref is used to allow easy access to read methods from this trait. +// This is probably considered a bad practice in Rust and should be fixed eventually, +// but will cause large code changes. +pub trait TimelineWriter: Deref { /// Put a new page version that can be constructed from a WAL record /// /// This will implicitly extend the relation, if the page is beyond the @@ -145,29 +178,6 @@ pub trait Timeline: Send + Sync { /// Advance requires aligned LSN as an argument and would wake wait_lsn() callers. /// Previous last record LSN is stored alongside the latest and can be read. fn advance_last_record_lsn(&self, lsn: Lsn); - /// Atomically get both last and prev. - fn get_last_record_rlsn(&self) -> RecordLsn; - /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev. - fn get_last_record_lsn(&self) -> Lsn; - fn get_prev_record_lsn(&self) -> Lsn; - fn get_start_lsn(&self) -> Lsn; - - /// - /// Flush to disk all data that was written with the put_* functions - /// - /// NOTE: This has nothing to do with checkpoint in PostgreSQL. We don't - /// know anything about them here in the repository. - fn checkpoint(&self) -> Result<()>; - - /// Retrieve current logical size of the timeline - /// - /// NOTE: counted incrementally, includes ancestors, - /// doesnt support TwoPhase relishes yet - fn get_current_logical_size(&self) -> usize; - - /// Does the same as get_current_logical_size but counted on demand. - /// Used in tests to ensure thet incremental and non incremental variants match. - fn get_current_logical_size_non_incremental(&self, lsn: Lsn) -> Result; } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -308,14 +318,15 @@ mod tests { // Create timeline to work on let tline = repo.create_empty_timeline(TIMELINE_ID)?; + let writer = tline.writer(); - tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; - tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; - tline.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?; - tline.put_page_image(TESTREL_A, 1, Lsn(0x40), TEST_IMG("foo blk 1 at 4"))?; - tline.put_page_image(TESTREL_A, 2, Lsn(0x50), TEST_IMG("foo blk 2 at 5"))?; + writer.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; + writer.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; + writer.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?; + writer.put_page_image(TESTREL_A, 1, Lsn(0x40), TEST_IMG("foo blk 1 at 4"))?; + writer.put_page_image(TESTREL_A, 2, Lsn(0x50), TEST_IMG("foo blk 2 at 5"))?; - tline.advance_last_record_lsn(Lsn(0x50)); + writer.advance_last_record_lsn(Lsn(0x50)); assert_current_logical_size(&tline, Lsn(0x50)); @@ -361,8 +372,8 @@ mod tests { ); // Truncate last block - tline.put_truncation(TESTREL_A, Lsn(0x60), 2)?; - tline.advance_last_record_lsn(Lsn(0x60)); + writer.put_truncation(TESTREL_A, Lsn(0x60), 2)?; + writer.advance_last_record_lsn(Lsn(0x60)); assert_current_logical_size(&tline, Lsn(0x60)); // Check reported size and contents after truncation @@ -384,13 +395,13 @@ mod tests { ); // Truncate to zero length - tline.put_truncation(TESTREL_A, Lsn(0x68), 0)?; - tline.advance_last_record_lsn(Lsn(0x68)); + writer.put_truncation(TESTREL_A, Lsn(0x68), 0)?; + writer.advance_last_record_lsn(Lsn(0x68)); assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x68))?.unwrap(), 0); // Extend from 0 to 2 blocks, leaving a gap - tline.put_page_image(TESTREL_A, 1, Lsn(0x70), TEST_IMG("foo blk 1"))?; - tline.advance_last_record_lsn(Lsn(0x70)); + writer.put_page_image(TESTREL_A, 1, Lsn(0x70), TEST_IMG("foo blk 1"))?; + writer.advance_last_record_lsn(Lsn(0x70)); assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x70))?.unwrap(), 2); assert_eq!(tline.get_page_at_lsn(TESTREL_A, 0, Lsn(0x70))?, ZERO_PAGE); assert_eq!( @@ -425,25 +436,26 @@ mod tests { // Create timeline to work on let tline = repo.create_empty_timeline(TIMELINE_ID)?; + let writer = tline.writer(); - tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; - tline.advance_last_record_lsn(Lsn(0x20)); + writer.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; + writer.advance_last_record_lsn(Lsn(0x20)); // Check that rel exists and size is correct assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x20))?, true); assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x20))?.unwrap(), 1); // Drop relish - tline.drop_relish(TESTREL_A, Lsn(0x30))?; - tline.advance_last_record_lsn(Lsn(0x30)); + writer.drop_relish(TESTREL_A, Lsn(0x30))?; + writer.advance_last_record_lsn(Lsn(0x30)); // Check that rel is not visible anymore assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x30))?, false); assert!(tline.get_relish_size(TESTREL_A, Lsn(0x30))?.is_none()); // Extend it again - tline.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("foo blk 0 at 4"))?; - tline.advance_last_record_lsn(Lsn(0x40)); + writer.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("foo blk 0 at 4"))?; + writer.advance_last_record_lsn(Lsn(0x40)); // Check that rel exists and size is correct assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x40))?, true); @@ -461,6 +473,7 @@ mod tests { // Create timeline to work on let tline = repo.create_empty_timeline(TIMELINE_ID)?; + let writer = tline.writer(); //from storage_layer.rs const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192; @@ -470,10 +483,10 @@ mod tests { for blkno in 0..relsize { let lsn = Lsn(0x20); let data = format!("foo blk {} at {}", blkno, lsn); - tline.put_page_image(TESTREL_A, blkno, lsn, TEST_IMG(&data))?; + writer.put_page_image(TESTREL_A, blkno, lsn, TEST_IMG(&data))?; } - tline.advance_last_record_lsn(Lsn(0x20)); + writer.advance_last_record_lsn(Lsn(0x20)); // The relation was created at LSN 2, not visible at LSN 1 yet. assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x10))?, false); @@ -497,8 +510,8 @@ mod tests { // Truncate relation so that second segment was dropped // - only leave one page - tline.put_truncation(TESTREL_A, Lsn(0x60), 1)?; - tline.advance_last_record_lsn(Lsn(0x60)); + writer.put_truncation(TESTREL_A, Lsn(0x60), 1)?; + writer.advance_last_record_lsn(Lsn(0x60)); // Check reported size and contents after truncation assert_eq!(tline.get_relish_size(TESTREL_A, Lsn(0x60))?.unwrap(), 1); @@ -531,9 +544,9 @@ mod tests { for blkno in 0..relsize { let lsn = Lsn(0x80); let data = format!("foo blk {} at {}", blkno, lsn); - tline.put_page_image(TESTREL_A, blkno, lsn, TEST_IMG(&data))?; + writer.put_page_image(TESTREL_A, blkno, lsn, TEST_IMG(&data))?; } - tline.advance_last_record_lsn(Lsn(0x80)); + writer.advance_last_record_lsn(Lsn(0x80)); assert_eq!(tline.get_rel_exists(TESTREL_A, Lsn(0x80))?, true); assert_eq!( @@ -559,14 +572,15 @@ mod tests { fn test_large_rel() -> Result<()> { let repo = RepoHarness::create("test_large_rel")?.load(); let tline = repo.create_empty_timeline(TIMELINE_ID)?; + let writer = tline.writer(); let mut lsn = 0x10; for blknum in 0..pg_constants::RELSEG_SIZE + 1 { let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn))); lsn += 0x10; - tline.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img)?; + writer.put_page_image(TESTREL_A, blknum as u32, Lsn(lsn), img)?; } - tline.advance_last_record_lsn(Lsn(lsn)); + writer.advance_last_record_lsn(Lsn(lsn)); assert_current_logical_size(&tline, Lsn(lsn)); @@ -577,8 +591,8 @@ mod tests { // Truncate one block lsn += 0x10; - tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?; - tline.advance_last_record_lsn(Lsn(lsn)); + writer.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE)?; + writer.advance_last_record_lsn(Lsn(lsn)); assert_eq!( tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(), pg_constants::RELSEG_SIZE @@ -587,8 +601,8 @@ mod tests { // Truncate another block lsn += 0x10; - tline.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?; - tline.advance_last_record_lsn(Lsn(lsn)); + writer.put_truncation(TESTREL_A, Lsn(lsn), pg_constants::RELSEG_SIZE - 1)?; + writer.advance_last_record_lsn(Lsn(lsn)); assert_eq!( tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(), pg_constants::RELSEG_SIZE - 1 @@ -600,8 +614,8 @@ mod tests { let mut size: i32 = 3000; while size >= 0 { lsn += 0x10; - tline.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?; - tline.advance_last_record_lsn(Lsn(lsn)); + writer.put_truncation(TESTREL_A, Lsn(lsn), size as u32)?; + writer.advance_last_record_lsn(Lsn(lsn)); assert_eq!( tline.get_relish_size(TESTREL_A, Lsn(lsn))?.unwrap(), size as u32 @@ -621,16 +635,17 @@ mod tests { fn test_list_rels_drop() -> Result<()> { let repo = RepoHarness::create("test_list_rels_drop")?.load(); let tline = repo.create_empty_timeline(TIMELINE_ID)?; + let writer = tline.writer(); const TESTDB: u32 = 111; // Import initial dummy checkpoint record, otherwise the get_timeline() call // after branching fails below - tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_CHECKPOINT.clone())?; + writer.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_CHECKPOINT.clone())?; // Create a relation on the timeline - tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; + writer.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; - tline.advance_last_record_lsn(Lsn(0x30)); + writer.advance_last_record_lsn(Lsn(0x30)); // Check that list_rels() lists it after LSN 2, but no before it assert!(!tline.list_rels(0, TESTDB, Lsn(0x10))?.contains(&TESTREL_A)); @@ -640,14 +655,17 @@ mod tests { // Create a branch, check that the relation is visible there repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x30))?; let newtline = repo.get_timeline(NEW_TIMELINE_ID)?; + let new_writer = newtline.writer(); assert!(newtline .list_rels(0, TESTDB, Lsn(0x30))? .contains(&TESTREL_A)); // Drop it on the branch - newtline.drop_relish(TESTREL_A, Lsn(0x40))?; - newtline.advance_last_record_lsn(Lsn(0x40)); + new_writer.drop_relish(TESTREL_A, Lsn(0x40))?; + new_writer.advance_last_record_lsn(Lsn(0x40)); + + drop(new_writer); // Check that it's no longer listed on the branch after the point where it was dropped assert!(newtline @@ -675,28 +693,30 @@ mod tests { fn test_branch() -> Result<()> { let repo = RepoHarness::create("test_branch")?.load(); let tline = repo.create_empty_timeline(TIMELINE_ID)?; + let writer = tline.writer(); // Import initial dummy checkpoint record, otherwise the get_timeline() call // after branching fails below - tline.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_CHECKPOINT.clone())?; + writer.put_page_image(RelishTag::Checkpoint, 0, Lsn(0x10), ZERO_CHECKPOINT.clone())?; // Create a relation on the timeline - tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; - tline.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?; - tline.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("foo blk 0 at 4"))?; + writer.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?; + writer.put_page_image(TESTREL_A, 0, Lsn(0x30), TEST_IMG("foo blk 0 at 3"))?; + writer.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("foo blk 0 at 4"))?; // Create another relation - tline.put_page_image(TESTREL_B, 0, Lsn(0x20), TEST_IMG("foobar blk 0 at 2"))?; + writer.put_page_image(TESTREL_B, 0, Lsn(0x20), TEST_IMG("foobar blk 0 at 2"))?; - tline.advance_last_record_lsn(Lsn(0x40)); + writer.advance_last_record_lsn(Lsn(0x40)); assert_current_logical_size(&tline, Lsn(0x40)); // Branch the history, modify relation differently on the new timeline repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x30))?; let newtline = repo.get_timeline(NEW_TIMELINE_ID)?; + let new_writer = newtline.writer(); - newtline.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("bar blk 0 at 4"))?; - newtline.advance_last_record_lsn(Lsn(0x40)); + new_writer.put_page_image(TESTREL_A, 0, Lsn(0x40), TEST_IMG("bar blk 0 at 4"))?; + new_writer.advance_last_record_lsn(Lsn(0x40)); // Check page contents on both branches assert_eq!( diff --git a/pageserver/src/restore_local_repo.rs b/pageserver/src/restore_local_repo.rs index 7f3e35e4b5..bf4fb12d0a 100644 --- a/pageserver/src/restore_local_repo.rs +++ b/pageserver/src/restore_local_repo.rs @@ -34,7 +34,7 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]); /// pub fn import_timeline_from_postgres_datadir( path: &Path, - timeline: &dyn Timeline, + writer: &dyn TimelineWriter, lsn: Lsn, ) -> Result<()> { // Scan 'global' @@ -44,10 +44,10 @@ pub fn import_timeline_from_postgres_datadir( None => continue, Some("pg_control") => { - import_control_file(timeline, lsn, &direntry.path())?; + import_control_file(writer, lsn, &direntry.path())?; } Some("pg_filenode.map") => import_nonrel_file( - timeline, + writer, lsn, RelishTag::FileNodeMap { spcnode: pg_constants::GLOBALTABLESPACE_OID, @@ -59,7 +59,7 @@ pub fn import_timeline_from_postgres_datadir( // Load any relation files into the page server _ => import_relfile( &direntry.path(), - timeline, + writer, lsn, pg_constants::GLOBALTABLESPACE_OID, 0, @@ -86,7 +86,7 @@ pub fn import_timeline_from_postgres_datadir( Some("PG_VERSION") => continue, Some("pg_filenode.map") => import_nonrel_file( - timeline, + writer, lsn, RelishTag::FileNodeMap { spcnode: pg_constants::DEFAULTTABLESPACE_OID, @@ -98,7 +98,7 @@ pub fn import_timeline_from_postgres_datadir( // Load any relation files into the page server _ => import_relfile( &direntry.path(), - timeline, + writer, lsn, pg_constants::DEFAULTTABLESPACE_OID, dboid, @@ -108,24 +108,24 @@ pub fn import_timeline_from_postgres_datadir( } for entry in fs::read_dir(path.join("pg_xact"))? { let entry = entry?; - import_slru_file(timeline, lsn, SlruKind::Clog, &entry.path())?; + import_slru_file(writer, lsn, SlruKind::Clog, &entry.path())?; } for entry in fs::read_dir(path.join("pg_multixact").join("members"))? { let entry = entry?; - import_slru_file(timeline, lsn, SlruKind::MultiXactMembers, &entry.path())?; + import_slru_file(writer, lsn, SlruKind::MultiXactMembers, &entry.path())?; } for entry in fs::read_dir(path.join("pg_multixact").join("offsets"))? { let entry = entry?; - import_slru_file(timeline, lsn, SlruKind::MultiXactOffsets, &entry.path())?; + import_slru_file(writer, lsn, SlruKind::MultiXactOffsets, &entry.path())?; } for entry in fs::read_dir(path.join("pg_twophase"))? { let entry = entry?; let xid = u32::from_str_radix(entry.path().to_str().unwrap(), 16)?; - import_nonrel_file(timeline, lsn, RelishTag::TwoPhase { xid }, &entry.path())?; + import_nonrel_file(writer, lsn, RelishTag::TwoPhase { xid }, &entry.path())?; } // TODO: Scan pg_tblspc - timeline.advance_last_record_lsn(lsn); + writer.advance_last_record_lsn(lsn); Ok(()) } @@ -133,7 +133,7 @@ pub fn import_timeline_from_postgres_datadir( // subroutine of import_timeline_from_postgres_datadir(), to load one relation file. fn import_relfile( path: &Path, - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, lsn: Lsn, spcoid: Oid, dboid: Oid, @@ -191,7 +191,7 @@ fn import_relfile( /// are just slurped into the repository as one blob. /// fn import_nonrel_file( - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, lsn: Lsn, tag: RelishTag, path: &Path, @@ -212,7 +212,7 @@ fn import_nonrel_file( /// /// The control file is imported as is, but we also extract the checkpoint record /// from it and store it separated. -fn import_control_file(timeline: &dyn Timeline, lsn: Lsn, path: &Path) -> Result<()> { +fn import_control_file(timeline: &dyn TimelineWriter, lsn: Lsn, path: &Path) -> Result<()> { let mut file = File::open(path)?; let mut buffer = Vec::new(); // read the whole file @@ -239,7 +239,12 @@ fn import_control_file(timeline: &dyn Timeline, lsn: Lsn, path: &Path) -> Result /// /// Import an SLRU segment file /// -fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Path) -> Result<()> { +fn import_slru_file( + timeline: &dyn TimelineWriter, + lsn: Lsn, + slru: SlruKind, + path: &Path, +) -> Result<()> { // Does it look like an SLRU file? let mut file = File::open(path)?; let mut buf: [u8; 8192] = [0u8; 8192]; @@ -287,7 +292,7 @@ fn import_slru_file(timeline: &dyn Timeline, lsn: Lsn, slru: SlruKind, path: &Pa pub fn save_decoded_record( checkpoint: &mut CheckPoint, checkpoint_modified: &mut bool, - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, decoded: &DecodedWALRecord, recdata: Bytes, lsn: Lsn, @@ -493,7 +498,11 @@ pub fn save_decoded_record( } /// Subroutine of save_decoded_record(), to handle an XLOG_DBASE_CREATE record. -fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatabase) -> Result<()> { +fn save_xlog_dbase_create( + timeline: &dyn TimelineWriter, + lsn: Lsn, + rec: &XlCreateDatabase, +) -> Result<()> { let db_id = rec.db_id; let tablespace_id = rec.tablespace_id; let src_db_id = rec.src_db_id; @@ -570,7 +579,11 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab /// Subroutine of save_decoded_record(), to handle an XLOG_SMGR_TRUNCATE record. /// /// This is the same logic as in PostgreSQL's smgr_redo() function. -fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTruncate) -> Result<()> { +fn save_xlog_smgr_truncate( + timeline: &dyn TimelineWriter, + lsn: Lsn, + rec: &XlSmgrTruncate, +) -> Result<()> { let spcnode = rec.rnode.spcnode; let dbnode = rec.rnode.dbnode; let relnode = rec.rnode.relnode; @@ -632,7 +645,7 @@ fn save_xlog_smgr_truncate(timeline: &dyn Timeline, lsn: Lsn, rec: &XlSmgrTrunca /// Subroutine of save_decoded_record(), to handle an XLOG_XACT_* records. /// fn save_xact_record( - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, lsn: Lsn, parsed: &XlXactParsedRecord, decoded: &DecodedWALRecord, @@ -690,7 +703,7 @@ fn save_xact_record( fn save_clog_truncate_record( checkpoint: &mut CheckPoint, checkpoint_modified: &mut bool, - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, lsn: Lsn, xlrec: &XlClogTruncate, ) -> Result<()> { @@ -752,7 +765,7 @@ fn save_clog_truncate_record( fn save_multixact_create_record( checkpoint: &mut CheckPoint, checkpoint_modified: &mut bool, - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, lsn: Lsn, xlrec: &XlMultiXactCreate, decoded: &DecodedWALRecord, @@ -831,7 +844,7 @@ fn save_multixact_create_record( fn save_multixact_truncate_record( checkpoint: &mut CheckPoint, checkpoint_modified: &mut bool, - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, lsn: Lsn, xlrec: &XlMultiXactTruncate, ) -> Result<()> { @@ -871,7 +884,7 @@ fn save_multixact_truncate_record( } fn save_relmap_page( - timeline: &dyn Timeline, + timeline: &dyn TimelineWriter, lsn: Lsn, xlrec: &XlRelmapUpdate, decoded: &DecodedWALRecord, diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index afb8c8c4b0..de5ee5340c 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -219,13 +219,15 @@ fn walreceiver_main( // at risk of hittind a deadlock. assert!(lsn.is_aligned()); + let writer = timeline.writer(); + let mut checkpoint_modified = false; let decoded = decode_wal_record(recdata.clone()); restore_local_repo::save_decoded_record( &mut checkpoint, &mut checkpoint_modified, - &*timeline, + writer.as_ref(), &decoded, recdata, lsn, @@ -235,7 +237,7 @@ fn walreceiver_main( if checkpoint_modified { let new_checkpoint_bytes = checkpoint.encode(); - timeline.put_page_image( + writer.put_page_image( RelishTag::Checkpoint, 0, lsn, @@ -245,7 +247,7 @@ fn walreceiver_main( // Now that this record has been fully handled, including updating the // checkpoint data, let the repository know that it is up-to-date to this LSN - timeline.advance_last_record_lsn(lsn); + writer.advance_last_record_lsn(lsn); last_rec_lsn = lsn; } From 160c4aff6111ef325d3a3d48528a0e6e9f123b35 Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Fri, 8 Oct 2021 17:27:08 -0700 Subject: [PATCH 18/46] pageserver - use write guard for checkpointing --- pageserver/src/layered_repository.rs | 146 ++++------ .../src/layered_repository/inmemory_layer.rs | 272 ++++++------------ .../src/layered_repository/page_versions.rs | 34 +-- 3 files changed, 142 insertions(+), 310 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index c6cd9094f6..4de6419444 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -33,7 +33,6 @@ use std::sync::{Arc, Mutex, MutexGuard}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; -use crate::layered_repository::inmemory_layer::FreezeLayers; use crate::relish::*; use crate::relish_storage::schedule_timeline_upload; use crate::repository::{GcResult, Repository, Timeline, TimelineWriter, WALRecord}; @@ -72,8 +71,6 @@ use storage_layer::{ Layer, PageReconstructData, PageReconstructResult, SegmentTag, RELISH_SEG_SIZE, }; -use self::inmemory_layer::{NonWriteableError, WriteResult}; - static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; 8192]); // Timeout when waiting for WAL receiver to catch up to an LSN given in a GetPage@LSN call. @@ -670,6 +667,13 @@ pub struct LayeredTimeline { /// If `true`, will backup its timeline files to remote storage after freezing. upload_relishes: bool, + + /// Ensures layers aren't frozen by checkpointer between + /// [`LayeredTimeline::get_layer_for_write`] and layer reads. + /// Locked automatically by [`LayeredTimelineWriter`] and checkpointer. + /// Must always be acquired before the layer map/individual layer lock + /// to avoid deadlock. + write_lock: Mutex<()>, } /// Public interface functions @@ -903,7 +907,10 @@ impl Timeline for LayeredTimeline { } fn writer<'a>(&'a self) -> Box { - Box::new(LayeredTimelineWriter(self)) + Box::new(LayeredTimelineWriter { + tl: self, + _write_guard: self.write_lock.lock().unwrap(), + }) } } @@ -945,6 +952,8 @@ impl LayeredTimeline { current_logical_size: AtomicUsize::new(current_logical_size), current_logical_size_gauge, upload_relishes, + + write_lock: Mutex::new(()), }; Ok(timeline) } @@ -1219,18 +1228,13 @@ impl LayeredTimeline { /// /// NOTE: This has nothing to do with checkpoint in PostgreSQL. fn checkpoint_internal(&self, checkpoint_distance: u64, forced: bool) -> Result<()> { - // Grab lock on the layer map. - // - // TODO: We hold it locked throughout the checkpoint operation. That's bad, - // the checkpointing could take many seconds, and any incoming get_page_at_lsn() - // requests will block. + let mut write_guard = self.write_lock.lock().unwrap(); let mut layers = self.layers.lock().unwrap(); // Bump the generation number in the layer map, so that we can distinguish // entries inserted after the checkpoint started let current_generation = layers.increment_generation(); - // Read 'last_record_lsn'. That becomes the cutoff LSN for frozen layers. let RecordLsn { last: last_record_lsn, prev: prev_record_lsn, @@ -1280,32 +1284,24 @@ impl LayeredTimeline { break; } - // Freeze the layer. - // - // This is a two-step process. First, we "freeze" the in-memory - // layer, to close it for new writes, and replace the original - // layer with the new frozen in-memory layer (and possibly a new - // open layer to hold changes newer than the cutoff.) Then we write - // the frozen layer to disk, and replace the in-memory frozen layer - // with the new on-disk layers. - let FreezeLayers { - frozen, - open: maybe_new_open, - } = oldest_layer.freeze(last_record_lsn)?; + // Mark the layer as no longer accepting writes and record the end_lsn. + // This happens in-place, no new layers are created now. + // We call `get_last_record_lsn` again, which may be different from the + // original load, as we may have released the write lock since then. + oldest_layer.freeze(self.get_last_record_lsn()); - // replace this layer with the new layers that 'freeze' returned + // The layer is no longer open, update the layer map to reflect this. + // We will replace it with on-disk historics below. layers.pop_oldest_open(); - if let Some(new_open) = maybe_new_open.clone() { - layers.insert_open(new_open); - } - - // We temporarily insert InMemory layer into historic list here. - // TODO: check that all possible concurrent users of 'historic' treat it right - layers.insert_historic(frozen.clone()); + layers.insert_historic(oldest_layer.clone()); // Write the now-frozen layer to disk. That could take a while, so release the lock while do it drop(layers); - let new_historics = frozen.write_to_disk(self)?; + drop(write_guard); + + let new_historics = oldest_layer.write_to_disk(self)?; + + write_guard = self.write_lock.lock().unwrap(); layers = self.layers.lock().unwrap(); if !new_historics.is_empty() { @@ -1313,7 +1309,7 @@ impl LayeredTimeline { } // Finally, replace the frozen in-memory layer with the new on-disk layers - layers.remove_historic(frozen.clone()); + layers.remove_historic(oldest_layer); // Add the historics to the LayerMap for delta_layer in new_historics.delta_layers { @@ -1334,6 +1330,7 @@ impl LayeredTimeline { } drop(layers); + drop(write_guard); if created_historics { // We must fsync the timeline dir to ensure the directory entries for @@ -1760,46 +1757,18 @@ impl LayeredTimeline { self.current_logical_size_gauge .set(val as i64 - diff as i64); } - - /// If a layer is in the process of being replaced in [`LayerMap`], write - /// operations will fail with [`NonWriteableError`]. This may happen due to - /// a race: the checkpointer thread freezes a layer just after - /// [`Self::get_layer_for_write`] returned it. To handle this error, we try - /// again getting the layer and attempt the write. - fn perform_write_op( - &self, - seg: SegmentTag, - lsn: Lsn, - write_op: impl Fn(&Arc) -> WriteResult, - ) -> anyhow::Result { - let mut layer = self.get_layer_for_write(seg, lsn)?; - loop { - match write_op(&layer) { - Ok(r) => return Ok(r), - Err(NonWriteableError {}) => {} - } - - info!( - "attempted to write to non-writeable layer, retrying {} {}", - seg, lsn - ); - - // layer was non-writeable, try again - let new_layer = self.get_layer_for_write(seg, lsn)?; - // the new layer does not have to be writeable, but it should at least be different - assert!(!Arc::ptr_eq(&layer, &new_layer)); - layer = new_layer; - } - } } -struct LayeredTimelineWriter<'a>(&'a LayeredTimeline); +struct LayeredTimelineWriter<'a> { + tl: &'a LayeredTimeline, + _write_guard: MutexGuard<'a, ()>, +} impl Deref for LayeredTimelineWriter<'_> { type Target = dyn Timeline; fn deref(&self) -> &Self::Target { - self.0 + self.tl } } @@ -1815,10 +1784,9 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { ensure!(rec.lsn.is_aligned(), "unaligned record LSN"); let seg = SegmentTag::from_blknum(rel, blknum); - let delta_size = self.0.perform_write_op(seg, rec.lsn, |layer| { - layer.put_wal_record(blknum, rec.clone()) - })?; - self.0 + let layer = self.tl.get_layer_for_write(seg, rec.lsn)?; + let delta_size = layer.put_wal_record(blknum, rec); + self.tl .increase_current_logical_size(delta_size * BLCKSZ as u32); Ok(()) } @@ -1835,11 +1803,10 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { let seg = SegmentTag::from_blknum(rel, blknum); - let delta_size = self.0.perform_write_op(seg, lsn, |layer| { - layer.put_page_image(blknum, lsn, img.clone()) - })?; + let layer = self.tl.get_layer_for_write(seg, lsn)?; + let delta_size = layer.put_page_image(blknum, lsn, img); - self.0 + self.tl .increase_current_logical_size(delta_size * BLCKSZ as u32); Ok(()) } @@ -1853,8 +1820,8 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { debug!("put_truncation: {} to {} blocks at {}", rel, relsize, lsn); let oldsize = self - .0 - .get_relish_size(rel, self.0.get_last_record_lsn())? + .tl + .get_relish_size(rel, self.tl.get_last_record_lsn())? .ok_or_else(|| { anyhow!( "attempted to truncate non-existent relish {} at {}", @@ -1880,8 +1847,9 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { rel, segno: remove_segno, }; - self.0 - .perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; + + let layer = self.tl.get_layer_for_write(seg, lsn)?; + layer.drop_segment(lsn); } // Truncate the last remaining segment to the specified size @@ -1890,11 +1858,10 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { rel, segno: last_remain_seg, }; - self.0.perform_write_op(seg, lsn, |layer| { - layer.put_truncation(lsn, relsize % RELISH_SEG_SIZE) - })?; + let layer = self.tl.get_layer_for_write(seg, lsn)?; + layer.put_truncation(lsn, relsize % RELISH_SEG_SIZE) } - self.0 + self.tl .decrease_current_logical_size((oldsize - relsize) * BLCKSZ as u32); Ok(()) } @@ -1903,7 +1870,10 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { trace!("drop_segment: {} at {}", rel, lsn); if rel.is_blocky() { - if let Some(oldsize) = self.0.get_relish_size(rel, self.0.get_last_record_lsn())? { + if let Some(oldsize) = self + .tl + .get_relish_size(rel, self.tl.get_last_record_lsn())? + { let old_last_seg = if oldsize == 0 { 0 } else { @@ -1916,10 +1886,10 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { rel, segno: remove_segno, }; - self.0 - .perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; + let layer = self.tl.get_layer_for_write(seg, lsn)?; + layer.drop_segment(lsn); } - self.0 + self.tl .decrease_current_logical_size(oldsize * BLCKSZ as u32); } else { warn!( @@ -1930,8 +1900,8 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { } else { // TODO handle TwoPhase relishes let seg = SegmentTag::from_blknum(rel, 0); - self.0 - .perform_write_op(seg, lsn, |layer| layer.drop_segment(lsn))?; + let layer = self.tl.get_layer_for_write(seg, lsn)?; + layer.drop_segment(lsn); } Ok(()) @@ -1943,7 +1913,7 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { fn advance_last_record_lsn(&self, new_lsn: Lsn) { assert!(new_lsn.is_aligned()); - self.0.last_record_lsn.advance(new_lsn); + self.tl.last_record_lsn.advance(new_lsn); } } diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index 8c44ac6296..bd1860fd47 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -15,12 +15,10 @@ use crate::{ZTenantId, ZTimelineId}; use anyhow::{bail, ensure, Result}; use bytes::Bytes; use log::*; -use std::cmp::min; use std::path::PathBuf; use std::sync::{Arc, RwLock}; use zenith_utils::vec_map::VecMap; -use zenith_utils::accum::Accum; use zenith_utils::lsn::Lsn; use super::page_versions::PageVersions; @@ -37,9 +35,6 @@ pub struct InMemoryLayer { /// start_lsn: Lsn, - /// Frozen in-memory layers have an inclusive end LSN. - end_lsn: Option, - /// LSN of the oldest page version stored in this layer oldest_pending_lsn: Lsn, @@ -52,8 +47,13 @@ pub struct InMemoryLayer { } pub struct InMemoryLayerInner { + /// Frozen in-memory layers have an exclusive end LSN. + /// Writes are only allowed when this is None + end_lsn: Option, + /// If this relation was dropped, remember when that happened. - drop_lsn: Option, + /// The drop LSN is recorded in [`end_lsn`]. + dropped: bool, /// /// All versions of all pages in the layer are are kept here. @@ -69,19 +69,11 @@ pub struct InMemoryLayerInner { /// a non-blocky rel, 'segsizes' is not used and is always empty. /// segsizes: VecMap, - - /// Writes are only allowed when true. - /// Set to false when this layer is in the process of being replaced. - writeable: bool, } impl InMemoryLayerInner { - fn check_writeable(&self) -> WriteResult<()> { - if self.writeable { - Ok(()) - } else { - Err(NonWriteableError) - } + fn assert_writeable(&self) { + assert!(self.end_lsn.is_none()); } fn get_seg_size(&self, lsn: Lsn) -> u32 { @@ -104,20 +96,17 @@ impl Layer for InMemoryLayer { let inner = self.inner.read().unwrap(); let end_lsn; - let dropped; - if let Some(drop_lsn) = inner.drop_lsn { + if let Some(drop_lsn) = inner.end_lsn { end_lsn = drop_lsn; - dropped = true; } else { end_lsn = Lsn(u64::MAX); - dropped = false; } let delta_filename = DeltaFileName { seg: self.seg, start_lsn: self.start_lsn, end_lsn, - dropped, + dropped: inner.dropped, } .to_string(); @@ -137,14 +126,10 @@ impl Layer for InMemoryLayer { } fn get_end_lsn(&self) -> Lsn { - if let Some(end_lsn) = self.end_lsn { - return Lsn(end_lsn.0 + 1); - } - let inner = self.inner.read().unwrap(); - if let Some(drop_lsn) = inner.drop_lsn { - drop_lsn + if let Some(end_lsn) = inner.end_lsn { + end_lsn } else { Lsn(u64::MAX) } @@ -152,7 +137,7 @@ impl Layer for InMemoryLayer { fn is_dropped(&self) -> bool { let inner = self.inner.read().unwrap(); - inner.drop_lsn.is_some() + inner.dropped } /// Look up given page in the cache. @@ -230,8 +215,8 @@ impl Layer for InMemoryLayer { assert!(lsn >= self.start_lsn); // Is the requested LSN after the segment was dropped? - if let Some(drop_lsn) = inner.drop_lsn { - if lsn >= drop_lsn { + if let Some(end_lsn) = inner.end_lsn { + if lsn >= end_lsn { return Ok(false); } } @@ -262,14 +247,14 @@ impl Layer for InMemoryLayer { let inner = self.inner.read().unwrap(); let end_str = inner - .drop_lsn + .end_lsn .as_ref() - .map(|drop_lsn| drop_lsn.to_string()) + .map(Lsn::to_string) .unwrap_or_default(); println!( - "----- in-memory layer for tli {} seg {} {}-{} ----", - self.timelineid, self.seg, self.start_lsn, end_str + "----- in-memory layer for tli {} seg {} {}-{} {} ----", + self.timelineid, self.seg, self.start_lsn, end_str, inner.dropped, ); for (k, v) in inner.segsizes.as_slice() { @@ -290,21 +275,6 @@ impl Layer for InMemoryLayer { } } -/// Write failed because the layer is in process of being replaced. -/// See [`LayeredTimeline::perform_write_op`] for how to handle this error. -#[derive(Debug)] -pub struct NonWriteableError; - -pub type WriteResult = std::result::Result; - -/// Helper struct to cleanup `InMemoryLayer::freeze` return signature. -pub struct FreezeLayers { - /// Replacement layer for the layer which freeze was called on. - pub frozen: Arc, - /// New open layer containing leftover data. - pub open: Option>, -} - /// A result of an inmemory layer data being written to disk. pub struct LayersOnDisk { pub delta_layers: Vec, @@ -318,10 +288,6 @@ impl LayersOnDisk { } impl InMemoryLayer { - fn assert_not_frozen(&self) { - assert!(self.end_lsn.is_none()); - } - /// Return the oldest page version that's stored in this layer pub fn get_oldest_pending_lsn(&self) -> Lsn { self.oldest_pending_lsn @@ -357,14 +323,13 @@ impl InMemoryLayer { tenantid, seg, start_lsn, - end_lsn: None, oldest_pending_lsn, incremental: false, inner: RwLock::new(InMemoryLayerInner { - drop_lsn: None, + end_lsn: None, + dropped: false, page_versions: PageVersions::default(), segsizes, - writeable: true, }), }) } @@ -372,7 +337,7 @@ impl InMemoryLayer { // Write operations /// Remember new page version, as a WAL record over previous version - pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> WriteResult { + pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> u32 { self.put_page_version( blknum, rec.lsn, @@ -384,7 +349,7 @@ impl InMemoryLayer { } /// Remember new page version, as a full page image - pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> WriteResult { + pub fn put_page_image(&self, blknum: u32, lsn: Lsn, img: Bytes) -> u32 { self.put_page_version( blknum, lsn, @@ -397,8 +362,7 @@ impl InMemoryLayer { /// Common subroutine of the public put_wal_record() and put_page_image() functions. /// Adds the page version to the in-memory tree - pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> WriteResult { - self.assert_not_frozen(); + pub fn put_page_version(&self, blknum: u32, lsn: Lsn, pv: PageVersion) -> u32 { assert!(self.seg.blknum_in_seg(blknum)); trace!( @@ -410,7 +374,7 @@ impl InMemoryLayer { ); let mut inner = self.inner.write().unwrap(); - inner.check_writeable()?; + inner.assert_writeable(); let old = inner.page_versions.append_or_update_last(blknum, lsn, pv); @@ -471,22 +435,22 @@ impl InMemoryLayer { } inner.segsizes.append_or_update_last(lsn, newsize).unwrap(); - return Ok(newsize - oldsize); + return newsize - oldsize; } } - Ok(0) + + 0 } /// Remember that the relation was truncated at given LSN - pub fn put_truncation(&self, lsn: Lsn, segsize: u32) -> WriteResult<()> { + pub fn put_truncation(&self, lsn: Lsn, segsize: u32) { assert!( self.seg.rel.is_blocky(), "put_truncation() called on a non-blocky rel" ); - self.assert_not_frozen(); let mut inner = self.inner.write().unwrap(); - inner.check_writeable()?; + inner.assert_writeable(); // check that this we truncate to a smaller size than segment was before the truncation let oldsize = inner.get_seg_size(lsn); @@ -498,25 +462,19 @@ impl InMemoryLayer { // We already had an entry for this LSN. That's odd.. warn!("Inserting truncation, but had an entry for the LSN already"); } - - Ok(()) } /// Remember that the segment was dropped at given LSN - pub fn drop_segment(&self, lsn: Lsn) -> WriteResult<()> { - self.assert_not_frozen(); - + pub fn drop_segment(&self, lsn: Lsn) { let mut inner = self.inner.write().unwrap(); - inner.check_writeable()?; - - assert!(inner.drop_lsn.is_none()); - inner.drop_lsn = Some(lsn); - inner.writeable = false; + assert!(inner.end_lsn.is_none()); + assert!(!inner.dropped); + inner.dropped = true; + assert!(self.start_lsn < lsn); + inner.end_lsn = Some(lsn); trace!("dropped segment {} at {}", self.seg, lsn); - - Ok(()) } /// @@ -556,116 +514,43 @@ impl InMemoryLayer { tenantid, seg, start_lsn, - end_lsn: None, oldest_pending_lsn, incremental: true, inner: RwLock::new(InMemoryLayerInner { - drop_lsn: None, + end_lsn: None, + dropped: false, page_versions: PageVersions::default(), segsizes, - writeable: true, }), }) } pub fn is_writeable(&self) -> bool { let inner = self.inner.read().unwrap(); - inner.writeable + inner.end_lsn.is_none() } - /// Splits `self` into two InMemoryLayers: `frozen` and `open`. - /// All data up to and including `cutoff_lsn` - /// is copied to `frozen`, while the remaining data is copied to `open`. - /// After completion, self is non-writeable, but not frozen. - pub fn freeze(self: Arc, cutoff_lsn: Lsn) -> Result { - info!( - "freezing in-memory layer {} at {} (oldest {})", - self.filename().display(), - cutoff_lsn, - self.oldest_pending_lsn - ); + /// Make the layer non-writeable. Only call once. + /// Records the end_lsn for non-dropped layers. + /// `end_lsn` is inclusive + pub fn freeze(&self, end_lsn: Lsn) { + let mut inner = self.inner.write().unwrap(); - self.assert_not_frozen(); - - let self_ref = self.clone(); - let mut inner = self_ref.inner.write().unwrap(); - // Dropped layers don't need any special freeze actions, - // they are marked as non-writeable at drop and just - // written out to disk by checkpointer. - if inner.drop_lsn.is_some() { - assert!(!inner.writeable); - info!( - "freezing in memory layer for {} on timeline {} is dropped at {}", - self.seg, - self.timelineid, - inner.drop_lsn.unwrap() - ); - - // There should be no newer layer that refers this non-writeable layer, - // because layer that is created after dropped one represents a new rel. - return Ok(FreezeLayers { - frozen: self, - open: None, - }); - } - assert!(inner.writeable); - inner.writeable = false; - - // Divide all the page versions into old and new - // at the 'cutoff_lsn' point. - let mut after_oldest_lsn: Accum = Accum(None); - - let cutoff_lsn_exclusive = Lsn(cutoff_lsn.0 + 1); - - let (before_segsizes, mut after_segsizes) = inner.segsizes.split_at(&cutoff_lsn_exclusive); - if let Some((lsn, _size)) = after_segsizes.as_slice().first() { - after_oldest_lsn.accum(min, *lsn); - } - - let (before_page_versions, after_page_versions) = inner - .page_versions - .split_at(cutoff_lsn_exclusive, &mut after_oldest_lsn); - - let frozen = Arc::new(InMemoryLayer { - conf: self.conf, - tenantid: self.tenantid, - timelineid: self.timelineid, - seg: self.seg, - start_lsn: self.start_lsn, - end_lsn: Some(cutoff_lsn), - oldest_pending_lsn: self.start_lsn, - incremental: self.incremental, - inner: RwLock::new(InMemoryLayerInner { - drop_lsn: inner.drop_lsn, - page_versions: before_page_versions, - segsizes: before_segsizes, - writeable: false, - }), - }); - - let open = if !after_segsizes.is_empty() || !after_page_versions.is_empty() { - let mut new_open = Self::create_successor_layer( - self.conf, - frozen.clone(), - self.timelineid, - self.tenantid, - cutoff_lsn + 1, - after_oldest_lsn.0.unwrap(), - )?; - - let new_inner = new_open.inner.get_mut().unwrap(); - // Ensure page_versions doesn't contain anything - // so we can just replace it - assert!(new_inner.page_versions.is_empty()); - new_inner.page_versions = after_page_versions; - new_inner.segsizes.extend(&mut after_segsizes).unwrap(); - - Some(Arc::new(new_open)) + if inner.end_lsn.is_some() { + assert!(inner.dropped); } else { - None - }; + assert!(!inner.dropped); + assert!(self.start_lsn < end_lsn + 1); + inner.end_lsn = Some(Lsn(end_lsn.0 + 1)); - Ok(FreezeLayers { frozen, open }) + if let Some((lsn, _)) = inner.segsizes.as_slice().last() { + assert!(lsn <= &end_lsn, "{:?} {:?}", lsn, end_lsn); + } + + for (_blk, lsn, _pv) in inner.page_versions.ordered_page_version_iter(None) { + assert!(lsn <= end_lsn); + } + } } /// Write the this frozen in-memory layer to disk. @@ -678,9 +563,8 @@ impl InMemoryLayer { /// end LSN are the same.) pub fn write_to_disk(&self, timeline: &LayeredTimeline) -> Result { trace!( - "write_to_disk {} end_lsn is {} get_end_lsn is {}", + "write_to_disk {} get_end_lsn is {}", self.filename().display(), - self.end_lsn.unwrap_or(Lsn(0)), self.get_end_lsn() ); @@ -694,16 +578,16 @@ impl InMemoryLayer { // would have to wait until we release it. That race condition is very // rare though, so we just accept the potential latency hit for now. let inner = self.inner.read().unwrap(); - assert!(!inner.writeable); + let end_lsn_exclusive = inner.end_lsn.unwrap(); - if let Some(drop_lsn) = inner.drop_lsn { + if inner.dropped { let delta_layer = DeltaLayer::create( self.conf, self.timelineid, self.tenantid, self.seg, self.start_lsn, - drop_lsn, + end_lsn_exclusive, true, inner.page_versions.ordered_page_version_iter(None), inner.segsizes.clone(), @@ -712,7 +596,7 @@ impl InMemoryLayer { "freeze: created delta layer for dropped segment {} {}-{}", self.seg, self.start_lsn, - drop_lsn + end_lsn_exclusive ); return Ok(LayersOnDisk { delta_layers: vec![delta_layer], @@ -720,14 +604,19 @@ impl InMemoryLayer { }); } - let end_lsn = self.end_lsn.unwrap(); + // Since `end_lsn` is inclusive, subtract 1. + // We want to make an ImageLayer for the last included LSN, + // so the DeltaLayer should exlcude that LSN. + let end_lsn_inclusive = Lsn(end_lsn_exclusive.0 - 1); - let mut before_page_versions = inner.page_versions.ordered_page_version_iter(Some(end_lsn)); + let mut page_versions = inner + .page_versions + .ordered_page_version_iter(Some(end_lsn_inclusive)); let mut delta_layers = Vec::new(); - if self.start_lsn != end_lsn { - let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1)); + if self.start_lsn != end_lsn_inclusive { + let (segsizes, _) = inner.segsizes.split_at(&end_lsn_exclusive); // Write the page versions before the cutoff to disk. let delta_layer = DeltaLayer::create( self.conf, @@ -735,27 +624,32 @@ impl InMemoryLayer { self.tenantid, self.seg, self.start_lsn, - end_lsn, + end_lsn_inclusive, false, - before_page_versions, - before_segsizes, + page_versions, + segsizes, )?; delta_layers.push(delta_layer); trace!( "freeze: created delta layer {} {}-{}", self.seg, self.start_lsn, - end_lsn + end_lsn_inclusive ); } else { - assert!(before_page_versions.next().is_none()); + assert!(page_versions.next().is_none()); } drop(inner); // Write a new base image layer at the cutoff point - let image_layer = ImageLayer::create_from_src(self.conf, timeline, self, end_lsn)?; - trace!("freeze: created image layer {} at {}", self.seg, end_lsn); + let image_layer = + ImageLayer::create_from_src(self.conf, timeline, self, end_lsn_inclusive)?; + trace!( + "freeze: created image layer {} at {}", + self.seg, + end_lsn_inclusive + ); Ok(LayersOnDisk { delta_layers, diff --git a/pageserver/src/layered_repository/page_versions.rs b/pageserver/src/layered_repository/page_versions.rs index fe7df0caa9..90321f96cd 100644 --- a/pageserver/src/layered_repository/page_versions.rs +++ b/pageserver/src/layered_repository/page_versions.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, ops::RangeBounds, slice}; -use zenith_utils::{accum::Accum, lsn::Lsn, vec_map::VecMap}; +use zenith_utils::{lsn::Lsn, vec_map::VecMap}; use super::storage_layer::PageVersion; @@ -10,10 +10,6 @@ const EMPTY_SLICE: &[(Lsn, PageVersion)] = &[]; pub struct PageVersions(HashMap>); impl PageVersions { - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - pub fn append_or_update_last( &mut self, blknum: u32, @@ -44,34 +40,6 @@ impl PageVersions { .unwrap_or(EMPTY_SLICE) } - /// Split the page version map into two. - /// - /// Left contains everything up to and not including [`cutoff_lsn`]. - /// Right contains [`cutoff_lsn`] and everything after. - pub fn split_at(&self, cutoff_lsn: Lsn, after_oldest_lsn: &mut Accum) -> (Self, Self) { - let mut before_blocks = HashMap::new(); - let mut after_blocks = HashMap::new(); - - for (blknum, vec_map) in self.0.iter() { - let (before_versions, after_versions) = vec_map.split_at(&cutoff_lsn); - - if !before_versions.is_empty() { - let old = before_blocks.insert(*blknum, before_versions); - assert!(old.is_none()); - } - - if !after_versions.is_empty() { - let (first_lsn, _first_pv) = &after_versions.as_slice()[0]; - after_oldest_lsn.accum(std::cmp::min, *first_lsn); - - let old = after_blocks.insert(*blknum, after_versions); - assert!(old.is_none()); - } - } - - (Self(before_blocks), Self(after_blocks)) - } - /// Iterate through [`PageVersion`]s in (block, lsn) order. /// If a [`cutoff_lsn`] is set, only show versions with `lsn < cutoff_lsn` pub fn ordered_page_version_iter(&self, cutoff_lsn: Option) -> OrderedPageVersionIter<'_> { From 7c5b99683ca42c21dfcd5d6e476db0baae2437ba Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 12 Oct 2021 15:23:38 +0300 Subject: [PATCH 19/46] Speed up builds by passing make jobserver to cargo This change brings the following improvements to our build system: * Now BUILD_TYPE also affects rust apps. * From now on, cargo will respect `-jN` passed via `make`. However, note that `rustc` may spawn multiple threads depending on compile flags. * Cargo is able to cooperate with make to better schedule parallel jobs, which leads to better build times (-20s in release mode on my machine). --- Makefile | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8716372b25..ef26ceee2d 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,8 @@ BUILD_TYPE ?= debug ifeq ($(BUILD_TYPE),release) PG_CONFIGURE_OPTS = --enable-debug PG_CFLAGS = -O2 -g3 $(CFLAGS) + # Unfortunately, `--profile=...` is a nightly feature + CARGO_BUILD_FLAGS += --release else ifeq ($(BUILD_TYPE),debug) PG_CONFIGURE_OPTS = --enable-debug --enable-cassert --enable-depend PG_CFLAGS = -O0 -g3 $(CFLAGS) @@ -21,25 +23,30 @@ else $(error Bad build type `$(BUILD_TYPE)', see Makefile for options) endif +# Choose whether we should be silent or verbose +CARGO_BUILD_FLAGS += --$(if $(filter s,$(MAKEFLAGS)),quiet,verbose) +# Fix for a corner case when make doesn't pass a jobserver +CARGO_BUILD_FLAGS += $(filter -j1,$(MAKEFLAGS)) + +# This option has a side effect of passing make jobserver to cargo. +# However, we shouldn't do this if `make -n` (--dry-run) has been asked. +CARGO_CMD_PREFIX += $(if $(filter n,$(MAKEFLAGS)),,+) +# Force cargo not to print progress bar +CARGO_CMD_PREFIX += CARGO_TERM_PROGRESS_WHEN=never CI=1 + # # Top level Makefile to build Zenith and PostgreSQL # .PHONY: all all: zenith postgres -# We don't want to run 'cargo build' in parallel with the postgres build, -# because interleaving cargo build output with postgres build output looks -# confusing. Also, 'cargo build' is parallel on its own, so it would be too -# much parallelism. (Recursive invocation of postgres target still gets any -# '-j' flag from the command line, so 'make -j' is still useful.) -.NOTPARALLEL: - ### Zenith Rust bits # # The 'postgres_ffi' depends on the Postgres headers. .PHONY: zenith zenith: postgres-headers - cargo build + +@echo "Compiling Zenith" + $(CARGO_CMD_PREFIX) cargo build $(CARGO_BUILD_FLAGS) ### PostgreSQL parts tmp_install/build/config.status: @@ -61,10 +68,10 @@ postgres-headers: postgres-configure +@echo "Installing PostgreSQL headers" $(MAKE) -C tmp_install/build/src/include MAKELEVEL=0 install - # Compile and install PostgreSQL and contrib/zenith .PHONY: postgres -postgres: postgres-configure +postgres: postgres-configure \ + postgres-headers # to prevent `make install` conflicts with zenith's `postgres-headers` +@echo "Compiling PostgreSQL" $(MAKE) -C tmp_install/build MAKELEVEL=0 install +@echo "Compiling contrib/zenith" @@ -80,13 +87,13 @@ postgres-clean: .PHONY: clean clean: cd tmp_install/build && $(MAKE) clean - cargo clean + $(CARGO_CMD_PREFIX) cargo clean # This removes everything .PHONY: distclean distclean: rm -rf tmp_install - cargo clean + $(CARGO_CMD_PREFIX) cargo clean .PHONY: fmt fmt: From 23f4c0a74229d111e7f77b7e6498ac22d55d680c Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Tue, 12 Oct 2021 22:03:06 +0300 Subject: [PATCH 20/46] Rename `wal_acceptor` binary to `safekeeper` (#740), stage 1/2 * Rename wal_acceptor binary to safekeeper * Rename wal_acceptor.pid and wal_acceptor.log to safekeeper.pid and safekeeper.log * Change some mentions of WAL acceptor to safekeeper * Dockerfile: alias wal_acceptor to safekeeper temporarily until internal scripts are updated --- .circleci/config.yml | 2 +- Dockerfile | 4 +++- Dockerfile.alpine | 4 +++- docs/docker.md | 2 +- docs/multitenancy.md | 2 +- test_runner/fixtures/zenith_fixtures.py | 8 ++++---- walkeeper/src/bin/{wal_acceptor.rs => safekeeper.rs} | 12 ++++++------ walkeeper/src/send_wal.rs | 1 - walkeeper/src/timeline.rs | 2 +- walkeeper/src/wal_service.rs | 2 +- 10 files changed, 21 insertions(+), 18 deletions(-) rename walkeeper/src/bin/{wal_acceptor.rs => safekeeper.rs} (94%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 24d151f765..4fab11b1d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -260,7 +260,7 @@ jobs: when: always command: | du -sh /tmp/test_output/* - find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" -delete + find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" -delete du -sh /tmp/test_output/* - store_artifacts: path: /tmp/test_output diff --git a/Dockerfile b/Dockerfile index b38bac4480..0bbf03d13c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,9 @@ RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl mkdir zenith_install COPY --from=build /zenith/target/release/pageserver /usr/local/bin -COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin +COPY --from=build /zenith/target/release/safekeeper /usr/local/bin +# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 +RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install postgres_install COPY docker-entrypoint.sh /docker-entrypoint.sh diff --git a/Dockerfile.alpine b/Dockerfile.alpine index a2a2fea1a4..e0b569de22 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -81,7 +81,9 @@ FROM alpine:3.13 RUN apk add --update openssl build-base libseccomp-dev RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb COPY --from=build /zenith/target/release/pageserver /usr/local/bin -COPY --from=build /zenith/target/release/wal_acceptor /usr/local/bin +COPY --from=build /zenith/target/release/safekeeper /usr/local/bin +# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 +RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install /usr/local COPY docker-entrypoint.sh /docker-entrypoint.sh diff --git a/docs/docker.md b/docs/docker.md index 9a909ebfe3..14ba2146cb 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -4,7 +4,7 @@ Currently we build two main images: -- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `wal_acceptor` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile). +- [zenithdb/zenith](https://hub.docker.com/repository/docker/zenithdb/zenith) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile). - [zenithdb/compute-node](https://hub.docker.com/repository/docker/zenithdb/compute-node) — compute node image with pre-built Postgres binaries from [zenithdb/postgres](https://github.com/zenithdb/postgres). And two intermediate images used either to reduce build time or to deliver some additional binary tools from other repos: diff --git a/docs/multitenancy.md b/docs/multitenancy.md index c9a95116c5..4f1d45e970 100644 --- a/docs/multitenancy.md +++ b/docs/multitenancy.md @@ -56,4 +56,4 @@ Tenant id is passed to postgres via GUC the same way as the timeline. Tenant id ### Safety -For now particular tenant can only appear on a particular pageserver. Set of WAL acceptors are also pinned to particular (tenantid, timeline) pair so there can only be one writer for particular (tenantid, timeline). +For now particular tenant can only appear on a particular pageserver. Set of safekeepers are also pinned to particular (tenantid, timeline) pair so there can only be one writer for particular (tenantid, timeline). diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 597c39db0b..bf861f8b00 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -64,7 +64,7 @@ def pytest_configure(config): raise Exception('Too many workers configured. Cannot distrubute ports for services.') # does not use -c as it is not supported on macOS - cmd = ['pgrep', 'pageserver|postgres|wal_acceptor'] + cmd = ['pgrep', 'pageserver|postgres|safekeeper'] result = subprocess.run(cmd, stdout=subprocess.DEVNULL) if result.returncode == 0: # returncode of 0 means it found something. @@ -72,7 +72,7 @@ def pytest_configure(config): # result of the test. # NOTE this shows as an internal pytest error, there might be a better way raise Exception( - 'Found interfering processes running. Stop all Zenith pageservers, nodes, WALs, as well as stand-alone Postgres.' + 'Found interfering processes running. Stop all Zenith pageservers, nodes, safekeepers, as well as stand-alone Postgres.' ) @@ -843,7 +843,7 @@ class WalAcceptor: @property def pidfile(self) -> Path: - return self.data_dir / "wal_acceptor.pid" + return self.data_dir / "safekeeper.pid" def get_pid(self) -> Optional[int]: if not self.pidfile.exists(): @@ -895,7 +895,7 @@ class WalAcceptor: class WalAcceptorFactory: """ An object representing multiple running wal acceptors. """ def __init__(self, zenith_binpath: Path, data_dir: Path, pageserver_port: int, port_distributor: PortDistributor): - self.wa_bin_path = zenith_binpath / 'wal_acceptor' + self.wa_bin_path = zenith_binpath / 'safekeeper' self.data_dir = data_dir self.instances: List[WalAcceptor] = [] self.port_distributor = port_distributor diff --git a/walkeeper/src/bin/wal_acceptor.rs b/walkeeper/src/bin/safekeeper.rs similarity index 94% rename from walkeeper/src/bin/wal_acceptor.rs rename to walkeeper/src/bin/safekeeper.rs index f961c784d7..0a6448ffd3 100644 --- a/walkeeper/src/bin/wal_acceptor.rs +++ b/walkeeper/src/bin/safekeeper.rs @@ -1,5 +1,5 @@ // -// Main entry point for the wal_acceptor executable +// Main entry point for the safekeeper executable // use anyhow::Result; use clap::{App, Arg}; @@ -20,14 +20,14 @@ use walkeeper::WalAcceptorConf; fn main() -> Result<()> { zenith_metrics::set_common_metrics_prefix("safekeeper"); - let arg_matches = App::new("Zenith wal_acceptor") + let arg_matches = App::new("Zenith safekeeper") .about("Store WAL stream to local file system and push it to WAL receivers") .arg( Arg::with_name("datadir") .short("D") .long("dir") .takes_value(true) - .help("Path to the WAL acceptor data directory"), + .help("Path to the safekeeper data directory"), ) .arg( Arg::with_name("listen-pg") @@ -128,7 +128,7 @@ fn main() -> Result<()> { } fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { - let log_filename = conf.data_dir.join("wal_acceptor.log"); + let log_filename = conf.data_dir.join("safekeeper.log"); let log_file = logging::init(log_filename, conf.daemonize)?; let http_listener = TcpListener::bind(conf.listen_http_addr.clone()).map_err(|e| { @@ -136,7 +136,7 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { e })?; - info!("Starting wal acceptor on {}", conf.listen_pg_addr); + info!("Starting safekeeper on {}", conf.listen_pg_addr); let pg_listener = TcpListener::bind(conf.listen_pg_addr.clone()).map_err(|e| { error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); e @@ -151,7 +151,7 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { let stderr = log_file; let daemonize = Daemonize::new() - .pid_file("wal_acceptor.pid") + .pid_file("safekeeper.pid") .working_directory(Path::new(".")) .stdout(stdout) .stderr(stderr); diff --git a/walkeeper/src/send_wal.rs b/walkeeper/src/send_wal.rs index e81b6c5eac..e458939b3c 100644 --- a/walkeeper/src/send_wal.rs +++ b/walkeeper/src/send_wal.rs @@ -20,7 +20,6 @@ use crate::timeline::CreateControlFile; /// Handler for streaming WAL from acceptor pub struct SendWalHandler { - /// wal acceptor configuration pub conf: WalAcceptorConf, /// assigned application name pub appname: Option, diff --git a/walkeeper/src/timeline.rs b/walkeeper/src/timeline.rs index f0d69bed10..5ee97a8a28 100644 --- a/walkeeper/src/timeline.rs +++ b/walkeeper/src/timeline.rs @@ -112,7 +112,7 @@ impl SharedState { } match opts.open(&control_file_path) { Ok(mut file) => { - // Lock file to prevent two or more active wal_acceptors + // Lock file to prevent two or more active safekeepers match file.try_lock_exclusive() { Ok(()) => {} Err(e) => { diff --git a/walkeeper/src/wal_service.rs b/walkeeper/src/wal_service.rs index 747c06a116..4a294e9c95 100644 --- a/walkeeper/src/wal_service.rs +++ b/walkeeper/src/wal_service.rs @@ -36,7 +36,7 @@ fn handle_socket(socket: TcpStream, conf: WalAcceptorConf) -> Result<()> { let mut conn_handler = SendWalHandler::new(conf); let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None, false)?; - // libpq replication protocol between wal_acceptor and replicas/pagers + // libpq replication protocol between safekeeper and replicas/pagers pgbackend.run(&mut conn_handler)?; Ok(()) From 64ca94772263b303c83e637d203cbc13f543a3fd Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Tue, 12 Oct 2021 19:05:16 +0000 Subject: [PATCH 21/46] Dockerfile: remove wal_acceptor alias for safekeeper --- Dockerfile | 2 -- Dockerfile.alpine | 2 -- 2 files changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0bbf03d13c..528f29597f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,8 +38,6 @@ RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl COPY --from=build /zenith/target/release/pageserver /usr/local/bin COPY --from=build /zenith/target/release/safekeeper /usr/local/bin -# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 -RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install postgres_install COPY docker-entrypoint.sh /docker-entrypoint.sh diff --git a/Dockerfile.alpine b/Dockerfile.alpine index e0b569de22..dafb7eaf6b 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -82,8 +82,6 @@ RUN apk add --update openssl build-base libseccomp-dev RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb COPY --from=build /zenith/target/release/pageserver /usr/local/bin COPY --from=build /zenith/target/release/safekeeper /usr/local/bin -# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 -RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install /usr/local COPY docker-entrypoint.sh /docker-entrypoint.sh From f6582635431ce934fc321ded7593802cb2c104df Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Tue, 12 Oct 2021 19:05:58 +0000 Subject: [PATCH 22/46] Revert "Dockerfile: remove wal_acceptor alias for safekeeper" This reverts commit 64ca94772263b303c83e637d203cbc13f543a3fd. --- Dockerfile | 2 ++ Dockerfile.alpine | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 528f29597f..0bbf03d13c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,8 @@ RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl COPY --from=build /zenith/target/release/pageserver /usr/local/bin COPY --from=build /zenith/target/release/safekeeper /usr/local/bin +# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 +RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install postgres_install COPY docker-entrypoint.sh /docker-entrypoint.sh diff --git a/Dockerfile.alpine b/Dockerfile.alpine index dafb7eaf6b..e0b569de22 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -82,6 +82,8 @@ RUN apk add --update openssl build-base libseccomp-dev RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb COPY --from=build /zenith/target/release/pageserver /usr/local/bin COPY --from=build /zenith/target/release/safekeeper /usr/local/bin +# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 +RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install /usr/local COPY docker-entrypoint.sh /docker-entrypoint.sh From 1c29de81dedbb64be88afbe6f9a7b22b28075c2b Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Tue, 12 Oct 2021 20:45:21 -0700 Subject: [PATCH 23/46] pageserver - remove `lsn` from `WALRecord` --- pageserver/src/layered_repository.rs | 10 +++++----- pageserver/src/layered_repository/delta_layer.rs | 4 ++-- .../src/layered_repository/inmemory_layer.rs | 8 ++++---- .../src/layered_repository/storage_layer.rs | 2 +- pageserver/src/repository.rs | 14 +++++--------- pageserver/src/restore_local_repo.rs | 9 +++++---- pageserver/src/walredo.rs | 16 ++++++++-------- 7 files changed, 30 insertions(+), 33 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 4de6419444..7c992ddb0c 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -1695,7 +1695,7 @@ impl LayeredTimeline { // // If we don't have a base image, then the oldest WAL record better initialize // the page - if data.page_img.is_none() && !data.records.first().unwrap().will_init { + if data.page_img.is_none() && !data.records.first().unwrap().1.will_init { // FIXME: this ought to be an error? warn!( "Base image for page {}/{} at {} not found, but got {} WAL records", @@ -1773,7 +1773,7 @@ impl Deref for LayeredTimelineWriter<'_> { } impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { - fn put_wal_record(&self, rel: RelishTag, blknum: u32, rec: WALRecord) -> Result<()> { + fn put_wal_record(&self, lsn: Lsn, rel: RelishTag, blknum: u32, rec: WALRecord) -> Result<()> { if !rel.is_blocky() && blknum != 0 { bail!( "invalid request for block {} for non-blocky relish {}", @@ -1781,11 +1781,11 @@ impl<'a> TimelineWriter for LayeredTimelineWriter<'a> { rel ); } - ensure!(rec.lsn.is_aligned(), "unaligned record LSN"); + ensure!(lsn.is_aligned(), "unaligned record LSN"); let seg = SegmentTag::from_blknum(rel, blknum); - let layer = self.tl.get_layer_for_write(seg, rec.lsn)?; - let delta_size = layer.put_wal_record(blknum, rec); + let layer = self.tl.get_layer_for_write(seg, lsn)?; + let delta_size = layer.put_wal_record(lsn, blknum, rec); self.tl .increase_current_logical_size(delta_size * BLCKSZ as u32); Ok(()) diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index e93eddb7e6..24ed9d6e69 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -198,7 +198,7 @@ impl Layer for DeltaLayer { .slice_range((Included(&minkey), Included(&maxkey))) .iter() .rev(); - for ((_blknum, _lsn), blob_range) in iter { + for ((_blknum, pv_lsn), blob_range) in iter { let pv = PageVersion::des(&read_blob(&page_version_reader, blob_range)?)?; if let Some(img) = pv.page_image { @@ -208,7 +208,7 @@ impl Layer for DeltaLayer { break; } else if let Some(rec) = pv.record { let will_init = rec.will_init; - reconstruct_data.records.push(rec); + reconstruct_data.records.push((*pv_lsn, rec)); if will_init { // This WAL record initializes the page, so no need to go further back need_image = false; diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index bd1860fd47..474eef09c4 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -160,13 +160,13 @@ impl Layer for InMemoryLayer { .get_block_lsn_range(blknum, ..=lsn) .iter() .rev(); - for (_entry_lsn, entry) in iter { + for (entry_lsn, entry) in iter { if let Some(img) = &entry.page_image { reconstruct_data.page_img = Some(img.clone()); need_image = false; break; } else if let Some(rec) = &entry.record { - reconstruct_data.records.push(rec.clone()); + reconstruct_data.records.push((*entry_lsn, rec.clone())); if rec.will_init { // This WAL record initializes the page, so no need to go further back need_image = false; @@ -337,10 +337,10 @@ impl InMemoryLayer { // Write operations /// Remember new page version, as a WAL record over previous version - pub fn put_wal_record(&self, blknum: u32, rec: WALRecord) -> u32 { + pub fn put_wal_record(&self, lsn: Lsn, blknum: u32, rec: WALRecord) -> u32 { self.put_page_version( blknum, - rec.lsn, + lsn, PageVersion { page_image: None, record: Some(rec), diff --git a/pageserver/src/layered_repository/storage_layer.rs b/pageserver/src/layered_repository/storage_layer.rs index c49fbbdd99..0a86fe407d 100644 --- a/pageserver/src/layered_repository/storage_layer.rs +++ b/pageserver/src/layered_repository/storage_layer.rs @@ -78,7 +78,7 @@ pub struct PageVersion { /// 'records' contains the records to apply over the base image. /// pub struct PageReconstructData { - pub records: Vec, + pub records: Vec<(Lsn, WALRecord)>, pub page_img: Option, } diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index f033c74c31..56e551a275 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -162,7 +162,7 @@ pub trait TimelineWriter: Deref { /// /// This will implicitly extend the relation, if the page is beyond the /// current end-of-file. - fn put_wal_record(&self, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>; + fn put_wal_record(&self, lsn: Lsn, tag: RelishTag, blknum: u32, rec: WALRecord) -> Result<()>; /// Like put_wal_record, but with ready-made image of the page. fn put_page_image(&self, tag: RelishTag, blknum: u32, lsn: Lsn, img: Bytes) -> Result<()>; @@ -182,7 +182,6 @@ pub trait TimelineWriter: Deref { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct WALRecord { - pub lsn: Lsn, // LSN at the *end* of the record pub will_init: bool, pub rec: Bytes, // Remember the offset of main_data in rec, @@ -193,22 +192,19 @@ pub struct WALRecord { impl WALRecord { pub fn pack(&self, buf: &mut BytesMut) { - buf.put_u64(self.lsn.0); buf.put_u8(self.will_init as u8); buf.put_u32(self.main_data_offset); buf.put_u32(self.rec.len() as u32); buf.put_slice(&self.rec[..]); } pub fn unpack(buf: &mut Bytes) -> WALRecord { - let lsn = Lsn::from(buf.get_u64()); let will_init = buf.get_u8() != 0; let main_data_offset = buf.get_u32(); - let mut dst = vec![0u8; buf.get_u32() as usize]; - buf.copy_to_slice(&mut dst); + let rec_len = buf.get_u32() as usize; + let rec = buf.split_to(rec_len); WALRecord { - lsn, will_init, - rec: Bytes::from(dst), + rec, main_data_offset, } } @@ -832,7 +828,7 @@ mod tests { blknum: u32, lsn: Lsn, base_img: Option, - records: Vec, + records: Vec<(Lsn, WALRecord)>, ) -> Result { let s = format!( "redo for {} blk {} to get to {}, with {} and {} records", diff --git a/pageserver/src/restore_local_repo.rs b/pageserver/src/restore_local_repo.rs index bf4fb12d0a..60eb9ce278 100644 --- a/pageserver/src/restore_local_repo.rs +++ b/pageserver/src/restore_local_repo.rs @@ -312,13 +312,12 @@ pub fn save_decoded_record( }); let rec = WALRecord { - lsn, will_init: blk.will_init || blk.apply_image, rec: recdata.clone(), main_data_offset: decoded.main_data_offset as u32, }; - timeline.put_wal_record(tag, blk.blkno, rec)?; + timeline.put_wal_record(lsn, tag, blk.blkno, rec)?; } let mut buf = decoded.record.clone(); @@ -656,12 +655,12 @@ fn save_xact_record( let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; let rec = WALRecord { - lsn, will_init: false, rec: decoded.record.clone(), main_data_offset: decoded.main_data_offset as u32, }; timeline.put_wal_record( + lsn, RelishTag::Slru { slru: SlruKind::Clog, segno, @@ -677,6 +676,7 @@ fn save_xact_record( let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; timeline.put_wal_record( + lsn, RelishTag::Slru { slru: SlruKind::Clog, segno, @@ -771,7 +771,6 @@ fn save_multixact_create_record( decoded: &DecodedWALRecord, ) -> Result<()> { let rec = WALRecord { - lsn, will_init: false, rec: decoded.record.clone(), main_data_offset: decoded.main_data_offset as u32, @@ -780,6 +779,7 @@ fn save_multixact_create_record( let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; timeline.put_wal_record( + lsn, RelishTag::Slru { slru: SlruKind::MultiXactOffsets, segno, @@ -799,6 +799,7 @@ fn save_multixact_create_record( let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; timeline.put_wal_record( + lsn, RelishTag::Slru { slru: SlruKind::MultiXactMembers, segno, diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index e9382c4da5..8cd696e8f3 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -82,7 +82,7 @@ pub trait WalRedoManager: Send + Sync { blknum: u32, lsn: Lsn, base_img: Option, - records: Vec, + records: Vec<(Lsn, WALRecord)>, ) -> Result; } @@ -99,7 +99,7 @@ impl crate::walredo::WalRedoManager for DummyRedoManager { _blknum: u32, _lsn: Lsn, _base_img: Option, - _records: Vec, + _records: Vec<(Lsn, WALRecord)>, ) -> Result { Err(WalRedoError::InvalidState) } @@ -150,7 +150,7 @@ struct WalRedoRequest { lsn: Lsn, base_img: Option, - records: Vec, + records: Vec<(Lsn, WALRecord)>, } /// An error happened in WAL redo @@ -179,7 +179,7 @@ impl WalRedoManager for PostgresRedoManager { blknum: u32, lsn: Lsn, base_img: Option, - records: Vec, + records: Vec<(Lsn, WALRecord)>, ) -> Result { let start_time; let lock_time; @@ -277,7 +277,7 @@ impl PostgresRedoManager { page.extend_from_slice(&ZERO_PAGE); } // Apply all collected WAL records - for record in records { + for (_lsn, record) in records { let mut buf = record.rec.clone(); WAL_REDO_RECORD_COUNTER.inc(); @@ -544,7 +544,7 @@ impl PostgresRedoProcess { &mut self, tag: BufferTag, base_img: Option, - records: &[WALRecord], + records: &[(Lsn, WALRecord)], ) -> Result { let stdout = &mut self.stdout; // Buffer the writes to avoid a lot of small syscalls. @@ -570,11 +570,11 @@ impl PostgresRedoProcess { } // Send WAL records. - for rec in records.iter() { + for (lsn, rec) in records.iter() { WAL_REDO_RECORD_COUNTER.inc(); stdin - .write_all(&build_apply_record_msg(rec.lsn, &rec.rec)) + .write_all(&build_apply_record_msg(*lsn, &rec.rec)) .await?; //debug!("sent WAL record to wal redo postgres process ({:X}/{:X}", From 96f1175a8010b577cc64ac992f81f16a54702463 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 13 Oct 2021 10:52:47 +0300 Subject: [PATCH 24/46] Cleanup hardcoded oids. --- walkeeper/src/send_wal.rs | 15 ++++++--------- zenith_utils/src/pq_proto.rs | 3 ++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/walkeeper/src/send_wal.rs b/walkeeper/src/send_wal.rs index e458939b3c..fcd8595e15 100644 --- a/walkeeper/src/send_wal.rs +++ b/walkeeper/src/send_wal.rs @@ -13,7 +13,7 @@ use std::str::FromStr; use std::sync::Arc; use zenith_utils::postgres_backend; use zenith_utils::postgres_backend::PostgresBackend; -use zenith_utils::pq_proto::{BeMessage, FeStartupMessage, RowDescriptor}; +use zenith_utils::pq_proto::{BeMessage, FeStartupMessage, RowDescriptor, INT4_OID, TEXT_OID}; use zenith_utils::zid::{ZTenantId, ZTimelineId}; use crate::timeline::CreateControlFile; @@ -71,19 +71,16 @@ impl postgres_backend::Handler for SendWalHandler { } if query_string.starts_with(b"IDENTIFY_SYSTEM") { self.handle_identify_system(pgb)?; - Ok(()) } else if query_string.starts_with(b"START_REPLICATION") { ReplicationConn::new(pgb).run(self, pgb, &query_string)?; - Ok(()) } else if query_string.starts_with(b"START_WAL_PUSH") { ReceiveWalConn::new(pgb)?.run(self)?; - Ok(()) } else if query_string.starts_with(b"JSON_CTRL") { handle_json_ctrl(self, pgb, &query_string)?; - Ok(()) } else { bail!("Unexpected command {:?}", query_string); } + Ok(()) } } @@ -113,25 +110,25 @@ impl SendWalHandler { pgb.write_message_noflush(&BeMessage::RowDescription(&[ RowDescriptor { name: b"systemid", - typoid: 25, + typoid: TEXT_OID, typlen: -1, ..Default::default() }, RowDescriptor { name: b"timeline", - typoid: 23, + typoid: INT4_OID, typlen: 4, ..Default::default() }, RowDescriptor { name: b"xlogpos", - typoid: 25, + typoid: TEXT_OID, typlen: -1, ..Default::default() }, RowDescriptor { name: b"dbname", - typoid: 25, + typoid: TEXT_OID, typlen: -1, ..Default::default() }, diff --git a/zenith_utils/src/pq_proto.rs b/zenith_utils/src/pq_proto.rs index 3a87906b22..fe66f9d5a2 100644 --- a/zenith_utils/src/pq_proto.rs +++ b/zenith_utils/src/pq_proto.rs @@ -15,8 +15,9 @@ use std::str; pub type Oid = u32; pub type SystemId = u64; -pub const TEXT_OID: Oid = 25; pub const INT8_OID: Oid = 20; +pub const INT4_OID: Oid = 23; +pub const TEXT_OID: Oid = 25; #[derive(Debug)] pub enum FeMessage { From 6b6b3f68bee8c7ca6caf8b068cbeb76493eb082b Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Wed, 13 Oct 2021 16:28:24 +0300 Subject: [PATCH 25/46] Safekeeper metrics refactor (#747) --- walkeeper/src/safekeeper.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/walkeeper/src/safekeeper.rs b/walkeeper/src/safekeeper.rs index 111cbbaf95..d7f5623002 100644 --- a/walkeeper/src/safekeeper.rs +++ b/walkeeper/src/safekeeper.rs @@ -290,13 +290,13 @@ lazy_static! { "Current flush_lsn, grouped by timeline", &["ztli"] ) - .expect("Failed to register safekeeper_flush_lsn int gauge vec"); + .expect("Failed to register safekeeper_flush_lsn gauge vec"); static ref COMMIT_LSN_GAUGE: GaugeVec = register_gauge_vec!( "safekeeper_commit_lsn", "Current commit_lsn (not necessarily persisted to disk), grouped by timeline", &["ztli"] ) - .expect("Failed to register safekeeper_commit_lsn int gauge vec"); + .expect("Failed to register safekeeper_commit_lsn gauge vec"); } struct SafeKeeperMetrics { @@ -312,6 +312,13 @@ impl SafeKeeperMetrics { commit_lsn: COMMIT_LSN_GAUGE.with_label_values(&[&ztli_str]), } } + + fn new_noname() -> SafeKeeperMetrics { + SafeKeeperMetrics { + flush_lsn: FLUSH_LSN_GAUGE.with_label_values(&["n/a"]), + commit_lsn: COMMIT_LSN_GAUGE.with_label_values(&["n/a"]), + } + } } /// SafeKeeper which consumes events (messages from compute) and provides @@ -322,7 +329,7 @@ pub struct SafeKeeper { pub flush_lsn: Lsn, pub tli: u32, // Cached metrics so we don't have to recompute labels on each update. - metrics: Option, + metrics: SafeKeeperMetrics, /// not-yet-flushed pairs of same named fields in s.* pub commit_lsn: Lsn, pub truncate_lsn: Lsn, @@ -341,7 +348,7 @@ where SafeKeeper { flush_lsn, tli, - metrics: None, + metrics: SafeKeeperMetrics::new_noname(), commit_lsn: state.commit_lsn, truncate_lsn: state.truncate_lsn, storage, @@ -393,7 +400,7 @@ where self.s.server.wal_seg_size = msg.wal_seg_size; self.storage.persist(&self.s, true)?; - self.metrics = Some(SafeKeeperMetrics::new(self.s.server.ztli)); + self.metrics = SafeKeeperMetrics::new(self.s.server.ztli); info!( "processed greeting from proposer {:?}, sending term {:?}", @@ -518,11 +525,7 @@ where } if last_rec_lsn > self.flush_lsn { self.flush_lsn = last_rec_lsn; - self.metrics - .as_ref() - .unwrap() - .flush_lsn - .set(u64::from(self.flush_lsn) as f64); + self.metrics.flush_lsn.set(u64::from(self.flush_lsn) as f64); } // Advance commit_lsn taking into account what we have locally. xxx this @@ -541,8 +544,6 @@ where commit_lsn >= msg.h.epoch_start_lsn && self.s.commit_lsn < msg.h.epoch_start_lsn; self.commit_lsn = commit_lsn; self.metrics - .as_ref() - .unwrap() .commit_lsn .set(u64::from(self.commit_lsn) as f64); } From 8a4f092e82aefb2136b96249daca52cdc4508100 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 13 Oct 2021 16:59:00 +0300 Subject: [PATCH 26/46] Skip syncing the temp initdb installation. Doesn't make much difference on my laptop with SSD, but every little helps, and with a slower disk it might be noticeable. --- pageserver/src/branches.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pageserver/src/branches.rs b/pageserver/src/branches.rs index 00fd50a2a2..15e56b18e5 100644 --- a/pageserver/src/branches.rs +++ b/pageserver/src/branches.rs @@ -183,6 +183,9 @@ fn run_initdb(conf: &'static PageServerConf, initdbpath: &Path) -> Result<()> { .args(&["-D", initdbpath.to_str().unwrap()]) .args(&["-U", &conf.superuser]) .arg("--no-instructions") + // This is only used for a temporary installation that is deleted shortly after, + // so no need to fsync it + .arg("--no-sync") .env_clear() .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap()) .env("DYLD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap()) From 43957f44012fa680b87aad8c8e355ee7b089271c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 13 Oct 2021 15:44:15 +0300 Subject: [PATCH 27/46] [cross-repo-ci] Use solely commit hash to test PRs in CI See #744 for the discussion. --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fab11b1d5..629de37522 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -325,8 +325,7 @@ jobs: \"inputs\": { \"ci_job_name\": \"zenith-remote-ci\", \"commit_hash\": \"$CIRCLE_SHA1\", - \"remote_repo\": \"$LOCAL_REPO\", - \"zenith_image_branch\": \"$CIRCLE_BRANCH\" + \"remote_repo\": \"$LOCAL_REPO\" } }" From 4b87acb1f6c3149e31f96231827f998dc5eff0ec Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 14 Oct 2021 13:10:09 +0300 Subject: [PATCH 28/46] Use logging in python tests (#674) * Use logging in python tests * Use f-strings for logs * Don't log test output while running * Use only pytest logging handler * Add more info about pytest logging --- .circleci/config.yml | 6 +-- .../batch_others/test_branch_behind.py | 12 ++--- .../batch_others/test_clog_truncate.py | 17 +++---- test_runner/batch_others/test_config.py | 3 +- test_runner/batch_others/test_createdropdb.py | 9 ++-- test_runner/batch_others/test_createuser.py | 3 +- test_runner/batch_others/test_multixact.py | 5 +- .../batch_others/test_old_request_lsn.py | 5 +- .../batch_others/test_pageserver_restart.py | 3 +- test_runner/batch_others/test_pgbench.py | 3 +- .../batch_others/test_restart_compute.py | 13 ++--- test_runner/batch_others/test_snapfiles_gc.py | 23 ++++----- .../batch_others/test_timeline_size.py | 4 +- test_runner/batch_others/test_twophase.py | 10 ++-- test_runner/batch_others/test_vm_bits.py | 5 +- test_runner/batch_others/test_wal_acceptor.py | 5 +- .../batch_others/test_wal_acceptor_async.py | 15 +++--- .../batch_pg_regress/test_zenith_regress.py | 3 +- test_runner/fixtures/benchmark_fixture.py | 2 - test_runner/fixtures/log_helper.py | 44 +++++++++++++++++ test_runner/fixtures/utils.py | 12 +---- test_runner/fixtures/zenith_fixtures.py | 48 +++++++++---------- test_runner/performance/test_bulk_insert.py | 3 +- test_runner/performance/test_gist_build.py | 3 +- test_runner/performance/test_perf_pgbench.py | 3 +- .../performance/test_write_amplification.py | 3 +- test_runner/pytest.ini | 2 + test_runner/test_broken.py | 6 ++- 28 files changed, 163 insertions(+), 107 deletions(-) create mode 100644 test_runner/fixtures/log_helper.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 629de37522..bd3564c71c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -245,13 +245,13 @@ jobs: # # The junit.xml file allows CircleCI to display more fine-grained test information # in its "Tests" tab in the results page. - # -s prevents pytest from capturing output, which helps to see - # what's going on if the test hangs # --verbose prints name of each test (helpful when there are # multiple tests in one file) # -rA prints summary in the end # -n4 uses four processes to run tests via pytest-xdist - pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS + # -s is not used to prevent pytest from capturing output, because tests are running + # in parallel and logs are mixed between different tests + pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short --verbose -rA $TEST_SELECTION $EXTRA_PARAMS - run: # CircleCI artifacts are preserved one file at a time, so skipping # this step isn't a good idea. If you want to extract the diff --git a/test_runner/batch_others/test_branch_behind.py b/test_runner/batch_others/test_branch_behind.py index 9189017050..47c2f0b2f9 100644 --- a/test_runner/batch_others/test_branch_behind.py +++ b/test_runner/batch_others/test_branch_behind.py @@ -1,6 +1,6 @@ import subprocess from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver - +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -13,7 +13,7 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg zenith_cli.run(["branch", "test_branch_behind", "empty"]) pgmain = postgres.create_start('test_branch_behind') - print("postgres is running on 'test_branch_behind' branch") + log.info("postgres is running on 'test_branch_behind' branch") main_pg_conn = pgmain.connect() main_cur = main_pg_conn.cursor() @@ -27,7 +27,7 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg ''') main_cur.execute('SELECT pg_current_wal_insert_lsn()') lsn_a = main_cur.fetchone()[0] - print('LSN after 100 rows: ' + lsn_a) + log.info(f'LSN after 100 rows: {lsn_a}') # Insert some more rows. (This generates enough WAL to fill a few segments.) main_cur.execute(''' @@ -37,7 +37,7 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg ''') main_cur.execute('SELECT pg_current_wal_insert_lsn()') lsn_b = main_cur.fetchone()[0] - print('LSN after 200100 rows: ' + lsn_b) + log.info(f'LSN after 200100 rows: {lsn_b}') # Branch at the point where only 100 rows were inserted zenith_cli.run(["branch", "test_branch_behind_hundred", "test_branch_behind@" + lsn_a]) @@ -52,7 +52,7 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg main_cur.execute('SELECT pg_current_wal_insert_lsn()') lsn_c = main_cur.fetchone()[0] - print('LSN after 400100 rows: ' + lsn_c) + log.info(f'LSN after 400100 rows: {lsn_c}') # Branch at the point where only 200100 rows were inserted zenith_cli.run(["branch", "test_branch_behind_more", "test_branch_behind@" + lsn_b]) @@ -89,4 +89,4 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg try: zenith_cli.run(["branch", "test_branch_preinitdb", "test_branch_behind@0/42"]) except subprocess.CalledProcessError: - print("Branch creation with pre-initdb LSN failed (as expected)") + log.info("Branch creation with pre-initdb LSN failed (as expected)") diff --git a/test_runner/batch_others/test_clog_truncate.py b/test_runner/batch_others/test_clog_truncate.py index e9233986e4..8ad3c22732 100644 --- a/test_runner/batch_others/test_clog_truncate.py +++ b/test_runner/batch_others/test_clog_truncate.py @@ -4,6 +4,7 @@ import os from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -24,7 +25,7 @@ def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: Postg ] pg = postgres.create_start('test_clog_truncate', config_lines=config) - print('postgres is running on test_clog_truncate branch') + log.info('postgres is running on test_clog_truncate branch') # Install extension containing function needed for test pg.safe_psql('CREATE EXTENSION zenith_test_utils') @@ -33,22 +34,22 @@ def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: Postg with closing(pg.connect()) as conn: with conn.cursor() as cur: cur.execute('select test_consume_xids(1000*1000*10);') - print('xids consumed') + log.info('xids consumed') # call a checkpoint to trigger TruncateSubtrans cur.execute('CHECKPOINT;') # ensure WAL flush cur.execute('select txid_current()') - print(cur.fetchone()) + log.info(cur.fetchone()) # wait for autovacuum to truncate the pg_xact # XXX Is it worth to add a timeout here? pg_xact_0000_path = os.path.join(pg.pg_xact_dir_path(), '0000') - print("pg_xact_0000_path = " + pg_xact_0000_path) + log.info(f"pg_xact_0000_path = {pg_xact_0000_path}") while os.path.isfile(pg_xact_0000_path): - print("file exists. wait for truncation. " "pg_xact_0000_path = " + pg_xact_0000_path) + log.info(f"file exists. wait for truncation. " "pg_xact_0000_path = {pg_xact_0000_path}") time.sleep(5) # checkpoint to advance latest lsn @@ -59,14 +60,14 @@ def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: Postg lsn_after_truncation = cur.fetchone()[0] # create new branch after clog truncation and start a compute node on it - print('create branch at lsn_after_truncation ' + lsn_after_truncation) + log.info(f'create branch at lsn_after_truncation {lsn_after_truncation}') zenith_cli.run( ["branch", "test_clog_truncate_new", "test_clog_truncate@" + lsn_after_truncation]) pg2 = postgres.create_start('test_clog_truncate_new') - print('postgres is running on test_clog_truncate_new branch') + log.info('postgres is running on test_clog_truncate_new branch') # check that new node doesn't contain truncated segment pg_xact_0000_path_new = os.path.join(pg2.pg_xact_dir_path(), '0000') - print("pg_xact_0000_path_new = " + pg_xact_0000_path_new) + log.info(f"pg_xact_0000_path_new = {pg_xact_0000_path_new}") assert os.path.isfile(pg_xact_0000_path_new) is False diff --git a/test_runner/batch_others/test_config.py b/test_runner/batch_others/test_config.py index d8cc798839..d7c59c4e77 100644 --- a/test_runner/batch_others/test_config.py +++ b/test_runner/batch_others/test_config.py @@ -1,6 +1,7 @@ from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -14,7 +15,7 @@ def test_config(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFact # change config pg = postgres.create_start('test_config', config_lines=['log_min_messages=debug1']) - print('postgres is running on test_config branch') + log.info('postgres is running on test_config branch') with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_createdropdb.py b/test_runner/batch_others/test_createdropdb.py index cbe89a77cb..4194538556 100644 --- a/test_runner/batch_others/test_createdropdb.py +++ b/test_runner/batch_others/test_createdropdb.py @@ -3,6 +3,7 @@ import pathlib from contextlib import closing from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory, ZenithCli, check_restored_datadir_content +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -19,7 +20,7 @@ def test_createdb( zenith_cli.run(["branch", "test_createdb", "empty"]) pg = postgres.create_start('test_createdb') - print("postgres is running on 'test_createdb' branch") + log.info("postgres is running on 'test_createdb' branch") with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -53,7 +54,7 @@ def test_dropdb( zenith_cli.run(["branch", "test_dropdb", "empty"]) pg = postgres.create_start('test_dropdb') - print("postgres is running on 'test_dropdb' branch") + log.info("postgres is running on 'test_dropdb' branch") with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -88,13 +89,13 @@ def test_dropdb( # Test that database subdir exists on the branch before drop dbpath = pathlib.Path(pg_before.pgdata_dir) / 'base' / str(dboid) - print(dbpath) + log.info(dbpath) assert os.path.isdir(dbpath) == True # Test that database subdir doesn't exist on the branch after drop dbpath = pathlib.Path(pg_after.pgdata_dir) / 'base' / str(dboid) - print(dbpath) + log.info(dbpath) assert os.path.isdir(dbpath) == False diff --git a/test_runner/batch_others/test_createuser.py b/test_runner/batch_others/test_createuser.py index f44df91c3c..57cc610f55 100644 --- a/test_runner/batch_others/test_createuser.py +++ b/test_runner/batch_others/test_createuser.py @@ -1,6 +1,7 @@ from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -12,7 +13,7 @@ def test_createuser(zenith_cli, pageserver: ZenithPageserver, postgres: Postgres zenith_cli.run(["branch", "test_createuser", "empty"]) pg = postgres.create_start('test_createuser') - print("postgres is running on 'test_createuser' branch") + log.info("postgres is running on 'test_createuser' branch") with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_multixact.py b/test_runner/batch_others/test_multixact.py index aaa9e7f58d..403eee9974 100644 --- a/test_runner/batch_others/test_multixact.py +++ b/test_runner/batch_others/test_multixact.py @@ -1,4 +1,5 @@ from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver, check_restored_datadir_content +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -15,7 +16,7 @@ def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, zenith_cli.run(["branch", "test_multixact", "empty"]) pg = postgres.create_start('test_multixact') - print("postgres is running on 'test_multixact' branch") + log.info("postgres is running on 'test_multixact' branch") pg_conn = pg.connect() cur = pg_conn.cursor() @@ -55,7 +56,7 @@ def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, zenith_cli.run(["branch", "test_multixact_new", "test_multixact@" + lsn]) pg_new = postgres.create_start('test_multixact_new') - print("postgres is running on 'test_multixact_new' branch") + log.info("postgres is running on 'test_multixact_new' branch") pg_new_conn = pg_new.connect() cur_new = pg_new_conn.cursor() diff --git a/test_runner/batch_others/test_old_request_lsn.py b/test_runner/batch_others/test_old_request_lsn.py index bb28bdd83f..49e87210e4 100644 --- a/test_runner/batch_others/test_old_request_lsn.py +++ b/test_runner/batch_others/test_old_request_lsn.py @@ -1,6 +1,7 @@ from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -18,7 +19,7 @@ def test_old_request_lsn(zenith_cli, pageserver: ZenithPageserver, postgres: Pos # Create a branch for us zenith_cli.run(["branch", "test_old_request_lsn", "empty"]) pg = postgres.create_start('test_old_request_lsn') - print('postgres is running on test_old_request_lsn branch') + log.info('postgres is running on test_old_request_lsn branch') pg_conn = pg.connect() cur = pg_conn.cursor() @@ -46,7 +47,7 @@ def test_old_request_lsn(zenith_cli, pageserver: ZenithPageserver, postgres: Pos from pg_settings where name = 'shared_buffers' ''') row = cur.fetchone() - print(f'shared_buffers is {row[0]}, table size {row[1]}'); + log.info(f'shared_buffers is {row[0]}, table size {row[1]}'); assert int(row[0]) < int(row[1]) cur.execute('VACUUM foo'); diff --git a/test_runner/batch_others/test_pageserver_restart.py b/test_runner/batch_others/test_pageserver_restart.py index 18b17a4efb..0656b7c6e8 100644 --- a/test_runner/batch_others/test_pageserver_restart.py +++ b/test_runner/batch_others/test_pageserver_restart.py @@ -5,6 +5,7 @@ import time from contextlib import closing from multiprocessing import Process, Value from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -40,7 +41,7 @@ def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: from pg_settings where name = 'shared_buffers' ''') row = cur.fetchone() - print("shared_buffers is {}, table size {}", row[0], row[1]); + log.info(f"shared_buffers is {row[0]}, table size {row[1]}"); assert int(row[0]) < int(row[1]) # Stop and restart pageserver. This is a more or less graceful shutdown, although diff --git a/test_runner/batch_others/test_pgbench.py b/test_runner/batch_others/test_pgbench.py index a5423cf3d7..46633daa34 100644 --- a/test_runner/batch_others/test_pgbench.py +++ b/test_runner/batch_others/test_pgbench.py @@ -1,4 +1,5 @@ from fixtures.zenith_fixtures import PostgresFactory +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -8,7 +9,7 @@ def test_pgbench(postgres: PostgresFactory, pg_bin, zenith_cli): zenith_cli.run(["branch", "test_pgbench", "empty"]) pg = postgres.create_start('test_pgbench') - print("postgres is running on 'test_pgbench' branch") + log.info("postgres is running on 'test_pgbench' branch") connstr = pg.connstr() diff --git a/test_runner/batch_others/test_restart_compute.py b/test_runner/batch_others/test_restart_compute.py index 193b675e23..48a19b490b 100644 --- a/test_runner/batch_others/test_restart_compute.py +++ b/test_runner/batch_others/test_restart_compute.py @@ -2,6 +2,7 @@ import pytest from contextlib import closing from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -27,7 +28,7 @@ def test_restart_compute( pg = postgres.create_start('test_restart_compute', wal_acceptors=wal_acceptor_connstrs) - print("postgres is running on 'test_restart_compute' branch") + log.info("postgres is running on 'test_restart_compute' branch") with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -36,7 +37,7 @@ def test_restart_compute( cur.execute('SELECT sum(key) FROM t') r = cur.fetchone() assert r == (5000050000, ) - print("res = ", r) + log.info(f"res = {r}") # Remove data directory and restart pg.stop_and_destroy().create_start('test_restart_compute', @@ -49,7 +50,7 @@ def test_restart_compute( cur.execute('SELECT sum(key) FROM t') r = cur.fetchone() assert r == (5000050000, ) - print("res = ", r) + log.info(f"res = {r}") # Insert another row cur.execute("INSERT INTO t VALUES (100001, 'payload2')") @@ -57,7 +58,7 @@ def test_restart_compute( r = cur.fetchone() assert r == (100001, ) - print("res = ", r) + log.info(f"res = {r}") # Again remove data directory and restart pg.stop_and_destroy().create_start('test_restart_compute', @@ -72,7 +73,7 @@ def test_restart_compute( r = cur.fetchone() assert r == (100001, ) - print("res = ", r) + log.info(f"res = {r}") # And again remove data directory and restart pg.stop_and_destroy().create_start('test_restart_compute', @@ -85,4 +86,4 @@ def test_restart_compute( r = cur.fetchone() assert r == (100001, ) - print("res = ", r) + log.info(f"res = {r}") diff --git a/test_runner/batch_others/test_snapfiles_gc.py b/test_runner/batch_others/test_snapfiles_gc.py index e01bf7f179..9cd01ca42e 100644 --- a/test_runner/batch_others/test_snapfiles_gc.py +++ b/test_runner/batch_others/test_snapfiles_gc.py @@ -1,13 +1,14 @@ from contextlib import closing import psycopg2.extras -import time; +import time +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") def print_gc_result(row): - print("GC duration {elapsed} ms".format_map(row)); - print(" REL total: {layer_relfiles_total}, needed_by_cutoff {layer_relfiles_needed_by_cutoff}, needed_by_branches: {layer_relfiles_needed_by_branches}, not_updated: {layer_relfiles_not_updated}, needed_as_tombstone {layer_relfiles_needed_as_tombstone}, removed: {layer_relfiles_removed}, dropped: {layer_relfiles_dropped}".format_map(row)) - print(" NONREL total: {layer_nonrelfiles_total}, needed_by_cutoff {layer_nonrelfiles_needed_by_cutoff}, needed_by_branches: {layer_nonrelfiles_needed_by_branches}, not_updated: {layer_nonrelfiles_not_updated}, needed_as_tombstone {layer_nonrelfiles_needed_as_tombstone}, removed: {layer_nonrelfiles_removed}, dropped: {layer_nonrelfiles_dropped}".format_map(row)) + log.info("GC duration {elapsed} ms".format_map(row)); + log.info(" REL total: {layer_relfiles_total}, needed_by_cutoff {layer_relfiles_needed_by_cutoff}, needed_by_branches: {layer_relfiles_needed_by_branches}, not_updated: {layer_relfiles_not_updated}, needed_as_tombstone {layer_relfiles_needed_as_tombstone}, removed: {layer_relfiles_removed}, dropped: {layer_relfiles_dropped}".format_map(row)) + log.info(" NONREL total: {layer_nonrelfiles_total}, needed_by_cutoff {layer_nonrelfiles_needed_by_cutoff}, needed_by_branches: {layer_nonrelfiles_needed_by_branches}, not_updated: {layer_nonrelfiles_not_updated}, needed_as_tombstone {layer_nonrelfiles_needed_as_tombstone}, removed: {layer_nonrelfiles_removed}, dropped: {layer_nonrelfiles_dropped}".format_map(row)) # @@ -35,7 +36,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): cur.execute("select relfilenode from pg_class where oid = 'foo'::regclass"); row = cur.fetchone(); - print("relfilenode is {}", row[0]); + log.info(f"relfilenode is {row[0]}"); # Run GC, to clear out any garbage left behind in the catalogs by # the CREATE TABLE command. We want to have a clean slate with no garbage @@ -50,7 +51,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): # update to confuse our numbers either. cur.execute("DELETE FROM foo") - print("Running GC before test") + log.info("Running GC before test") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row); @@ -61,7 +62,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): # Insert a row and run GC. Checkpoint should freeze the layer # so that there is only the most recent image layer left for the rel, # removing the old image and delta layer. - print("Inserting one row and running GC") + log.info("Inserting one row and running GC") cur.execute("INSERT INTO foo VALUES (1)") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() @@ -73,7 +74,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): # Insert two more rows and run GC. # This should create new image and delta layer file with the new contents, and # then remove the old one image and the just-created delta layer. - print("Inserting two more rows and running GC") + log.info("Inserting two more rows and running GC") cur.execute("INSERT INTO foo VALUES (2)") cur.execute("INSERT INTO foo VALUES (3)") @@ -85,7 +86,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): assert row['layer_relfiles_dropped'] == 0 # Do it again. Should again create two new layer files and remove old ones. - print("Inserting two more rows and running GC") + log.info("Inserting two more rows and running GC") cur.execute("INSERT INTO foo VALUES (2)") cur.execute("INSERT INTO foo VALUES (3)") @@ -97,7 +98,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): assert row['layer_relfiles_dropped'] == 0 # Run GC again, with no changes in the database. Should not remove anything. - print("Run GC again, with nothing to do") + log.info("Run GC again, with nothing to do") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row); @@ -108,7 +109,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): # # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage # - print("Drop table and run GC again"); + log.info("Drop table and run GC again"); cur.execute("DROP TABLE foo") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") diff --git a/test_runner/batch_others/test_timeline_size.py b/test_runner/batch_others/test_timeline_size.py index 45b0c98d40..acc2394306 100644 --- a/test_runner/batch_others/test_timeline_size.py +++ b/test_runner/batch_others/test_timeline_size.py @@ -2,7 +2,7 @@ from contextlib import closing from uuid import UUID import psycopg2.extras from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver - +from fixtures.log_helper import log def test_timeline_size( zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin @@ -15,7 +15,7 @@ def test_timeline_size( assert res["current_logical_size"] == res["current_logical_size_non_incremental"] pgmain = postgres.create_start("test_timeline_size") - print("postgres is running on 'test_timeline_size' branch") + log.info("postgres is running on 'test_timeline_size' branch") with closing(pgmain.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_twophase.py b/test_runner/batch_others/test_twophase.py index d818f04da4..a6315fed15 100644 --- a/test_runner/batch_others/test_twophase.py +++ b/test_runner/batch_others/test_twophase.py @@ -1,7 +1,7 @@ import os from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver, PgBin - +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -13,7 +13,7 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa zenith_cli.run(["branch", "test_twophase", "empty"]) pg = postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5']) - print("postgres is running on 'test_twophase' branch") + log.info("postgres is running on 'test_twophase' branch") conn = pg.connect() cur = conn.cursor() @@ -45,7 +45,7 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa cur.execute('CHECKPOINT') twophase_files = os.listdir(pg.pg_twophase_dir_path()) - print(twophase_files) + log.info(twophase_files) assert len(twophase_files) == 4 cur.execute("COMMIT PREPARED 'insert_three'") @@ -53,7 +53,7 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa cur.execute('CHECKPOINT') twophase_files = os.listdir(pg.pg_twophase_dir_path()) - print(twophase_files) + log.info(twophase_files) assert len(twophase_files) == 2 # Create a branch with the transaction in prepared state @@ -67,7 +67,7 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa # Check that we restored only needed twophase files twophase_files2 = os.listdir(pg2.pg_twophase_dir_path()) - print(twophase_files2) + log.info(twophase_files2) assert twophase_files2.sort() == twophase_files.sort() conn2 = pg2.connect() diff --git a/test_runner/batch_others/test_vm_bits.py b/test_runner/batch_others/test_vm_bits.py index 92509fcbbb..86c56ddb9c 100644 --- a/test_runner/batch_others/test_vm_bits.py +++ b/test_runner/batch_others/test_vm_bits.py @@ -1,4 +1,5 @@ from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -11,7 +12,7 @@ def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, p zenith_cli.run(["branch", "test_vm_bit_clear", "empty"]) pg = postgres.create_start('test_vm_bit_clear') - print("postgres is running on 'test_vm_bit_clear' branch") + log.info("postgres is running on 'test_vm_bit_clear' branch") pg_conn = pg.connect() cur = pg_conn.cursor() @@ -63,7 +64,7 @@ def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, p # server at the right point-in-time avoids that full-page image. pg_new = postgres.create_start('test_vm_bit_clear_new') - print("postgres is running on 'test_vm_bit_clear_new' branch") + log.info("postgres is running on 'test_vm_bit_clear_new' branch") pg_new_conn = pg_new.connect() cur_new = pg_new_conn.cursor() diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index b5577f28d0..6f5e877714 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -9,6 +9,7 @@ from contextlib import closing from multiprocessing import Process, Value from fixtures.zenith_fixtures import WalAcceptorFactory, ZenithPageserver, PostgresFactory, PgBin from fixtures.utils import lsn_to_hex, mkdir_if_needed +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -284,10 +285,10 @@ def test_sync_safekeepers(repo_dir: str, pg_bin: PgBin, wa_factory: WalAcceptorF ) lsn_hex = lsn_to_hex(res["inserted_wal"]["end_lsn"]) lsn_after_append.append(lsn_hex) - print(f"safekeeper[{i}] lsn after append: {lsn_hex}") + log.info(f"safekeeper[{i}] lsn after append: {lsn_hex}") # run sync safekeepers lsn_after_sync = pg.sync_safekeepers() - print(f"lsn after sync = {lsn_after_sync}") + log.info(f"lsn after sync = {lsn_after_sync}") assert all(lsn_after_sync == lsn for lsn in lsn_after_append) diff --git a/test_runner/batch_others/test_wal_acceptor_async.py b/test_runner/batch_others/test_wal_acceptor_async.py index b1647a8544..b2faa9b985 100644 --- a/test_runner/batch_others/test_wal_acceptor_async.py +++ b/test_runner/batch_others/test_wal_acceptor_async.py @@ -3,9 +3,10 @@ import asyncpg import random from fixtures.zenith_fixtures import WalAcceptor, WalAcceptorFactory, ZenithPageserver, PostgresFactory, Postgres +from fixtures.log_helper import getLogger from typing import List -from fixtures.utils import debug_print +log = getLogger('root.wal_acceptor_async') pytest_plugins = ("fixtures.zenith_fixtures") @@ -63,18 +64,18 @@ class WorkerStats(object): self.counters[worker_id] += 1 def check_progress(self): - debug_print("Workers progress: {}".format(self.counters)) + log.debug("Workers progress: {}".format(self.counters)) # every worker should finish at least one tx assert all(cnt > 0 for cnt in self.counters) progress = sum(self.counters) - print('All workers made {} transactions'.format(progress)) + log.info('All workers made {} transactions'.format(progress)) async def run_random_worker(stats: WorkerStats, pg: Postgres, worker_id, n_accounts, max_transfer): pg_conn = await pg.connect_async() - debug_print('Started worker {}'.format(worker_id)) + log.debug('Started worker {}'.format(worker_id)) while stats.running: from_uid = random.randint(0, n_accounts - 1) @@ -84,9 +85,9 @@ async def run_random_worker(stats: WorkerStats, pg: Postgres, worker_id, n_accou await bank_transfer(pg_conn, from_uid, to_uid, amount) stats.inc_progress(worker_id) - debug_print('Executed transfer({}) {} => {}'.format(amount, from_uid, to_uid)) + log.debug('Executed transfer({}) {} => {}'.format(amount, from_uid, to_uid)) - debug_print('Finished worker {}'.format(worker_id)) + log.debug('Finished worker {}'.format(worker_id)) await pg_conn.close() @@ -134,7 +135,7 @@ async def run_restarts_under_load(pg: Postgres, acceptors: List[WalAcceptor], n_ victim.start() - print('Iterations are finished, exiting coroutines...') + log.info('Iterations are finished, exiting coroutines...') stats.running = False # await all workers await asyncio.gather(*workers) diff --git a/test_runner/batch_pg_regress/test_zenith_regress.py b/test_runner/batch_pg_regress/test_zenith_regress.py index 09f5f83933..ab43f511ef 100644 --- a/test_runner/batch_pg_regress/test_zenith_regress.py +++ b/test_runner/batch_pg_regress/test_zenith_regress.py @@ -2,6 +2,7 @@ import os from fixtures.utils import mkdir_if_needed from fixtures.zenith_fixtures import PageserverPort, PostgresFactory, check_restored_datadir_content +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") @@ -38,7 +39,7 @@ def test_zenith_regress(postgres: PostgresFactory, pg_bin, zenith_cli, test_outp '--inputdir={}'.format(src_path), ] - print(pg_regress_command) + log.info(pg_regress_command) env = { 'PGPORT': str(pg.port), 'PGUSER': pg.username, diff --git a/test_runner/fixtures/benchmark_fixture.py b/test_runner/fixtures/benchmark_fixture.py index 328ebcc1f8..c70ee36256 100644 --- a/test_runner/fixtures/benchmark_fixture.py +++ b/test_runner/fixtures/benchmark_fixture.py @@ -1,5 +1,3 @@ -from pprint import pprint - import os import re import timeit diff --git a/test_runner/fixtures/log_helper.py b/test_runner/fixtures/log_helper.py new file mode 100644 index 0000000000..f253576e7b --- /dev/null +++ b/test_runner/fixtures/log_helper.py @@ -0,0 +1,44 @@ +import logging +import logging.config + +""" +This file configures logging to use in python tests. +Logs are automatically captured and shown in their +own section after all tests are executed. + +To see logs for all (even successful) tests, run +pytest with the following command: +- `pipenv run pytest -n8 -rA` + +Other log config can be set in pytest.ini file. +You can add `log_cli = true` to it to watch +logs in real time. + +To get more info about logging with pytest, see +https://docs.pytest.org/en/6.2.x/logging.html +""" + +# this config is only used for default log levels, +# log format is specified in pytest.ini file +LOGGING = { + "version": 1, + "loggers": { + "root": { + "level": "INFO" + }, + "root.wal_acceptor_async": { + "level": "INFO" # a lot of logs on DEBUG level + } + } +} + +def getLogger(name='root') -> logging.Logger: + """Method to get logger for tests. + + Should be used to get correctly initialized logger. """ + return logging.getLogger(name) + +# default logger for tests +log = getLogger() + +logging.config.dictConfig(LOGGING) diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index 92bd25ed24..dd90a36dbb 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -2,7 +2,7 @@ import os import subprocess from typing import Any, List - +from fixtures.log_helper import log def get_self_dir() -> str: """ Get the path to the directory where this script lives. """ @@ -39,7 +39,7 @@ def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> str: with open(stdout_filename, 'w') as stdout_f: with open(stderr_filename, 'w') as stderr_f: - print('(capturing output to "{}.stdout")'.format(base)) + log.info('(capturing output to "{}.stdout")'.format(base)) subprocess.run(cmd, **kwargs, stdout=stdout_f, stderr=stderr_f) return basepath @@ -58,14 +58,6 @@ def global_counter() -> int: _global_counter += 1 return _global_counter -def debug_print(*args, **kwargs) -> None: - """ Print to the console if TEST_DEBUG_PRINT is set in env. - - All parameters are passed to print(). - """ - if os.environ.get('TEST_DEBUG_PRINT') is not None: - print(*args, **kwargs) - def lsn_to_hex(num: int) -> str: """ Convert lsn from int to standard hex notation. """ return "{:X}/{:X}".format(num >> 32, num & 0xffffffff) diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index bf861f8b00..efdd966f02 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -27,6 +27,8 @@ from typing_extensions import Literal import requests from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture) +from fixtures.log_helper import log + """ This file contains pytest fixtures. A fixture is a test resource that can be summoned by placing its name in the test's arguments. @@ -188,13 +190,13 @@ class ZenithCli: >>> result = zenith_cli.run(...) >>> assert result.stderr == "" - >>> print(result.stdout) + >>> log.info(result.stdout) """ assert type(arguments) == list args = [self.bin_zenith] + arguments - print('Running command "{}"'.format(' '.join(args))) + log.info('Running command "{}"'.format(' '.join(args))) # Interceipt CalledProcessError and print more info try: @@ -211,7 +213,7 @@ class ZenithCli: stdout: {exc.stdout} stderr: {exc.stderr} """ - print(msg) + log.info(msg) raise Exception(msg) from exc @@ -392,7 +394,7 @@ class ZenithPageserver(PgProtocol): if immediate: cmd.append('immediate') - print(cmd) + log.info(f"Stopping pageserver with {cmd}") if self.running: self.zenith_cli.run(cmd) self.running = False @@ -424,7 +426,7 @@ class ZenithPageserver(PgProtocol): def pageserver_port(port_distributor: PortDistributor) -> PageserverPort: pg = port_distributor.get_port() http = port_distributor.get_port() - print(f"pageserver_port: pg={pg} http={http}") + log.info(f"pageserver_port: pg={pg} http={http}") return PageserverPort(pg=pg, http=http) @@ -448,7 +450,7 @@ def pageserver(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: Pageserver yield ps # After the yield comes any cleanup code we need. - print('Starting pageserver cleanup') + log.info('Starting pageserver cleanup') ps.stop(True) class PgBin: @@ -486,7 +488,7 @@ class PgBin: """ self._fixpath(command) - print('Running command "{}"'.format(' '.join(command))) + log.info('Running command "{}"'.format(' '.join(command))) env = self._build_env(env) subprocess.run(command, env=env, cwd=cwd, check=True) @@ -503,7 +505,7 @@ class PgBin: """ self._fixpath(command) - print('Running command "{}"'.format(' '.join(command))) + log.info('Running command "{}"'.format(' '.join(command))) env = self._build_env(env) return subprocess_capture(self.log_dir, command, env=env, cwd=cwd, check=True, **kwargs) @@ -571,12 +573,12 @@ class Postgres(PgProtocol): assert self.branch is not None - print(f"Starting postgres on branch {self.branch}") + log.info(f"Starting postgres on branch {self.branch}") run_result = self.zenith_cli.run(['pg', 'start', self.branch, f'--tenantid={self.tenant_id}', f'--port={self.port}']) self.running = True - print(f"stdout: {run_result.stdout}") + log.info(f"stdout: {run_result.stdout}") return self @@ -791,7 +793,7 @@ def postgres(zenith_cli: ZenithCli, initial_tenant: str, repo_dir: str, pg_bin: yield pgfactory # After the yield comes any cleanup code we need. - print('Starting postgres cleanup') + log.info('Starting postgres cleanup') pgfactory.stop_all() def read_pid(path: Path): @@ -828,7 +830,7 @@ class WalAcceptor: # Tell page server it can receive WAL from this WAL safekeeper cmd.extend(["--pageserver", f"localhost:{self.pageserver_port}"]) cmd.extend(["--recall", "1 second"]) - print('Running command "{}"'.format(' '.join(cmd))) + log.info('Running command "{}"'.format(' '.join(cmd))) env = {'PAGESERVER_AUTH_TOKEN': self.auth_token} if self.auth_token else None subprocess.run(cmd, check=True, env=env) @@ -857,10 +859,10 @@ class WalAcceptor: return pid def stop(self) -> 'WalAcceptor': - print('Stopping wal acceptor {}'.format(self.num)) + log.info('Stopping wal acceptor {}'.format(self.num)) pid = self.get_pid() if pid is None: - print("Wal acceptor {} is not running".format(self.num)) + log.info("Wal acceptor {} is not running".format(self.num)) return self try: @@ -886,10 +888,10 @@ class WalAcceptor: conn.autocommit = True with conn.cursor() as cur: request_json = json.dumps(request) - print(f"JSON_CTRL request on port {self.port.pg}: {request_json}") + log.info(f"JSON_CTRL request on port {self.port.pg}: {request_json}") cur.execute("JSON_CTRL " + request_json) all = cur.fetchall() - print(f"JSON_CTRL response: {all[0][0]}") + log.info(f"JSON_CTRL response: {all[0][0]}") return json.loads(all[0][0]) class WalAcceptorFactory: @@ -950,7 +952,7 @@ def wa_factory(zenith_binpath: str, repo_dir: str, pageserver_port: PageserverPo ) yield wafactory # After the yield comes any cleanup code we need. - print('Starting wal acceptors cleanup') + log.info('Starting wal acceptors cleanup') wafactory.stop_all() @@ -959,7 +961,7 @@ def base_dir() -> str: """ find the base directory (currently this is the git root) """ base_dir = os.path.normpath(os.path.join(get_self_dir(), '../..')) - print('\nbase_dir is', base_dir) + log.info(f'base_dir is {base_dir}') return base_dir @@ -988,7 +990,7 @@ def test_output_dir(request: Any, top_output_dir: str) -> str: test_name = 'shared' test_output_dir = os.path.join(top_output_dir, test_name) - print('test_output_dir is', test_output_dir) + log.info(f'test_output_dir is {test_output_dir}') shutil.rmtree(test_output_dir, ignore_errors=True) mkdir_if_needed(test_output_dir) return test_output_dir @@ -1030,7 +1032,7 @@ def pg_distrib_dir(base_dir: str) -> str: pg_dir = env_postgres_bin else: pg_dir = os.path.normpath(os.path.join(base_dir, DEFAULT_POSTGRES_DIR)) - print('postgres dir is', pg_dir) + log.info(f'postgres dir is {pg_dir}') if not os.path.exists(os.path.join(pg_dir, 'bin/postgres')): raise Exception('postgres not found at "{}"'.format(pg_dir)) return pg_dir @@ -1069,7 +1071,7 @@ def list_files_to_compare(pgdata_dir: str): pgdata_files.append(rel_file) pgdata_files.sort() - print(pgdata_files) + log.info(pgdata_files) return pgdata_files # pg is the existing and running compute node, that we want to compare with a basebackup @@ -1115,9 +1117,7 @@ def check_restored_datadir_content(zenith_cli: ZenithCli, test_output_dir: str, restored_dir_path, pgdata_files, shallow=False) - print('filecmp result mismatch and error lists:') - print(mismatch) - print(error) + log.info(f'filecmp result mismatch and error lists:\n\t mismatch={mismatch}\n\t error={error}') for f in mismatch: diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py index 95f1ea5e4a..1effa56ee9 100644 --- a/test_runner/performance/test_bulk_insert.py +++ b/test_runner/performance/test_bulk_insert.py @@ -1,6 +1,7 @@ import os from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") @@ -19,7 +20,7 @@ def test_bulk_insert(postgres: PostgresFactory, pageserver: ZenithPageserver, pg zenith_cli.run(["branch", "test_bulk_insert", "empty"]) pg = postgres.create_start('test_bulk_insert') - print("postgres is running on 'test_bulk_insert' branch") + log.info("postgres is running on 'test_bulk_insert' branch") # Open a connection directly to the page server that we'll use to force # flushing the layers to disk diff --git a/test_runner/performance/test_gist_build.py b/test_runner/performance/test_gist_build.py index 040adb1242..b9ef0a3d4b 100644 --- a/test_runner/performance/test_gist_build.py +++ b/test_runner/performance/test_gist_build.py @@ -1,6 +1,7 @@ import os from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") @@ -14,7 +15,7 @@ def test_gist_buffering_build(postgres: PostgresFactory, pageserver: ZenithPages zenith_cli.run(["branch", "test_gist_buffering_build", "empty"]) pg = postgres.create_start('test_gist_buffering_build') - print("postgres is running on 'test_gist_buffering_build' branch") + log.info("postgres is running on 'test_gist_buffering_build' branch") # Open a connection directly to the page server that we'll use to force # flushing the layers to disk diff --git a/test_runner/performance/test_perf_pgbench.py b/test_runner/performance/test_perf_pgbench.py index 18db78f12a..22e8f2aee3 100644 --- a/test_runner/performance/test_perf_pgbench.py +++ b/test_runner/performance/test_perf_pgbench.py @@ -1,6 +1,7 @@ import os from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") @@ -18,7 +19,7 @@ def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin zenith_cli.run(["branch", "test_pgbench_perf", "empty"]) pg = postgres.create_start('test_pgbench_perf') - print("postgres is running on 'test_pgbench_perf' branch") + log.info("postgres is running on 'test_pgbench_perf' branch") # Open a connection directly to the page server that we'll use to force # flushing the layers to disk diff --git a/test_runner/performance/test_write_amplification.py b/test_runner/performance/test_write_amplification.py index 09310c702b..8410499bd2 100644 --- a/test_runner/performance/test_write_amplification.py +++ b/test_runner/performance/test_write_amplification.py @@ -13,6 +13,7 @@ import os from contextlib import closing from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver +from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") @@ -21,7 +22,7 @@ def test_write_amplification(postgres: PostgresFactory, pageserver: ZenithPagese zenith_cli.run(["branch", "test_write_amplification", "empty"]) pg = postgres.create_start('test_write_amplification') - print("postgres is running on 'test_write_amplification' branch") + log.info("postgres is running on 'test_write_amplification' branch") # Open a connection directly to the page server that we'll use to force # flushing the layers to disk diff --git a/test_runner/pytest.ini b/test_runner/pytest.ini index 78b5304f78..e6c7013559 100644 --- a/test_runner/pytest.ini +++ b/test_runner/pytest.ini @@ -1,2 +1,4 @@ [pytest] minversion = 6.0 +log_format = %(asctime)s.%(msecs)-3d %(levelname)s [%(filename)s:%(lineno)d] %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/test_runner/test_broken.py b/test_runner/test_broken.py index da715d7387..66bfe1192c 100644 --- a/test_runner/test_broken.py +++ b/test_runner/test_broken.py @@ -1,6 +1,8 @@ import pytest import os +from fixtures.log_helper import log + pytest_plugins = ("fixtures.zenith_fixtures") """ Use this test to see what happens when tests fail. @@ -22,7 +24,7 @@ def test_broken(zenith_cli, pageserver, postgres, pg_bin): zenith_cli.run(["branch", "test_broken", "empty"]) postgres.create_start("test_broken") - print('postgres is running') + log.info('postgres is running') - print('THIS NEXT COMMAND WILL FAIL:') + log.info('THIS NEXT COMMAND WILL FAIL:') pg_bin.run('pgbench -i_am_a_broken_test'.split()) From 0e026371eca1c595a45697dc06e536d159b9f3f4 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 14 Oct 2021 14:21:23 +0300 Subject: [PATCH 29/46] Optimize WAL decoding slightly. This adds a fast-path for the common case that the record doesn't cross a page boundary. We now split off a new Bytes directly from the original input buffer in that case, instead of copying the record to a new BytesMut. Shaves about 5% of the page server's CPU time on my laptop, in the 'test_bulk_insert' test. --- pageserver/src/waldecoder.rs | 101 ++++++++++++++++++--------------- postgres_ffi/src/xlog_utils.rs | 7 ++- 2 files changed, 62 insertions(+), 46 deletions(-) diff --git a/pageserver/src/waldecoder.rs b/pageserver/src/waldecoder.rs index cb94b9248b..b1e8e3b54f 100644 --- a/pageserver/src/waldecoder.rs +++ b/pageserver/src/waldecoder.rs @@ -72,6 +72,10 @@ impl WalStreamDecoder { /// Err(WalDecodeError): an error occured while decoding, meaning the input was invalid. /// pub fn poll_decode(&mut self) -> Result, WalDecodeError> { + let recordbuf; + + // Run state machine that validates page headers, and reassembles records + // that cross page boundaries. loop { // parse and verify page boundaries as we go if self.lsn.segment_offset(pg_constants::WAL_SEGMENT_SIZE) == 0 { @@ -120,29 +124,41 @@ impl WalStreamDecoder { self.lsn += self.padlen as u64; self.padlen = 0; } else if self.contlen == 0 { - // need to have at least the xl_tot_len field + assert!(self.recordbuf.is_empty()); + // need to have at least the xl_tot_len field if self.inputbuf.remaining() < 4 { return Ok(None); } - // read xl_tot_len FIXME: assumes little-endian + // peek xl_tot_len at the beginning of the record. + // FIXME: assumes little-endian self.startlsn = self.lsn; - let xl_tot_len = self.inputbuf.get_u32_le(); + let xl_tot_len = (&self.inputbuf[0..4]).get_u32_le(); if (xl_tot_len as usize) < XLOG_SIZE_OF_XLOG_RECORD { return Err(WalDecodeError { msg: format!("invalid xl_tot_len {}", xl_tot_len), lsn: self.lsn, }); } - self.lsn += 4; - self.recordbuf.clear(); - self.recordbuf.reserve(xl_tot_len as usize); - self.recordbuf.put_u32_le(xl_tot_len); - - self.contlen = xl_tot_len - 4; - continue; + // Fast path for the common case that the whole record fits on the page. + let pageleft = self.lsn.remaining_in_block() as u32; + if self.inputbuf.remaining() >= xl_tot_len as usize && xl_tot_len <= pageleft { + // Take the record from the 'inputbuf', and validate it. + recordbuf = self.inputbuf.copy_to_bytes(xl_tot_len as usize); + self.lsn += xl_tot_len as u64; + break; + } else { + // Need to assemble the record from pieces. Remember the size of the + // record, and loop back. On next iteration, we will reach the 'else' + // branch below, and copy the part of the record that was on this page + // to 'recordbuf'. Subsequent iterations will skip page headers, and + // append the continuations from the next pages to 'recordbuf'. + self.recordbuf.reserve(xl_tot_len as usize); + self.contlen = xl_tot_len; + continue; + } } else { // we're continuing a record, possibly from previous page. let pageleft = self.lsn.remaining_in_block() as u32; @@ -159,47 +175,42 @@ impl WalStreamDecoder { self.contlen -= n as u32; if self.contlen == 0 { - let recordbuf = std::mem::replace(&mut self.recordbuf, BytesMut::new()); - - let recordbuf = recordbuf.freeze(); - let mut buf = recordbuf.clone(); - - let xlogrec = XLogRecord::from_bytes(&mut buf); - - // XLOG_SWITCH records are special. If we see one, we need to skip - // to the next WAL segment. - if xlogrec.is_xlog_switch_record() { - trace!("saw xlog switch record at {}", self.lsn); - self.padlen = - self.lsn.calc_padding(pg_constants::WAL_SEGMENT_SIZE as u64) as u32; - } else { - // Pad to an 8-byte boundary - self.padlen = self.lsn.calc_padding(8u32) as u32; - } - - let mut crc = crc32c_append(0, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]); - crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]); - if crc != xlogrec.xl_crc { - return Err(WalDecodeError { - msg: "WAL record crc mismatch".into(), - lsn: self.lsn, - }); - } - - // Always align resulting LSN on 0x8 boundary -- that is important for getPage() - // and WalReceiver integration. Since this code is used both for WalReceiver and - // initial WAL import let's force alignment right here. - let result = (self.lsn.align(), recordbuf); - return Ok(Some(result)); + // The record is now complete. + recordbuf = std::mem::replace(&mut self.recordbuf, BytesMut::new()).freeze(); + break; } continue; } } - // check record boundaries - // deal with continuation records + // We now have a record in the 'recordbuf' local variable. + let xlogrec = XLogRecord::from_slice(&recordbuf[0..XLOG_SIZE_OF_XLOG_RECORD]); - // deal with xlog_switch records + let mut crc = 0; + crc = crc32c_append(crc, &recordbuf[XLOG_RECORD_CRC_OFFS + 4..]); + crc = crc32c_append(crc, &recordbuf[0..XLOG_RECORD_CRC_OFFS]); + if crc != xlogrec.xl_crc { + return Err(WalDecodeError { + msg: "WAL record crc mismatch".into(), + lsn: self.lsn, + }); + } + + // XLOG_SWITCH records are special. If we see one, we need to skip + // to the next WAL segment. + if xlogrec.is_xlog_switch_record() { + trace!("saw xlog switch record at {}", self.lsn); + self.padlen = self.lsn.calc_padding(pg_constants::WAL_SEGMENT_SIZE as u64) as u32; + } else { + // Pad to an 8-byte boundary + self.padlen = self.lsn.calc_padding(8u32) as u32; + } + + // Always align resulting LSN on 0x8 boundary -- that is important for getPage() + // and WalReceiver integration. Since this code is used both for WalReceiver and + // initial WAL import let's force alignment right here. + let result = (self.lsn.align(), recordbuf); + Ok(Some(result)) } } diff --git a/postgres_ffi/src/xlog_utils.rs b/postgres_ffi/src/xlog_utils.rs index 5cdaec4a2a..7826630a78 100644 --- a/postgres_ffi/src/xlog_utils.rs +++ b/postgres_ffi/src/xlog_utils.rs @@ -329,7 +329,12 @@ pub fn main() { } impl XLogRecord { - pub fn from_bytes(buf: &mut Bytes) -> XLogRecord { + pub fn from_slice(buf: &[u8]) -> XLogRecord { + use zenith_utils::bin_ser::LeSer; + XLogRecord::des(buf).unwrap() + } + + pub fn from_bytes(buf: &mut B) -> XLogRecord { use zenith_utils::bin_ser::LeSer; XLogRecord::des_from(&mut buf.reader()).unwrap() } From de744a44dd63d22cff56ceb8d200897e54414a0c Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 12 Oct 2021 22:04:17 +0300 Subject: [PATCH 30/46] Add /timeline http request to safekeeper returning its status. Which is mainly generational state (terms) and useful LSNs. Also add /status basic healthcheck request which is now used in tests to determine the safekeeper is up; this fixes #726. ref #115 --- Cargo.lock | 2 + pageserver/src/http/routes.rs | 30 +------ test_runner/batch_others/test_wal_acceptor.py | 29 ++++++ test_runner/fixtures/zenith_fixtures.py | 45 ++++++++-- walkeeper/Cargo.toml | 2 + walkeeper/src/bin/safekeeper.rs | 6 +- walkeeper/src/http/mod.rs | 2 + walkeeper/src/http/routes.rs | 88 +++++++++++++++++++ walkeeper/src/lib.rs | 1 + walkeeper/src/timeline.rs | 2 +- zenith_utils/src/http/mod.rs | 1 + zenith_utils/src/http/request.rs | 33 +++++++ 12 files changed, 202 insertions(+), 39 deletions(-) create mode 100644 walkeeper/src/http/mod.rs create mode 100644 walkeeper/src/http/routes.rs create mode 100644 zenith_utils/src/http/request.rs diff --git a/Cargo.lock b/Cargo.lock index 6ea1fd5593..5f36f48966 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2326,6 +2326,7 @@ dependencies = [ "fs2", "hex", "humantime", + "hyper", "lazy_static", "log", "pageserver", @@ -2333,6 +2334,7 @@ dependencies = [ "postgres-protocol", "postgres_ffi", "regex", + "routerify", "rust-s3", "serde", "serde_json", diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 8b15e438da..cacb98ec84 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1,4 +1,3 @@ -use std::str::FromStr; use std::sync::Arc; use anyhow::Result; @@ -16,6 +15,8 @@ use zenith_utils::http::{ endpoint, error::HttpErrorBody, json::{json_request, json_response}, + request::get_request_param, + request::parse_request_param, }; use super::models::BranchCreateRequest; @@ -57,33 +58,6 @@ fn get_config(request: &Request) -> &'static PageServerConf { get_state(request).conf } -fn get_request_param<'a>( - request: &'a Request, - param_name: &str, -) -> Result<&'a str, ApiError> { - match request.param(param_name) { - Some(arg) => Ok(arg), - None => { - return Err(ApiError::BadRequest(format!( - "no {} specified in path param", - param_name - ))) - } - } -} - -fn parse_request_param( - request: &Request, - param_name: &str, -) -> Result { - match get_request_param(request, param_name)?.parse() { - Ok(v) => Ok(v), - Err(_) => Err(ApiError::BadRequest( - "failed to parse tenant id".to_string(), - )), - } -} - // healthcheck handler async fn status_handler(_: Request) -> Result, ApiError> { Ok(Response::builder() diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index 6f5e877714..263757e2e7 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -204,6 +204,7 @@ def test_race_conditions(zenith_cli, pageserver: ZenithPageserver, postgres: Pos stop_value.value = 1 proc.join() + class ProposerPostgres: """Object for running safekeepers sync with walproposer""" def __init__(self, pgdata_dir: str, pg_bin: PgBin, timeline_id: str, tenant_id: str): @@ -292,3 +293,31 @@ def test_sync_safekeepers(repo_dir: str, pg_bin: PgBin, wa_factory: WalAcceptorF log.info(f"lsn after sync = {lsn_after_sync}") assert all(lsn_after_sync == lsn for lsn in lsn_after_append) + + +def test_timeline_status(zenith_cli, pageserver, postgres, wa_factory: WalAcceptorFactory): + wa_factory.start_n_new(1) + + zenith_cli.run(["branch", "test_timeline_status", "empty"]) + pg = postgres.create_start('test_timeline_status', wal_acceptors=wa_factory.get_connstrs()) + + wa = wa_factory.instances[0] + wa_http_cli = wa.http_client() + wa_http_cli.check_status() + + # learn zenith timeline from compute + tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0] + timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0] + + # fetch something sensible from status + epoch = wa_http_cli.timeline_status(tenant_id, timeline_id).acceptor_epoch + + pg.safe_psql("create table t(i int)") + + # ensure epoch goes up after reboot + pg.stop().start() + pg.safe_psql("insert into t values(10)") + + epoch_after_reboot = wa_http_cli.timeline_status(tenant_id, + timeline_id).acceptor_epoch + assert epoch_after_reboot > epoch diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index efdd966f02..80246193fa 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -834,14 +834,21 @@ class WalAcceptor: env = {'PAGESERVER_AUTH_TOKEN': self.auth_token} if self.auth_token else None subprocess.run(cmd, check=True, env=env) - # wait for wal acceptor start by checkking that pid is readable - for _ in range(3): - pid = self.get_pid() - if pid is not None: - return self - time.sleep(0.5) - - raise RuntimeError("cannot get wal acceptor pid") + # wait for wal acceptor start by checking its status + started_at = time.time() + while True: + try: + http_cli = self.http_client() + http_cli.check_status() + except Exception as e: + elapsed = time.time() - started_at + if elapsed > 3: + raise RuntimeError( + f"timed out waiting {elapsed:.0f}s for wal acceptor start: {e}") + time.sleep(0.5) + else: + break # success + return self @property def pidfile(self) -> Path: @@ -894,6 +901,10 @@ class WalAcceptor: log.info(f"JSON_CTRL response: {all[0][0]}") return json.loads(all[0][0]) + def http_client(self): + return WalAcceptorHttpClient(port=self.port.http) + + class WalAcceptorFactory: """ An object representing multiple running wal acceptors. """ def __init__(self, zenith_binpath: Path, data_dir: Path, pageserver_port: int, port_distributor: PortDistributor): @@ -955,6 +966,24 @@ def wa_factory(zenith_binpath: str, repo_dir: str, pageserver_port: PageserverPo log.info('Starting wal acceptors cleanup') wafactory.stop_all() +@dataclass +class PageserverTimelineStatus: + acceptor_epoch: int + +class WalAcceptorHttpClient(requests.Session): + def __init__(self, port: int) -> None: + super().__init__() + self.port = port + + def check_status(self): + self.get(f"http://localhost:{self.port}/v1/status").raise_for_status() + + def timeline_status(self, tenant_id: str, timeline_id: str) -> PageserverTimelineStatus: + res = self.get(f"http://localhost:{self.port}/v1/timeline/{tenant_id}/{timeline_id}") + res.raise_for_status() + resj = res.json() + return PageserverTimelineStatus(acceptor_epoch=resj['acceptor_state']['epoch']) + @zenfixture def base_dir() -> str: diff --git a/walkeeper/Cargo.toml b/walkeeper/Cargo.toml index f31126eb05..2e2e435236 100644 --- a/walkeeper/Cargo.toml +++ b/walkeeper/Cargo.toml @@ -11,6 +11,8 @@ regex = "1.4.5" bincode = "1.3" bytes = "1.0.1" byteorder = "1.4.3" +hyper = "0.14" +routerify = "2" fs2 = "0.4.3" lazy_static = "1.4.0" serde_json = "1" diff --git a/walkeeper/src/bin/safekeeper.rs b/walkeeper/src/bin/safekeeper.rs index 0a6448ffd3..7ce8765789 100644 --- a/walkeeper/src/bin/safekeeper.rs +++ b/walkeeper/src/bin/safekeeper.rs @@ -14,6 +14,7 @@ use zenith_utils::http::endpoint; use zenith_utils::logging; use walkeeper::defaults::{DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_PG_LISTEN_ADDR}; +use walkeeper::http; use walkeeper::s3_offload; use walkeeper::wal_service; use walkeeper::WalAcceptorConf; @@ -164,11 +165,12 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<()> { let mut threads = Vec::new(); + let conf_cloned = conf.clone(); let http_endpoint_thread = thread::Builder::new() .name("http_endpoint_thread".into()) .spawn(|| { - // No authentication at all: read-only metrics only, early stage. - let router = endpoint::make_router(); + // TODO authentication + let router = http::make_router(conf_cloned); endpoint::serve_thread_main(router, http_listener).unwrap(); }) .unwrap(); diff --git a/walkeeper/src/http/mod.rs b/walkeeper/src/http/mod.rs new file mode 100644 index 0000000000..c82d1c0362 --- /dev/null +++ b/walkeeper/src/http/mod.rs @@ -0,0 +1,2 @@ +pub mod routes; +pub use routes::make_router; diff --git a/walkeeper/src/http/routes.rs b/walkeeper/src/http/routes.rs new file mode 100644 index 0000000000..8ab405508e --- /dev/null +++ b/walkeeper/src/http/routes.rs @@ -0,0 +1,88 @@ +use hyper::{Body, Request, Response, StatusCode}; +use routerify::ext::RequestExt; +use routerify::RouterBuilder; +use serde::Serialize; +use serde::Serializer; +use std::fmt::Display; +use std::sync::Arc; +use zenith_utils::lsn::Lsn; + +use crate::safekeeper::AcceptorState; +use crate::timeline::CreateControlFile; +use crate::timeline::GlobalTimelines; +use crate::WalAcceptorConf; +use zenith_utils::http::endpoint; +use zenith_utils::http::error::ApiError; +use zenith_utils::http::json::json_response; +use zenith_utils::http::request::parse_request_param; +use zenith_utils::zid::{ZTenantId, ZTimelineId}; + +/// Healthcheck handler. +async fn status_handler(_: Request) -> Result, ApiError> { + Ok(json_response(StatusCode::OK, "")?) +} + +fn get_conf(request: &Request) -> &WalAcceptorConf { + request + .data::>() + .expect("unknown state type") + .as_ref() +} + +fn display_serialize(z: &F, s: S) -> Result +where + S: Serializer, + F: Display, +{ + s.serialize_str(&format!("{}", z)) +} + +/// Info about timeline on safekeeper ready for reporting. +#[derive(Debug, Serialize)] +struct TimelineStatus { + #[serde(serialize_with = "display_serialize")] + tenant_id: ZTenantId, + #[serde(serialize_with = "display_serialize")] + timeline_id: ZTimelineId, + acceptor_state: AcceptorState, + #[serde(serialize_with = "display_serialize")] + commit_lsn: Lsn, + #[serde(serialize_with = "display_serialize")] + truncate_lsn: Lsn, +} + +/// Report info about timeline. +async fn timeline_status_handler(request: Request) -> Result, ApiError> { + let tenant_id: ZTenantId = parse_request_param(&request, "tenant_id")?; + let timeline_id: ZTimelineId = parse_request_param(&request, "timeline_id")?; + + let tli = GlobalTimelines::get( + get_conf(&request), + tenant_id, + timeline_id, + CreateControlFile::False, + ) + .map_err(ApiError::from_err)?; + let sk_state = tli.get_info(); + + let status = TimelineStatus { + tenant_id, + timeline_id, + acceptor_state: sk_state.acceptor_state, + commit_lsn: sk_state.commit_lsn, + truncate_lsn: sk_state.truncate_lsn, + }; + Ok(json_response(StatusCode::OK, status)?) +} + +/// Safekeeper http router. +pub fn make_router(conf: WalAcceptorConf) -> RouterBuilder { + let router = endpoint::make_router(); + router + .data(Arc::new(conf)) + .get("/v1/status", status_handler) + .get( + "/v1/timeline/:tenant_id/:timeline_id", + timeline_status_handler, + ) +} diff --git a/walkeeper/src/lib.rs b/walkeeper/src/lib.rs index 6c1f70efa2..4406823076 100644 --- a/walkeeper/src/lib.rs +++ b/walkeeper/src/lib.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use std::time::Duration; +pub mod http; pub mod json_ctrl; pub mod receive_wal; pub mod replication; diff --git a/walkeeper/src/timeline.rs b/walkeeper/src/timeline.rs index 5ee97a8a28..82aa6d6d36 100644 --- a/walkeeper/src/timeline.rs +++ b/walkeeper/src/timeline.rs @@ -289,7 +289,7 @@ lazy_static! { } /// A zero-sized struct used to manage access to the global timelines map. -struct GlobalTimelines; +pub struct GlobalTimelines; impl GlobalTimelines { /// Get a timeline with control file loaded from the global TIMELINES map. diff --git a/zenith_utils/src/http/mod.rs b/zenith_utils/src/http/mod.rs index b6740ad543..16b7e87721 100644 --- a/zenith_utils/src/http/mod.rs +++ b/zenith_utils/src/http/mod.rs @@ -1,3 +1,4 @@ pub mod endpoint; pub mod error; pub mod json; +pub mod request; diff --git a/zenith_utils/src/http/request.rs b/zenith_utils/src/http/request.rs new file mode 100644 index 0000000000..3bc8993c26 --- /dev/null +++ b/zenith_utils/src/http/request.rs @@ -0,0 +1,33 @@ +use std::str::FromStr; + +use super::error::ApiError; +use hyper::{Body, Request}; +use routerify::ext::RequestExt; + +pub fn get_request_param<'a>( + request: &'a Request, + param_name: &str, +) -> Result<&'a str, ApiError> { + match request.param(param_name) { + Some(arg) => Ok(arg), + None => { + return Err(ApiError::BadRequest(format!( + "no {} specified in path param", + param_name + ))) + } + } +} + +pub fn parse_request_param( + request: &Request, + param_name: &str, +) -> Result { + match get_request_param(request, param_name)?.parse() { + Ok(v) => Ok(v), + Err(_) => Err(ApiError::BadRequest(format!( + "failed to parse {}", + param_name + ))), + } +} From 100da024b60a5409ea078aaff2a9f648deb965bb Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Fri, 15 Oct 2021 00:26:38 +0300 Subject: [PATCH 31/46] expose pageserver http socket in docker --- docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 3754c18193..566e77c1a4 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -7,7 +7,7 @@ if [ "$1" = 'pageserver' ]; then pageserver --init -D /data --postgres-distrib /usr/local fi echo "Staring pageserver at 0.0.0.0:6400" - pageserver -l 0.0.0.0:6400 -D /data + pageserver -l 0.0.0.0:6400 --listen-http 0.0.0.0:9898 -D /data else "$@" fi From 4ade0bb41c2fe6301b55f72549744c38d419bd56 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 13 Oct 2021 21:34:33 +0300 Subject: [PATCH 32/46] Refactor upload/download_relish function signatures. This makes them more generic, by taking any Read / Write trait implementation, instead of operating directly on a a file. --- pageserver/src/relish_storage.rs | 14 ++++-- pageserver/src/relish_storage/local_fs.rs | 59 +++++++++++++++++------ pageserver/src/relish_storage/rust_s3.rs | 30 +++++++----- 3 files changed, 72 insertions(+), 31 deletions(-) diff --git a/pageserver/src/relish_storage.rs b/pageserver/src/relish_storage.rs index 70d75c34bf..885ca9581f 100644 --- a/pageserver/src/relish_storage.rs +++ b/pageserver/src/relish_storage.rs @@ -55,15 +55,21 @@ pub trait RelishStorage: Send + Sync { async fn list_relishes(&self) -> anyhow::Result>; - async fn download_relish( + async fn download_relish( &self, from: &Self::RelishStoragePath, - to: &Path, - ) -> anyhow::Result<()>; + // rust_s3 `get_object_stream` method requires `std::io::BufWriter` for some reason, not the async counterpart + // that forces us to consume and return the writer to satisfy the blocking operation async wrapper requirements + to: std::io::BufWriter, + ) -> anyhow::Result>; async fn delete_relish(&self, path: &Self::RelishStoragePath) -> anyhow::Result<()>; - async fn upload_relish(&self, from: &Path, to: &Self::RelishStoragePath) -> anyhow::Result<()>; + async fn upload_relish( + &self, + from: &mut tokio::io::BufReader, + to: &Self::RelishStoragePath, + ) -> anyhow::Result<()>; } fn strip_workspace_prefix<'a>( diff --git a/pageserver/src/relish_storage/local_fs.rs b/pageserver/src/relish_storage/local_fs.rs index 78ee858a5b..49d656d5a6 100644 --- a/pageserver/src/relish_storage/local_fs.rs +++ b/pageserver/src/relish_storage/local_fs.rs @@ -9,11 +9,13 @@ use std::{ future::Future, + io::Write, path::{Path, PathBuf}, pin::Pin, }; use anyhow::{bail, Context}; +use tokio::{fs, io}; use super::{strip_workspace_prefix, RelishStorage}; @@ -64,16 +66,33 @@ impl RelishStorage for LocalFs { Ok(get_all_files(&self.root).await?.into_iter().collect()) } - async fn download_relish( + async fn download_relish( &self, from: &Self::RelishStoragePath, - to: &Path, - ) -> anyhow::Result<()> { + mut to: std::io::BufWriter, + ) -> anyhow::Result> { let file_path = self.resolve_in_storage(from)?; if file_path.exists() && file_path.is_file() { - create_target_directory(to).await?; - tokio::fs::copy(file_path, to).await?; - Ok(()) + let updated_buffer = tokio::task::spawn_blocking(move || { + let mut source = std::io::BufReader::new( + std::fs::OpenOptions::new() + .read(true) + .open(&file_path) + .with_context(|| { + format!( + "Failed to open source file '{}' to use in the download", + file_path.display() + ) + })?, + ); + std::io::copy(&mut source, &mut to) + .context("Failed to download the relish file")?; + to.flush().context("Failed to flush the download buffer")?; + Ok::<_, anyhow::Error>(to) + }) + .await + .context("Failed to spawn a blocking task")??; + Ok(updated_buffer) } else { bail!( "File '{}' either does not exist or is not a file", @@ -94,18 +113,30 @@ impl RelishStorage for LocalFs { } } - async fn upload_relish(&self, from: &Path, to: &Self::RelishStoragePath) -> anyhow::Result<()> { + async fn upload_relish( + &self, + from: &mut io::BufReader, + to: &Self::RelishStoragePath, + ) -> anyhow::Result<()> { let target_file_path = self.resolve_in_storage(to)?; create_target_directory(&target_file_path).await?; + let mut destination = io::BufWriter::new( + fs::OpenOptions::new() + .write(true) + .create(true) + .open(&target_file_path) + .await + .with_context(|| { + format!( + "Failed to open target fs destination at '{}'", + target_file_path.display() + ) + })?, + ); - tokio::fs::copy(&from, &target_file_path) + io::copy_buf(from, &mut destination) .await - .with_context(|| { - format!( - "Failed to upload relish '{}' to local storage", - from.display(), - ) - })?; + .context("Failed to upload relish to local storage")?; Ok(()) } } diff --git a/pageserver/src/relish_storage/rust_s3.rs b/pageserver/src/relish_storage/rust_s3.rs index dc29752e99..d32d357b27 100644 --- a/pageserver/src/relish_storage/rust_s3.rs +++ b/pageserver/src/relish_storage/rust_s3.rs @@ -1,5 +1,6 @@ //! A wrapper around AWS S3 client library `rust_s3` to be used a relish storage. +use std::io::Write; use std::path::Path; use anyhow::Context; @@ -83,18 +84,14 @@ impl RelishStorage for RustS3 { .collect()) } - async fn download_relish( + async fn download_relish( &self, from: &Self::RelishStoragePath, - to: &Path, - ) -> anyhow::Result<()> { - let mut target_file = std::fs::OpenOptions::new() - .write(true) - .open(to) - .with_context(|| format!("Failed to open target s3 destination at {}", to.display()))?; + mut to: std::io::BufWriter, + ) -> anyhow::Result> { let code = self .bucket - .get_object_stream(from.key(), &mut target_file) + .get_object_stream(from.key(), &mut to) .await .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?; if code != 200 { @@ -103,7 +100,12 @@ impl RelishStorage for RustS3 { code )) } else { - Ok(()) + tokio::task::spawn_blocking(move || { + to.flush().context("Failed to fluch the downoad buffer")?; + Ok::<_, anyhow::Error>(to) + }) + .await + .context("Failed to joim the download buffer flush task")? } } @@ -124,12 +126,14 @@ impl RelishStorage for RustS3 { } } - async fn upload_relish(&self, from: &Path, to: &Self::RelishStoragePath) -> anyhow::Result<()> { - let mut local_file = tokio::fs::OpenOptions::new().read(true).open(from).await?; - + async fn upload_relish( + &self, + from: &mut tokio::io::BufReader, + to: &Self::RelishStoragePath, + ) -> anyhow::Result<()> { let code = self .bucket - .put_object_stream(&mut local_file, to.key()) + .put_object_stream(from, to.key()) .await .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?; if code != 200 { From 2dde20a227462a46f1d1893543c85199b13bb170 Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Thu, 14 Oct 2021 10:22:51 -0700 Subject: [PATCH 33/46] Bump MSRV to 1.55 --- .circleci/config.yml | 6 +++--- README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd3564c71c..78d49cf74b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ executors: zenith-build-executor: resource_class: xlarge docker: - - image: cimg/rust:1.52.1 + - image: cimg/rust:1.55.0 jobs: check-codestyle: @@ -110,7 +110,7 @@ jobs: # Require an exact match. While an out of date cache might speed up the build, # there's no way to clean out old packages, so the cache grows every time something # changes. - - v03-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} + - v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} # Build the rust code, including test binaries - run: @@ -128,7 +128,7 @@ jobs: - save_cache: name: Save rust cache - key: v03-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} + key: v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }} paths: - ~/.cargo/registry - ~/.cargo/git diff --git a/README.md b/README.md index 1e0f20fd45..977d015bfc 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libsec libssl-dev clang pkg-config libpq-dev ``` -[Rust] 1.52 or later is also required. +[Rust] 1.55 or later is also required. To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `tmp_install/bin` and `tmp_install/lib`, respectively. From ba557d126ba8a66fbee085333228528e930b737e Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Fri, 15 Oct 2021 13:17:30 +0300 Subject: [PATCH 34/46] React on sigint --- pageserver/src/bin/pageserver.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 4e2c02a83a..d1ec4f9323 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -552,7 +552,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> { info!("Got SIGQUIT. Terminate pageserver in immediate shutdown mode"); exit(111); } - SIGTERM => { + SIGINT | SIGTERM => { info!("Got SIGINT/SIGTERM. Terminate gracefully in fast shutdown mode"); // Terminate postgres backends postgres_backend::set_pgbackend_shutdown_requested(); @@ -577,8 +577,8 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> { info!("Pageserver shut down successfully completed"); exit(0); } - _ => { - debug!("Unknown signal."); + unknown_signal => { + debug!("Unknown signal {}", unknown_signal); } } } From b405eef3247f843fdbf5a56a9e723b15d4c66d06 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sun, 17 Oct 2021 14:54:39 +0300 Subject: [PATCH 35/46] Avoid writing the metadata file when it hasn't changed. --- pageserver/src/layered_repository.rs | 82 +++++++++++++++------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 7c992ddb0c..4da146ae34 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -1340,50 +1340,54 @@ impl LayeredTimeline { timeline_dir.sync_all()?; } - // Save the metadata, with updated 'disk_consistent_lsn', to a - // file in the timeline dir. After crash, we will restart WAL - // streaming and processing from that point. + // If we were able to advance 'disk_consistent_lsn', save it the metadata file. + // After crash, we will restart WAL streaming and processing from that point. + let old_disk_consistent_lsn = self.disk_consistent_lsn.load(); + if disk_consistent_lsn != old_disk_consistent_lsn { + assert!(disk_consistent_lsn > old_disk_consistent_lsn); - // We can only save a valid 'prev_record_lsn' value on disk if we - // flushed *all* in-memory changes to disk. We only track - // 'prev_record_lsn' in memory for the latest processed record, so we - // don't remember what the correct value that corresponds to some old - // LSN is. But if we flush everything, then the value corresponding - // current 'last_record_lsn' is correct and we can store it on disk. - let ondisk_prev_record_lsn = if disk_consistent_lsn == last_record_lsn { - Some(prev_record_lsn) - } else { - None - }; + // We can only save a valid 'prev_record_lsn' value on disk if we + // flushed *all* in-memory changes to disk. We only track + // 'prev_record_lsn' in memory for the latest processed record, so we + // don't remember what the correct value that corresponds to some old + // LSN is. But if we flush everything, then the value corresponding + // current 'last_record_lsn' is correct and we can store it on disk. + let ondisk_prev_record_lsn = if disk_consistent_lsn == last_record_lsn { + Some(prev_record_lsn) + } else { + None + }; - let ancestor_timelineid = self.ancestor_timeline.as_ref().map(|x| x.timelineid); + let ancestor_timelineid = self.ancestor_timeline.as_ref().map(|x| x.timelineid); - let metadata = TimelineMetadata { - disk_consistent_lsn, - prev_record_lsn: ondisk_prev_record_lsn, - ancestor_timeline: ancestor_timelineid, - ancestor_lsn: self.ancestor_lsn, - }; - LayeredRepository::save_metadata( - self.conf, - self.timelineid, - self.tenantid, - &metadata, - false, - )?; - if self.upload_relishes { - schedule_timeline_upload(()) - // schedule_timeline_upload( - // self.tenantid, - // self.timelineid, - // layer_uploads, - // disk_consistent_lsn, - // }); + let metadata = TimelineMetadata { + disk_consistent_lsn, + prev_record_lsn: ondisk_prev_record_lsn, + ancestor_timeline: ancestor_timelineid, + ancestor_lsn: self.ancestor_lsn, + }; + LayeredRepository::save_metadata( + self.conf, + self.timelineid, + self.tenantid, + &metadata, + false, + )?; + + // Also update the in-memory copy + self.disk_consistent_lsn.store(disk_consistent_lsn); + + if self.upload_relishes { + schedule_timeline_upload(()) + // schedule_timeline_upload( + // self.tenantid, + // self.timelineid, + // layer_uploads, + // disk_consistent_lsn, + // }); + } } - // Also update the in-memory copy - self.disk_consistent_lsn.store(disk_consistent_lsn); - Ok(()) } From bdd039a9ee7269d3e824a669fdb7946278e9217f Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sun, 17 Oct 2021 16:01:44 +0300 Subject: [PATCH 36/46] S3 DELETE call returns 204, not 200. According to the S3 API docs, the DELETE call returns code "204 No content" on success. --- pageserver/src/relish_storage/rust_s3.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pageserver/src/relish_storage/rust_s3.rs b/pageserver/src/relish_storage/rust_s3.rs index d32d357b27..5dddaa36ca 100644 --- a/pageserver/src/relish_storage/rust_s3.rs +++ b/pageserver/src/relish_storage/rust_s3.rs @@ -115,9 +115,9 @@ impl RelishStorage for RustS3 { .delete_object(path.key()) .await .with_context(|| format!("Failed to delete s3 object with key {}", path.key()))?; - if code != 200 { + if code != 204 { Err(anyhow::format_err!( - "Received non-200 exit code during deleting object with key '{}', code: {}", + "Received non-204 exit code during deleting object with key '{}', code: {}", path.key(), code )) From e9b5224a8a47cac0513032a48747b816f3d5b43a Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Fri, 15 Oct 2021 15:01:31 +0300 Subject: [PATCH 37/46] Fix toml serde gotchas --- pageserver/src/bin/pageserver.rs | 158 ++++++++++++++++++++++++++++--- 1 file changed, 146 insertions(+), 12 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index d1ec4f9323..3a577476dc 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -2,7 +2,6 @@ // Main entry point for the Page Server executable // -use pageserver::defaults::*; use serde::{Deserialize, Serialize}; use std::{ env, @@ -28,13 +27,8 @@ use clap::{App, Arg, ArgMatches}; use daemonize::Daemonize; use pageserver::{ - branches, - defaults::{ - DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_PG_LISTEN_ADDR, - DEFAULT_RELISH_STORAGE_MAX_CONCURRENT_SYNC_LIMITS, - }, - http, page_service, relish_storage, tenant_mgr, PageServerConf, RelishStorageConfig, - RelishStorageKind, S3Config, LOG_FILE_NAME, + branches, defaults::*, http, page_service, relish_storage, tenant_mgr, PageServerConf, + RelishStorageConfig, RelishStorageKind, S3Config, LOG_FILE_NAME, }; use zenith_utils::http::endpoint; use zenith_utils::postgres_backend; @@ -42,7 +36,7 @@ use zenith_utils::postgres_backend; use const_format::formatcp; /// String arguments that can be declared via CLI or config file -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)] struct CfgFileParams { listen_pg_addr: Option, listen_http_addr: Option, @@ -53,12 +47,21 @@ struct CfgFileParams { pg_distrib_dir: Option, auth_validation_public_key_path: Option, auth_type: Option, - // see https://github.com/alexcrichton/toml-rs/blob/6c162e6562c3e432bf04c82a3d1d789d80761a86/examples/enum_external.rs for enum deserialisation examples - relish_storage: Option, relish_storage_max_concurrent_sync: Option, + ///////////////////////////////// + //// Don't put `Option` and other "simple" values below. + //// + /// `Option` is a table in TOML. + /// Values in TOML cannot be defined after tables (other tables can), + /// and [`toml`] crate serializes all fields in the order of their appearance. + //////////////////////////////// + relish_storage: Option, } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)] +// Without this attribute, enums with values won't be serialized by the `toml` library (but can be deserialized nonetheless!). +// See https://github.com/alexcrichton/toml-rs/blob/6c162e6562c3e432bf04c82a3d1d789d80761a86/examples/enum_external.rs for the examples +#[serde(untagged)] enum RelishStorage { Local { local_path: String, @@ -585,3 +588,134 @@ fn start_pageserver(conf: &'static PageServerConf) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn page_server_conf_toml_serde() { + let params = CfgFileParams { + listen_pg_addr: Some("listen_pg_addr_VALUE".to_string()), + listen_http_addr: Some("listen_http_addr_VALUE".to_string()), + checkpoint_distance: Some("checkpoint_distance_VALUE".to_string()), + checkpoint_period: Some("checkpoint_period_VALUE".to_string()), + gc_horizon: Some("gc_horizon_VALUE".to_string()), + gc_period: Some("gc_period_VALUE".to_string()), + pg_distrib_dir: Some("pg_distrib_dir_VALUE".to_string()), + auth_validation_public_key_path: Some( + "auth_validation_public_key_path_VALUE".to_string(), + ), + auth_type: Some("auth_type_VALUE".to_string()), + relish_storage: Some(RelishStorage::Local { + local_path: "relish_storage_local_VALUE".to_string(), + }), + relish_storage_max_concurrent_sync: Some( + "relish_storage_max_concurrent_sync_VALUE".to_string(), + ), + }; + + let toml_string = toml::to_string(¶ms).expect("Failed to serialize correct config"); + let toml_pretty_string = + toml::to_string_pretty(¶ms).expect("Failed to serialize correct config"); + assert_eq!( + r#"listen_pg_addr = 'listen_pg_addr_VALUE' +listen_http_addr = 'listen_http_addr_VALUE' +checkpoint_distance = 'checkpoint_distance_VALUE' +checkpoint_period = 'checkpoint_period_VALUE' +gc_horizon = 'gc_horizon_VALUE' +gc_period = 'gc_period_VALUE' +pg_distrib_dir = 'pg_distrib_dir_VALUE' +auth_validation_public_key_path = 'auth_validation_public_key_path_VALUE' +auth_type = 'auth_type_VALUE' +relish_storage_max_concurrent_sync = 'relish_storage_max_concurrent_sync_VALUE' + +[relish_storage] +local_path = 'relish_storage_local_VALUE' +"#, + toml_pretty_string + ); + + let params_from_serialized: CfgFileParams = toml::from_str(&toml_string) + .expect("Failed to deserialize the serialization result of the config"); + let params_from_serialized_pretty: CfgFileParams = toml::from_str(&toml_pretty_string) + .expect("Failed to deserialize the prettified serialization result of the config"); + assert!( + params_from_serialized == params, + "Expected the same config in the end of config -> serialize -> deserialize chain" + ); + assert!( + params_from_serialized_pretty == params, + "Expected the same config in the end of config -> serialize pretty -> deserialize chain" + ); + } + + #[test] + fn credentials_omitted_during_serialization() { + let params = CfgFileParams { + listen_pg_addr: Some("listen_pg_addr_VALUE".to_string()), + listen_http_addr: Some("listen_http_addr_VALUE".to_string()), + checkpoint_distance: Some("checkpoint_distance_VALUE".to_string()), + checkpoint_period: Some("checkpoint_period_VALUE".to_string()), + gc_horizon: Some("gc_horizon_VALUE".to_string()), + gc_period: Some("gc_period_VALUE".to_string()), + pg_distrib_dir: Some("pg_distrib_dir_VALUE".to_string()), + auth_validation_public_key_path: Some( + "auth_validation_public_key_path_VALUE".to_string(), + ), + auth_type: Some("auth_type_VALUE".to_string()), + relish_storage: Some(RelishStorage::AwsS3 { + bucket_name: "bucket_name_VALUE".to_string(), + bucket_region: "bucket_region_VALUE".to_string(), + access_key_id: Some("access_key_id_VALUE".to_string()), + secret_access_key: Some("secret_access_key_VALUE".to_string()), + }), + relish_storage_max_concurrent_sync: Some( + "relish_storage_max_concurrent_sync_VALUE".to_string(), + ), + }; + + let toml_string = toml::to_string(¶ms).expect("Failed to serialize correct config"); + let toml_pretty_string = + toml::to_string_pretty(¶ms).expect("Failed to serialize correct config"); + assert_eq!( + r#"listen_pg_addr = 'listen_pg_addr_VALUE' +listen_http_addr = 'listen_http_addr_VALUE' +checkpoint_distance = 'checkpoint_distance_VALUE' +checkpoint_period = 'checkpoint_period_VALUE' +gc_horizon = 'gc_horizon_VALUE' +gc_period = 'gc_period_VALUE' +pg_distrib_dir = 'pg_distrib_dir_VALUE' +auth_validation_public_key_path = 'auth_validation_public_key_path_VALUE' +auth_type = 'auth_type_VALUE' +relish_storage_max_concurrent_sync = 'relish_storage_max_concurrent_sync_VALUE' + +[relish_storage] +bucket_name = 'bucket_name_VALUE' +bucket_region = 'bucket_region_VALUE' +"#, + toml_pretty_string + ); + + let params_from_serialized: CfgFileParams = toml::from_str(&toml_string) + .expect("Failed to deserialize the serialization result of the config"); + let params_from_serialized_pretty: CfgFileParams = toml::from_str(&toml_pretty_string) + .expect("Failed to deserialize the prettified serialization result of the config"); + + let mut expected_params = params; + expected_params.relish_storage = Some(RelishStorage::AwsS3 { + bucket_name: "bucket_name_VALUE".to_string(), + bucket_region: "bucket_region_VALUE".to_string(), + access_key_id: None, + secret_access_key: None, + }); + assert!( + params_from_serialized == expected_params, + "Expected the config without credentials in the end of a 'config -> serialize -> deserialize' chain" + ); + assert!( + params_from_serialized_pretty == expected_params, + "Expected the config without credentials in the end of a 'config -> serialize pretty -> deserialize' chain" + ); + } +} From a1bc0ada59afcef11363e49a9281b6209c4315f0 Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Mon, 18 Oct 2021 14:56:30 +0300 Subject: [PATCH 38/46] Dockerfile: remove wal_acceptor alias for safekeeper (#743) --- Dockerfile | 2 -- Dockerfile.alpine | 2 -- 2 files changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0bbf03d13c..528f29597f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,8 +38,6 @@ RUN apt-get update && apt-get -yq install libreadline-dev libseccomp-dev openssl COPY --from=build /zenith/target/release/pageserver /usr/local/bin COPY --from=build /zenith/target/release/safekeeper /usr/local/bin -# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 -RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install postgres_install COPY docker-entrypoint.sh /docker-entrypoint.sh diff --git a/Dockerfile.alpine b/Dockerfile.alpine index e0b569de22..dafb7eaf6b 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -82,8 +82,6 @@ RUN apk add --update openssl build-base libseccomp-dev RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb COPY --from=build /zenith/target/release/pageserver /usr/local/bin COPY --from=build /zenith/target/release/safekeeper /usr/local/bin -# TODO: temporary alias for compatibility, see https://github.com/zenithdb/zenith/pull/740 -RUN ln -s /usr/local/bin/safekeeper /usr/local/bin/wal_acceptor COPY --from=build /zenith/target/release/proxy /usr/local/bin COPY --from=pg-build /zenith/tmp_install /usr/local COPY docker-entrypoint.sh /docker-entrypoint.sh From 0dc7a3fc1595fae0eb05d344d1c4409886acdea4 Mon Sep 17 00:00:00 2001 From: anastasia Date: Mon, 18 Oct 2021 13:27:25 +0300 Subject: [PATCH 39/46] Change tenant_mgr to use TenantState. It allows to avoid locking entire TENANTS list while one tenant is bootstrapping and prepares the code for remote storage integration. --- pageserver/src/tenant_mgr.rs | 135 ++++++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 34 deletions(-) diff --git a/pageserver/src/tenant_mgr.rs b/pageserver/src/tenant_mgr.rs index 1712cf1b8a..be3a36fda4 100644 --- a/pageserver/src/tenant_mgr.rs +++ b/pageserver/src/tenant_mgr.rs @@ -9,8 +9,8 @@ use crate::PageServerConf; use anyhow::{anyhow, bail, Context, Result}; use lazy_static::lazy_static; use log::{debug, info}; -use std::collections::hash_map::Entry; use std::collections::HashMap; +use std::fmt; use std::fs; use std::str::FromStr; use std::sync::atomic::{AtomicBool, Ordering}; @@ -19,13 +19,47 @@ use std::thread::JoinHandle; use zenith_utils::zid::{ZTenantId, ZTimelineId}; lazy_static! { - static ref REPOSITORY: Mutex>> = - Mutex::new(HashMap::new()); + static ref TENANTS: Mutex> = Mutex::new(HashMap::new()); } -fn access_repository() -> MutexGuard<'static, HashMap>> { - REPOSITORY.lock().unwrap() +struct Tenant { + state: TenantState, + repo: Option>, } + +#[derive(Debug)] +enum TenantState { + // This tenant only exists in cloud storage. It cannot be accessed. + CloudOnly, + // This tenant exists in cloud storage, and we are currently downloading it to local disk. + // It cannot be accessed yet, not until it's been fully downloaded to local disk. + Downloading, + // All data for this tenant is complete on local disk, but we haven't loaded the Repository, + // Timeline and Layer structs into memory yet, so it cannot be accessed yet. + //Ready, + // This tenant exists on local disk, and the layer map has been loaded into memory. + // The local disk might have some newer files that don't exist in cloud storage yet. + Active, + // This tenant exists on local disk, and the layer map has been loaded into memory. + // The local disk might have some newer files that don't exist in cloud storage yet. + // The tenant cannot be accessed anymore for any reason, but graceful shutdown. + //Stopping, +} + +impl fmt::Display for TenantState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TenantState::CloudOnly => f.write_str("CloudOnly"), + TenantState::Downloading => f.write_str("Downloading"), + TenantState::Active => f.write_str("Active"), + } + } +} + +fn access_tenants() -> MutexGuard<'static, HashMap> { + TENANTS.lock().unwrap() +} + struct TenantHandleEntry { checkpointer_handle: Option>, gc_handle: Option>, @@ -41,17 +75,25 @@ lazy_static! { static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false); pub fn init(conf: &'static PageServerConf) { - let mut m = access_repository(); for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() { let tenantid = ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap(); - let repo = init_repo(conf, tenantid); + + { + let mut m = access_tenants(); + let tenant = Tenant { + state: TenantState::CloudOnly, + repo: None, + }; + m.insert(tenantid, tenant); + } + + init_repo(conf, tenantid); info!("initialized storage for tenant: {}", &tenantid); - m.insert(tenantid, repo); } } -fn init_repo(conf: &'static PageServerConf, tenant_id: ZTenantId) -> Arc { +fn init_repo(conf: &'static PageServerConf, tenant_id: ZTenantId) { // Set up a WAL redo manager, for applying WAL records. let walredo_mgr = PostgresRedoManager::new(conf, tenant_id); @@ -74,7 +116,10 @@ fn init_repo(conf: &'static PageServerConf, tenant_id: ZTenantId) -> Arc init_timeline(o.get().as_ref(), timeline_id), - Entry::Vacant(v) => { - log::info!("New repo initialized"); - let new_repo = init_repo(conf, tenant_id); - init_timeline(new_repo.as_ref(), timeline_id); - v.insert(new_repo); + + { + let mut m = access_tenants(); + let mut tenant = m.get_mut(&tenant_id).unwrap(); + tenant.state = TenantState::Downloading; + match &tenant.repo { + Some(repo) => init_timeline(repo.as_ref(), timeline_id), + None => { + log::info!("Initialize new repo"); + } } } + + // init repo updates Tenant state + init_repo(conf, tenant_id); + let new_repo = get_repository_for_tenant(tenant_id).unwrap(); + init_timeline(new_repo.as_ref(), timeline_id); } fn init_timeline(repo: &dyn Repository, timeline_id: ZTimelineId) { @@ -125,8 +178,8 @@ pub fn stop_tenant_threads(tenantid: ZTenantId) { pub fn shutdown_all_tenants() -> Result<()> { SHUTDOWN_REQUESTED.swap(true, Ordering::Relaxed); - let tenants = list_tenants()?; - for tenantid in tenants { + let tenantids = list_tenantids()?; + for tenantid in tenantids { stop_tenant_threads(tenantid); let repo = get_repository_for_tenant(tenantid)?; debug!("shutdown tenant {}", tenantid); @@ -140,25 +193,40 @@ pub fn create_repository_for_tenant( conf: &'static PageServerConf, tenantid: ZTenantId, ) -> Result<()> { - let mut m = access_repository(); - - // First check that the tenant doesn't exist already - if m.get(&tenantid).is_some() { - bail!("tenant {} already exists", tenantid); + { + let mut m = access_tenants(); + // First check that the tenant doesn't exist already + if m.get(&tenantid).is_some() { + bail!("tenant {} already exists", tenantid); + } + let tenant = Tenant { + state: TenantState::CloudOnly, + repo: None, + }; + m.insert(tenantid, tenant); } + let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid)); let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?; - m.insert(tenantid, repo); + let mut m = access_tenants(); + let tenant = m.get_mut(&tenantid).unwrap(); + tenant.repo = Some(repo); + tenant.state = TenantState::Active; Ok(()) } pub fn get_repository_for_tenant(tenantid: ZTenantId) -> Result> { - access_repository() + let m = access_tenants(); + let tenant = m .get(&tenantid) - .map(Arc::clone) - .ok_or_else(|| anyhow!("repository not found for tenant name {}", tenantid)) + .ok_or_else(|| anyhow!("Tenant not found for tenant {}", tenantid)); + + match &tenant.unwrap().repo { + Some(repo) => Ok(Arc::clone(repo)), + None => anyhow::bail!("Repository for tenant {} is not yet valid", tenantid), + } } pub fn get_timeline_for_tenant( @@ -170,12 +238,11 @@ pub fn get_timeline_for_tenant( .with_context(|| format!("cannot fetch timeline {}", timelineid)) } -fn list_tenants() -> Result> { - let o = &mut REPOSITORY.lock().unwrap(); - - o.iter() - .map(|tenant| { - let (tenantid, _) = tenant; +fn list_tenantids() -> Result> { + let m = access_tenants(); + m.iter() + .map(|v| { + let (tenantid, _) = v; Ok(*tenantid) }) .collect() From e272a380b467ff69f1f0a90ef3e37e63739febb5 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 19 Oct 2021 09:48:04 +0300 Subject: [PATCH 40/46] On new repo, start writing WAL only after the initial checkpoint record. Previously, the first WAL record on the 'main' branch overwrote the initial checkpoint record, with invalid 'xl_prev'. That's harmless, but also pretty ugly. I bumped into this while I was trying to tighen up the checks for when a valid 'prev_lsn' is required. With this patch, the first WAL record gets a valid 'xl_prev' value. It doesn't matter much currently, but let's be tidy. --- pageserver/src/restore_local_repo.rs | 128 +++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/pageserver/src/restore_local_repo.rs b/pageserver/src/restore_local_repo.rs index 60eb9ce278..8afa2676e2 100644 --- a/pageserver/src/restore_local_repo.rs +++ b/pageserver/src/restore_local_repo.rs @@ -7,10 +7,10 @@ use postgres_ffi::nonrelfile_utils::slru_may_delete_clogsegment; use std::cmp::min; use std::fs; use std::fs::File; -use std::io::Read; -use std::path::Path; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; -use anyhow::{bail, Result}; +use anyhow::{anyhow, bail, Result}; use bytes::{Buf, Bytes}; use tracing::*; @@ -37,6 +37,8 @@ pub fn import_timeline_from_postgres_datadir( writer: &dyn TimelineWriter, lsn: Lsn, ) -> Result<()> { + let mut pg_control: Option = None; + // Scan 'global' for direntry in fs::read_dir(path.join("global"))? { let direntry = direntry?; @@ -44,7 +46,7 @@ pub fn import_timeline_from_postgres_datadir( None => continue, Some("pg_control") => { - import_control_file(writer, lsn, &direntry.path())?; + pg_control = Some(import_control_file(writer, lsn, &direntry.path())?); } Some("pg_filenode.map") => import_nonrel_file( writer, @@ -127,6 +129,18 @@ pub fn import_timeline_from_postgres_datadir( writer.advance_last_record_lsn(lsn); + // Import WAL. This is needed even when starting from a shutdown checkpoint, because + // this reads the checkpoint record itself, advancing the tip of the timeline to + // *after* the checkpoint record. And crucially, it initializes the 'prev_lsn' + let pg_control = pg_control.ok_or_else(|| anyhow!("pg_control file not found"))?; + import_wal( + &path.join("pg_wal"), + writer, + Lsn(pg_control.checkPointCopy.redo), + lsn, + &mut pg_control.checkPointCopy.clone(), + )?; + Ok(()) } @@ -212,7 +226,11 @@ fn import_nonrel_file( /// /// The control file is imported as is, but we also extract the checkpoint record /// from it and store it separated. -fn import_control_file(timeline: &dyn TimelineWriter, lsn: Lsn, path: &Path) -> Result<()> { +fn import_control_file( + timeline: &dyn TimelineWriter, + lsn: Lsn, + path: &Path, +) -> Result { let mut file = File::open(path)?; let mut buffer = Vec::new(); // read the whole file @@ -233,7 +251,7 @@ fn import_control_file(timeline: &dyn TimelineWriter, lsn: Lsn, path: &Path) -> let checkpoint_bytes = pg_control.checkPointCopy.encode(); timeline.put_page_image(RelishTag::Checkpoint, 0, lsn, checkpoint_bytes)?; - Ok(()) + Ok(pg_control) } /// @@ -285,6 +303,104 @@ fn import_slru_file( Ok(()) } +/// Scan PostgreSQL WAL files in given directory and load all records between +/// 'startpoint' and 'endpoint' into the repository. +fn import_wal( + walpath: &Path, + timeline: &dyn TimelineWriter, + startpoint: Lsn, + endpoint: Lsn, + checkpoint: &mut CheckPoint, +) -> Result<()> { + let mut waldecoder = WalStreamDecoder::new(startpoint); + + let mut segno = startpoint.segment_number(pg_constants::WAL_SEGMENT_SIZE); + let mut offset = startpoint.segment_offset(pg_constants::WAL_SEGMENT_SIZE); + let mut last_lsn = startpoint; + + while last_lsn <= endpoint { + // FIXME: assume postgresql tli 1 for now + let filename = XLogFileName(1, segno, pg_constants::WAL_SEGMENT_SIZE); + let mut buf = Vec::new(); + + // Read local file + let mut path = walpath.join(&filename); + + // It could be as .partial + if !PathBuf::from(&path).exists() { + path = walpath.join(filename + ".partial"); + } + + // Slurp the WAL file + let mut file = File::open(&path)?; + + if offset > 0 { + file.seek(SeekFrom::Start(offset as u64))?; + } + + let nread = file.read_to_end(&mut buf)?; + if nread != pg_constants::WAL_SEGMENT_SIZE - offset as usize { + // Maybe allow this for .partial files? + error!("read only {} bytes from WAL file", nread); + } + + waldecoder.feed_bytes(&buf); + + let mut nrecords = 0; + while last_lsn <= endpoint { + if let Some((lsn, recdata)) = waldecoder.poll_decode()? { + let mut checkpoint_modified = false; + + let decoded = decode_wal_record(recdata.clone()); + save_decoded_record( + checkpoint, + &mut checkpoint_modified, + timeline, + &decoded, + recdata, + lsn, + )?; + last_lsn = lsn; + + if checkpoint_modified { + let checkpoint_bytes = checkpoint.encode(); + timeline.put_page_image( + RelishTag::Checkpoint, + 0, + last_lsn, + checkpoint_bytes, + )?; + } + + // Now that this record has been fully handled, including updating the + // checkpoint data, let the repository know that it is up-to-date to this LSN + timeline.advance_last_record_lsn(last_lsn); + nrecords += 1; + + trace!("imported record at {} (end {})", lsn, endpoint); + } + } + + debug!("imported {} records up to {}", nrecords, last_lsn); + + segno += 1; + offset = 0; + } + + if last_lsn != startpoint { + debug!( + "reached end of WAL at {}, updating checkpoint info", + last_lsn + ); + + timeline.advance_last_record_lsn(last_lsn); + } else { + info!("no WAL to import at {}", last_lsn); + } + + Ok(()) +} + /// /// Helper function to parse a WAL record and call the Timeline's PUT functions for all the /// relations/pages that the record affects. From c2b468c9583c67d7f653967f052db9f7c558f44b Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 19 Oct 2021 09:48:10 +0300 Subject: [PATCH 41/46] Separate node name from the branch name in ComputeControlPlane This is in preparation for supporting read-only nodes. You can launch multiple read-only nodes on the same brach, so we need an identifier for each node, separate from the branch name. --- control_plane/src/compute.rs | 4 +- test_runner/batch_others/test_tenants.py | 2 + test_runner/fixtures/zenith_fixtures.py | 51 +++++++++++-------- .../performance/test_bulk_tenant_create.py | 1 + zenith/src/main.rs | 12 +++-- 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index f33d77720c..9a262a075a 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -87,9 +87,11 @@ impl ComputeControlPlane { pub fn new_node( &mut self, tenantid: ZTenantId, + name: &str, branch_name: &str, port: Option, ) -> Result> { + // Resolve the timeline ID, given the human-readable branch name let timeline_id = self .pageserver .branch_get_by_name(&tenantid, branch_name)? @@ -97,7 +99,7 @@ impl ComputeControlPlane { let port = port.unwrap_or_else(|| self.get_port()); let node = Arc::new(PostgresNode { - name: branch_name.to_owned(), + name: name.to_owned(), address: SocketAddr::new("127.0.0.1".parse().unwrap(), port), env: self.env.clone(), pageserver: Arc::clone(&self.pageserver), diff --git a/test_runner/batch_others/test_tenants.py b/test_runner/batch_others/test_tenants.py index ee6bb0bfd3..b05a4a9f62 100644 --- a/test_runner/batch_others/test_tenants.py +++ b/test_runner/batch_others/test_tenants.py @@ -28,11 +28,13 @@ def test_tenants_normal_work( pg_tenant1 = postgres.create_start( f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", + None, # branch name, None means same as node name tenant_1, wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None, ) pg_tenant2 = postgres.create_start( f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", + None, # branch name, None means same as node name tenant_2, wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None, ) diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 80246193fa..eda04a8da6 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -530,15 +530,16 @@ class Postgres(PgProtocol): self.zenith_cli = zenith_cli self.running = False self.repo_dir = repo_dir - self.branch: Optional[str] = None # dubious, see asserts below + self.node_name: Optional[str] = None # dubious, see asserts below self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA self.tenant_id = tenant_id self.pg_bin = pg_bin - # path to conf is /pgdatadirs/tenants///postgresql.conf + # path to conf is /pgdatadirs/tenants///postgresql.conf def create( self, - branch: str, + node_name: str, + branch: Optional[str] = None, wal_acceptors: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': @@ -552,9 +553,12 @@ class Postgres(PgProtocol): if not config_lines: config_lines = [] - self.zenith_cli.run(['pg', 'create', branch, f'--tenantid={self.tenant_id}', f'--port={self.port}']) - self.branch = branch - path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch + if branch is None: + branch = node_name + + self.zenith_cli.run(['pg', 'create', f'--tenantid={self.tenant_id}', f'--port={self.port}', node_name, branch]) + self.node_name = node_name + path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.node_name self.pgdata_dir = os.path.join(self.repo_dir, path) if wal_acceptors is not None: @@ -571,11 +575,11 @@ class Postgres(PgProtocol): Returns self. """ - assert self.branch is not None + assert self.node_name is not None - log.info(f"Starting postgres on branch {self.branch}") + log.info(f"Starting postgres node {self.node_name}") - run_result = self.zenith_cli.run(['pg', 'start', self.branch, f'--tenantid={self.tenant_id}', f'--port={self.port}']) + run_result = self.zenith_cli.run(['pg', 'start', f'--tenantid={self.tenant_id}', f'--port={self.port}', self.node_name]) self.running = True log.info(f"stdout: {run_result.stdout}") @@ -584,7 +588,7 @@ class Postgres(PgProtocol): def pg_data_dir_path(self) -> str: """ Path to data directory """ - path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch + path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.node_name return os.path.join(self.repo_dir, path) def pg_xact_dir_path(self) -> str: @@ -641,8 +645,8 @@ class Postgres(PgProtocol): """ if self.running: - assert self.branch is not None - self.zenith_cli.run(['pg', 'stop', self.branch, f'--tenantid={self.tenant_id}']) + assert self.node_name is not None + self.zenith_cli.run(['pg', 'stop', self.node_name, f'--tenantid={self.tenant_id}']) self.running = False return self @@ -653,15 +657,16 @@ class Postgres(PgProtocol): Returns self. """ - assert self.branch is not None + assert self.node_name is not None assert self.tenant_id is not None - self.zenith_cli.run(['pg', 'stop', '--destroy', self.branch, f'--tenantid={self.tenant_id}']) + self.zenith_cli.run(['pg', 'stop', '--destroy', self.node_name, f'--tenantid={self.tenant_id}']) return self def create_start( self, - branch: str, + node_name: str, + branch: Optional[str] = None, wal_acceptors: Optional[str] = None, config_lines: Optional[List[str]] = None, ) -> 'Postgres': @@ -672,6 +677,7 @@ class Postgres(PgProtocol): """ self.create( + node_name=node_name, branch=branch, wal_acceptors=wal_acceptors, config_lines=config_lines, @@ -698,11 +704,13 @@ class PostgresFactory: def create_start( self, - branch: str = "main", + node_name: str = "main", + branch: Optional[str] = None, tenant_id: Optional[str] = None, wal_acceptors: Optional[str] = None, config_lines: Optional[List[str]] = None ) -> Postgres: + pg = Postgres( zenith_cli=self.zenith_cli, repo_dir=self.repo_dir, @@ -714,6 +722,7 @@ class PostgresFactory: self.instances.append(pg) return pg.create_start( + node_name=node_name, branch=branch, wal_acceptors=wal_acceptors, config_lines=config_lines, @@ -721,7 +730,8 @@ class PostgresFactory: def create( self, - branch: str = "main", + node_name: str = "main", + branch: Optional[str] = None, tenant_id: Optional[str] = None, wal_acceptors: Optional[str] = None, config_lines: Optional[List[str]] = None @@ -739,6 +749,7 @@ class PostgresFactory: self.instances.append(pg) return pg.create( + node_name=node_name, branch=branch, wal_acceptors=wal_acceptors, config_lines=config_lines, @@ -746,7 +757,7 @@ class PostgresFactory: def config( self, - branch: str = "main", + node_name: str = "main", tenant_id: Optional[str] = None, wal_acceptors: Optional[str] = None, config_lines: Optional[List[str]] = None @@ -764,7 +775,7 @@ class PostgresFactory: self.instances.append(pg) return pg.config( - branch=branch, + node_name=node_name, wal_acceptors=wal_acceptors, config_lines=config_lines, ) @@ -1116,7 +1127,7 @@ def check_restored_datadir_content(zenith_cli: ZenithCli, test_output_dir: str, pg.stop() # Take a basebackup from pageserver - restored_dir_path = os.path.join(test_output_dir, f"{pg.branch}_restored_datadir") + restored_dir_path = os.path.join(test_output_dir, f"{pg.node_name}_restored_datadir") mkdir_if_needed(restored_dir_path) psql_path = os.path.join(pg.pg_bin.pg_bin_path, 'psql') diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index e1de1dd014..3612189544 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -46,6 +46,7 @@ def test_bulk_tenant_create( pg_tenant = postgres.create_start( f"test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}", + None, # branch name, None means same as node name tenant, wal_acceptors=wa_factory.get_connstrs() if use_wal_acceptors == 'with_wa' else None, ) diff --git a/zenith/src/main.rs b/zenith/src/main.rs index e86ce10041..1692695767 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -457,26 +457,28 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let tenantid: ZTenantId = create_match .value_of("tenantid") .map_or(Ok(env.tenantid), |value| value.parse())?; - let timeline_name = create_match.value_of("timeline").unwrap_or("main"); + let node_name = start_match.value_of("node").unwrap_or("main"); + let timeline_name = start_match.value_of("timeline"); let port: Option = match create_match.value_of("port") { Some(p) => Some(p.parse()?), None => None, }; - cplane.new_node(tenantid, timeline_name, port)?; + cplane.new_node(tenantid, node_name, timeline_name, port)?; } ("start", Some(start_match)) => { let tenantid: ZTenantId = start_match .value_of("tenantid") .map_or(Ok(env.tenantid), |value| value.parse())?; - let timeline_name = start_match.value_of("timeline").unwrap_or("main"); + let node_name = start_match.value_of("node").unwrap_or("main"); + let timeline_name = start_match.value_of("timeline"); let port: Option = match start_match.value_of("port") { Some(p) => Some(p.parse()?), None => None, }; - let node = cplane.nodes.get(&(tenantid, timeline_name.to_owned())); + let node = cplane.nodes.get(&(tenantid, node_name.to_owned())); let auth_token = if matches!(env.auth_type, AuthType::ZenithJWT) { let claims = Claims::new(Some(tenantid), Scope::Tenant); @@ -498,7 +500,7 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { // start --port X // stop // start <-- will also use port X even without explicit port argument - let node = cplane.new_node(tenantid, timeline_name, port)?; + let node = cplane.new_node(tenantid, node_name, timeline_name, port)?; node.start(&auth_token)?; } } From feae7f39c1ed0a107d3af03e588cfc8ec2c52ffe Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 19 Oct 2021 09:48:12 +0300 Subject: [PATCH 42/46] Support read-only nodes Change 'zenith.signal' file to a human-readable format, similar to backup_label. It can contain a "PREV LSN: %X/%X" line, or a special value to indicate that it's OK to start with invalid LSN ('none'), or that it's a read-only node and generating WAL is forbidden ('invalid'). The 'zenith pg create' and 'zenith pg start' commands now take a node name parameter, separate from the branch name. If the node name is not given, it defaults to the branch name, so this doesn't break existing scripts. If you pass "foo@" as the branch name, a read-only node anchored at that LSN is created. The anchoring is performed by setting the 'recovery_target_lsn' option in the postgresql.conf file, and putting the server into standby mode with 'standby.signal'. We no longer store the synthetic checkpoint record in the WAL segment. The postgres startup code has been changed to use the copy of the checkpoint record in the pg_control file, when starting in zenith mode. --- control_plane/src/compute.rs | 62 ++++++++++--- control_plane/src/postgresql_conf.rs | 16 ++++ pageserver/src/basebackup.rs | 35 ++++---- pageserver/src/layered_repository.rs | 4 + pageserver/src/repository.rs | 3 + postgres_ffi/src/xlog_utils.rs | 45 ++-------- .../batch_others/test_branch_behind.py | 3 + .../batch_others/test_readonly_node.py | 86 +++++++++++++++++++ vendor/postgres | 2 +- zenith/src/main.rs | 72 ++++++++++------ 10 files changed, 233 insertions(+), 95 deletions(-) create mode 100644 test_runner/batch_others/test_readonly_node.py diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 9a262a075a..c0107a431e 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -84,18 +84,43 @@ impl ComputeControlPlane { } } + // FIXME: see also parse_point_in_time in branches.rs. + fn parse_point_in_time( + &self, + tenantid: ZTenantId, + s: &str, + ) -> Result<(ZTimelineId, Option)> { + let mut strings = s.split('@'); + let name = strings.next().unwrap(); + + let lsn: Option; + if let Some(lsnstr) = strings.next() { + lsn = Some( + Lsn::from_str(lsnstr) + .with_context(|| "invalid LSN in point-in-time specification")?, + ); + } else { + lsn = None + } + + // Resolve the timeline ID, given the human-readable branch name + let timeline_id = self + .pageserver + .branch_get_by_name(&tenantid, name)? + .timeline_id; + + Ok((timeline_id, lsn)) + } + pub fn new_node( &mut self, tenantid: ZTenantId, name: &str, - branch_name: &str, + timeline_spec: &str, port: Option, ) -> Result> { - // Resolve the timeline ID, given the human-readable branch name - let timeline_id = self - .pageserver - .branch_get_by_name(&tenantid, branch_name)? - .timeline_id; + // Resolve the human-readable timeline spec into timeline ID and LSN + let (timelineid, lsn) = self.parse_point_in_time(tenantid, timeline_spec)?; let port = port.unwrap_or_else(|| self.get_port()); let node = Arc::new(PostgresNode { @@ -104,7 +129,8 @@ impl ComputeControlPlane { env: self.env.clone(), pageserver: Arc::clone(&self.pageserver), is_test: false, - timelineid: timeline_id, + timelineid, + lsn, tenantid, uses_wal_proposer: false, }); @@ -129,6 +155,7 @@ pub struct PostgresNode { pageserver: Arc, is_test: bool, pub timelineid: ZTimelineId, + pub lsn: Option, // if it's a read-only node. None for primary pub tenantid: ZTenantId, uses_wal_proposer: bool, } @@ -163,9 +190,12 @@ impl PostgresNode { let port: u16 = conf.parse_field("port", &context)?; let timelineid: ZTimelineId = conf.parse_field("zenith.zenith_timeline", &context)?; let tenantid: ZTenantId = conf.parse_field("zenith.zenith_tenant", &context)?; - let uses_wal_proposer = conf.get("wal_acceptors").is_some(); + // parse recovery_target_lsn, if any + let recovery_target_lsn: Option = + conf.parse_field_optional("recovery_target_lsn", &context)?; + // ok now Ok(PostgresNode { address: SocketAddr::new("127.0.0.1".parse().unwrap(), port), @@ -174,6 +204,7 @@ impl PostgresNode { pageserver: Arc::clone(pageserver), is_test: false, timelineid, + lsn: recovery_target_lsn, tenantid, uses_wal_proposer, }) @@ -235,7 +266,7 @@ impl PostgresNode { // Read the archive directly from the `CopyOutReader` tar::Archive::new(copyreader) .unpack(&self.pgdata()) - .with_context(|| "extracting page backup failed")?; + .with_context(|| "extracting base backup failed")?; Ok(()) } @@ -303,6 +334,9 @@ impl PostgresNode { conf.append("zenith.page_server_connstring", &pageserver_connstr); conf.append("zenith.zenith_tenant", &self.tenantid.to_string()); conf.append("zenith.zenith_timeline", &self.timelineid.to_string()); + if let Some(lsn) = self.lsn { + conf.append("recovery_target_lsn", &lsn.to_string()); + } conf.append_line(""); // Configure the node to stream WAL directly to the pageserver @@ -316,7 +350,9 @@ impl PostgresNode { } fn load_basebackup(&self) -> Result<()> { - let lsn = if self.uses_wal_proposer { + let backup_lsn = if let Some(lsn) = self.lsn { + Some(lsn) + } else if self.uses_wal_proposer { // LSN 0 means that it is bootstrap and we need to download just // latest data from the pageserver. That is a bit clumsy but whole bootstrap // procedure evolves quite actively right now, so let's think about it again @@ -331,7 +367,7 @@ impl PostgresNode { None }; - self.do_basebackup(lsn)?; + self.do_basebackup(backup_lsn)?; Ok(()) } @@ -408,6 +444,10 @@ impl PostgresNode { // 3. Load basebackup self.load_basebackup()?; + if self.lsn.is_some() { + File::create(self.pgdata().join("standby.signal"))?; + } + // 4. Finally start the compute node postgres println!("Starting postgres node at '{}'", self.connstr()); self.pg_ctl(&["start"], auth_token) diff --git a/control_plane/src/postgresql_conf.rs b/control_plane/src/postgresql_conf.rs index bcd463999b..7f50fe9c2f 100644 --- a/control_plane/src/postgresql_conf.rs +++ b/control_plane/src/postgresql_conf.rs @@ -83,6 +83,22 @@ impl PostgresConf { .with_context(|| format!("could not parse '{}' option {}", field_name, context)) } + pub fn parse_field_optional(&self, field_name: &str, context: &str) -> Result> + where + T: FromStr, + ::Err: std::error::Error + Send + Sync + 'static, + { + if let Some(val) = self.get(field_name) { + let result = val + .parse::() + .with_context(|| format!("could not parse '{}' option {}", field_name, context))?; + + Ok(Some(result)) + } else { + Ok(None) + } + } + /// /// Note: if you call this multiple times for the same option, the config /// file will a line for each call. It would be nice to have a function diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index a4ee89918c..def815a32d 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -13,6 +13,7 @@ use anyhow::Result; use bytes::{BufMut, BytesMut}; use log::*; +use std::fmt::Write as FmtWrite; use std::io; use std::io::Write; use std::sync::Arc; @@ -83,7 +84,7 @@ impl<'a> Basebackup<'a> { info!( "taking basebackup lsn={}, prev_lsn={}", - backup_prev, backup_lsn + backup_lsn, backup_prev ); Ok(Basebackup { @@ -248,13 +249,7 @@ impl<'a> Basebackup<'a> { let mut pg_control = ControlFileData::decode(&pg_control_bytes)?; let mut checkpoint = CheckPoint::decode(&checkpoint_bytes)?; - // Generate new pg_control and WAL needed for bootstrap - let checkpoint_segno = self.lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE); - let checkpoint_lsn = XLogSegNoOffsetToRecPtr( - checkpoint_segno, - XLOG_SIZE_OF_XLOG_LONG_PHD as u32, - pg_constants::WAL_SEGMENT_SIZE, - ); + // Generate new pg_control needed for bootstrap checkpoint.redo = normalize_lsn(self.lsn, pg_constants::WAL_SEGMENT_SIZE).0; //reset some fields we don't want to preserve @@ -263,19 +258,24 @@ impl<'a> Basebackup<'a> { checkpoint.oldestActiveXid = 0; //save new values in pg_control - pg_control.checkPoint = checkpoint_lsn; + pg_control.checkPoint = 0; pg_control.checkPointCopy = checkpoint; pg_control.state = pg_constants::DB_SHUTDOWNED; // add zenith.signal file - let xl_prev = if self.prev_record_lsn == Lsn(0) { - 0xBAD0 // magic value to indicate that we don't know prev_lsn + let mut zenith_signal = String::new(); + if self.prev_record_lsn == Lsn(0) { + if self.lsn == self.timeline.get_ancestor_lsn() { + write!(zenith_signal, "PREV LSN: none")?; + } else { + write!(zenith_signal, "PREV LSN: invalid")?; + } } else { - self.prev_record_lsn.0 - }; + write!(zenith_signal, "PREV LSN: {}", self.prev_record_lsn)?; + } self.ar.append( - &new_tar_header("zenith.signal", 8)?, - &xl_prev.to_le_bytes()[..], + &new_tar_header("zenith.signal", zenith_signal.len() as u64)?, + zenith_signal.as_bytes(), )?; //send pg_control @@ -284,14 +284,15 @@ impl<'a> Basebackup<'a> { self.ar.append(&header, &pg_control_bytes[..])?; //send wal segment + let segno = self.lsn.segment_number(pg_constants::WAL_SEGMENT_SIZE); let wal_file_name = XLogFileName( 1, // FIXME: always use Postgres timeline 1 - checkpoint_segno, + segno, pg_constants::WAL_SEGMENT_SIZE, ); let wal_file_path = format!("pg_wal/{}", wal_file_name); let header = new_tar_header(&wal_file_path, pg_constants::WAL_SEGMENT_SIZE as u64)?; - let wal_seg = generate_wal_segment(&pg_control); + let wal_seg = generate_wal_segment(segno, pg_control.system_identifier); assert!(wal_seg.len() == pg_constants::WAL_SEGMENT_SIZE); self.ar.append(&header, &wal_seg[..])?; Ok(()) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 4da146ae34..b17d08a33a 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -678,6 +678,10 @@ pub struct LayeredTimeline { /// Public interface functions impl Timeline for LayeredTimeline { + fn get_ancestor_lsn(&self) -> Lsn { + self.ancestor_lsn + } + /// Wait until WAL has been received up to the given LSN. fn wait_lsn(&self, lsn: Lsn) -> Result<()> { // This should never be called from the WAL receiver thread, because that could lead diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 56e551a275..3009d51352 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -119,6 +119,9 @@ pub trait Timeline: Send + Sync { /// Get a list of all existing non-relational objects fn list_nonrels(&self, lsn: Lsn) -> Result>; + /// Get the LSN where this branch was created + fn get_ancestor_lsn(&self) -> Lsn; + //------------------------------------------------------------------------------ // Public PUT functions, to update the repository with new page versions. // diff --git a/postgres_ffi/src/xlog_utils.rs b/postgres_ffi/src/xlog_utils.rs index 7826630a78..7f88de4c85 100644 --- a/postgres_ffi/src/xlog_utils.rs +++ b/postgres_ffi/src/xlog_utils.rs @@ -9,7 +9,6 @@ use crate::pg_constants; use crate::CheckPoint; -use crate::ControlFileData; use crate::FullTransactionId; use crate::XLogLongPageHeaderData; use crate::XLogPageHeaderData; @@ -18,8 +17,8 @@ use crate::XLOG_PAGE_MAGIC; use anyhow::{bail, Result}; use byteorder::{ByteOrder, LittleEndian}; +use bytes::BytesMut; use bytes::{Buf, Bytes}; -use bytes::{BufMut, BytesMut}; use crc32c::*; use log::*; use std::cmp::max; @@ -410,27 +409,25 @@ impl CheckPoint { } // -// Generate new WAL segment with single XLOG_CHECKPOINT_SHUTDOWN record. +// Generate new, empty WAL segment. // We need this segment to start compute node. -// In order to minimize changes in Postgres core, we prefer to -// provide WAL segment from which is can extract checkpoint record in standard way, -// rather then implement some alternative mechanism. // -pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes { +pub fn generate_wal_segment(segno: u64, system_id: u64) -> Bytes { let mut seg_buf = BytesMut::with_capacity(pg_constants::WAL_SEGMENT_SIZE as usize); + let pageaddr = XLogSegNoOffsetToRecPtr(segno, 0, pg_constants::WAL_SEGMENT_SIZE); let hdr = XLogLongPageHeaderData { std: { XLogPageHeaderData { xlp_magic: XLOG_PAGE_MAGIC as u16, xlp_info: pg_constants::XLP_LONG_HEADER, xlp_tli: 1, // FIXME: always use Postgres timeline 1 - xlp_pageaddr: pg_control.checkPoint - XLOG_SIZE_OF_XLOG_LONG_PHD as u64, + xlp_pageaddr: pageaddr, xlp_rem_len: 0, ..Default::default() // Put 0 in padding fields. } }, - xlp_sysid: pg_control.system_identifier, + xlp_sysid: system_id, xlp_seg_size: pg_constants::WAL_SEGMENT_SIZE as u32, xlp_xlog_blcksz: XLOG_BLCKSZ as u32, }; @@ -438,36 +435,6 @@ pub fn generate_wal_segment(pg_control: &ControlFileData) -> Bytes { let hdr_bytes = hdr.encode(); seg_buf.extend_from_slice(&hdr_bytes); - let rec_hdr = XLogRecord { - xl_tot_len: (XLOG_SIZE_OF_XLOG_RECORD - + SIZE_OF_XLOG_RECORD_DATA_HEADER_SHORT - + SIZEOF_CHECKPOINT) as u32, - xl_xid: 0, //0 is for InvalidTransactionId - xl_prev: 0, - xl_info: pg_constants::XLOG_CHECKPOINT_SHUTDOWN, - xl_rmid: pg_constants::RM_XLOG_ID, - xl_crc: 0, - ..Default::default() // Put 0 in padding fields. - }; - - let mut rec_shord_hdr_bytes = BytesMut::new(); - rec_shord_hdr_bytes.put_u8(pg_constants::XLR_BLOCK_ID_DATA_SHORT); - rec_shord_hdr_bytes.put_u8(SIZEOF_CHECKPOINT as u8); - - let rec_bytes = rec_hdr.encode(); - let checkpoint_bytes = pg_control.checkPointCopy.encode(); - - //calculate record checksum - let mut crc = 0; - crc = crc32c_append(crc, &rec_shord_hdr_bytes[..]); - crc = crc32c_append(crc, &checkpoint_bytes[..]); - crc = crc32c_append(crc, &rec_bytes[0..XLOG_RECORD_CRC_OFFS]); - - seg_buf.extend_from_slice(&rec_bytes[0..XLOG_RECORD_CRC_OFFS]); - seg_buf.put_u32_le(crc); - seg_buf.extend_from_slice(&rec_shord_hdr_bytes); - seg_buf.extend_from_slice(&checkpoint_bytes); - //zero out the rest of the file seg_buf.resize(pg_constants::WAL_SEGMENT_SIZE, 0); seg_buf.freeze() diff --git a/test_runner/batch_others/test_branch_behind.py b/test_runner/batch_others/test_branch_behind.py index 47c2f0b2f9..887671bf99 100644 --- a/test_runner/batch_others/test_branch_behind.py +++ b/test_runner/batch_others/test_branch_behind.py @@ -86,7 +86,10 @@ def test_branch_behind(zenith_cli, pageserver: ZenithPageserver, postgres: Postg assert cur.fetchone() == (1, ) # branch at pre-initdb lsn + # + # FIXME: This works currently, but probably shouldn't be allowed try: zenith_cli.run(["branch", "test_branch_preinitdb", "test_branch_behind@0/42"]) + # FIXME: assert false, "branch with invalid LSN should have failed" except subprocess.CalledProcessError: log.info("Branch creation with pre-initdb LSN failed (as expected)") diff --git a/test_runner/batch_others/test_readonly_node.py b/test_runner/batch_others/test_readonly_node.py new file mode 100644 index 0000000000..e6df7df0ef --- /dev/null +++ b/test_runner/batch_others/test_readonly_node.py @@ -0,0 +1,86 @@ +import subprocess +from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver + +pytest_plugins = ("fixtures.zenith_fixtures") + +# +# Create read-only compute nodes, anchored at historical points in time. +# +# This is very similar to the 'test_branch_behind' test, but instead of +# creating branches, creates read-only nodes. +# +def test_readonly_node(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin): + zenith_cli.run(["branch", "test_readonly_node", "empty"]) + + pgmain = postgres.create_start('test_readonly_node') + print("postgres is running on 'test_readonly_node' branch") + + main_pg_conn = pgmain.connect() + main_cur = main_pg_conn.cursor() + + # Create table, and insert the first 100 rows + main_cur.execute('CREATE TABLE foo (t text)') + main_cur.execute(''' + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 100) g + ''') + main_cur.execute('SELECT pg_current_wal_insert_lsn()') + lsn_a = main_cur.fetchone()[0] + print('LSN after 100 rows: ' + lsn_a) + + # Insert some more rows. (This generates enough WAL to fill a few segments.) + main_cur.execute(''' + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 200000) g + ''') + main_cur.execute('SELECT pg_current_wal_insert_lsn()') + lsn_b = main_cur.fetchone()[0] + print('LSN after 200100 rows: ' + lsn_b) + + # Insert many more rows. This generates enough WAL to fill a few segments. + main_cur.execute(''' + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 200000) g + ''') + + main_cur.execute('SELECT pg_current_wal_insert_lsn()') + lsn_c = main_cur.fetchone()[0] + print('LSN after 400100 rows: ' + lsn_c) + + # Create first read-only node at the point where only 100 rows were inserted + pg_hundred = postgres.create_start("test_readonly_node_hundred", branch=f'test_readonly_node@{lsn_a}') + + # And another at the point where 200100 rows were inserted + pg_more = postgres.create_start("test_readonly_node_more", branch=f'test_readonly_node@{lsn_b}') + + # On the 'hundred' node, we should see only 100 rows + hundred_pg_conn = pg_hundred.connect() + hundred_cur = hundred_pg_conn.cursor() + hundred_cur.execute('SELECT count(*) FROM foo') + assert hundred_cur.fetchone() == (100, ) + + # On the 'more' node, we should see 100200 rows + more_pg_conn = pg_more.connect() + more_cur = more_pg_conn.cursor() + more_cur.execute('SELECT count(*) FROM foo') + assert more_cur.fetchone() == (200100, ) + + # All the rows are visible on the main branch + main_cur.execute('SELECT count(*) FROM foo') + assert main_cur.fetchone() == (400100, ) + + # Check creating a node at segment boundary + pg = postgres.create_start("test_branch_segment_boundary", branch="test_readonly_node@0/3000000") + cur = pg.connect().cursor() + cur.execute('SELECT 1') + assert cur.fetchone() == (1, ) + + # Create node at pre-initdb lsn + try: + zenith_cli.run(["pg", "start", "test_branch_preinitdb", "test_readonly_node@0/42"]) + assert false, "compute node startup with invalid LSN should have failed" + except Exception: + print("Node creation with pre-initdb LSN failed (as expected)") diff --git a/vendor/postgres b/vendor/postgres index 5387eb4a3b..9160deb05a 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit 5387eb4a3b892d3ff18a3a93f4bd996d43ea3b33 +Subproject commit 9160deb05a08986354721173ba36e3ebc50a9e21 diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 1692695767..e79d42377e 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -32,12 +32,16 @@ struct BranchTreeEl { // * Providing CLI api to the pageserver // * TODO: export/import to/from usual postgres fn main() -> Result<()> { - let timeline_arg = Arg::with_name("timeline") - .short("n") + let node_arg = Arg::with_name("node") .index(1) - .help("Timeline name") + .help("Node name") .required(true); + let timeline_arg = Arg::with_name("timeline") + .index(2) + .help("Branch name or a point-in time specification") + .required(false); + let tenantid_arg = Arg::with_name("tenantid") .long("tenantid") .help("Tenant id. Represented as a hexadecimal string 32 symbols length") @@ -102,7 +106,10 @@ fn main() -> Result<()> { .subcommand(SubCommand::with_name("list").arg(tenantid_arg.clone())) .subcommand(SubCommand::with_name("create") .about("Create a postgres compute node") - .arg(timeline_arg.clone()).arg(tenantid_arg.clone()).arg(port_arg.clone()) + .arg(node_arg.clone()) + .arg(timeline_arg.clone()) + .arg(tenantid_arg.clone()) + .arg(port_arg.clone()) .arg( Arg::with_name("config-only") .help("Don't do basebackup, create compute node with only config files") @@ -111,13 +118,13 @@ fn main() -> Result<()> { )) .subcommand(SubCommand::with_name("start") .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files") - .arg( - timeline_arg.clone() - ).arg( - tenantid_arg.clone() - ).arg(port_arg.clone())) + .arg(node_arg.clone()) + .arg(timeline_arg.clone()) + .arg(tenantid_arg.clone()) + .arg(port_arg.clone())) .subcommand( SubCommand::with_name("stop") + .arg(node_arg.clone()) .arg(timeline_arg.clone()) .arg(tenantid_arg.clone()) .arg( @@ -430,25 +437,32 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let tenantid: ZTenantId = list_match .value_of("tenantid") .map_or(Ok(env.tenantid), |value| value.parse())?; + let branch_infos = get_branch_infos(env, &tenantid).unwrap_or_else(|e| { eprintln!("Failed to load branch info: {}", e); HashMap::new() }); - println!("BRANCH\tADDRESS\t\tLSN\t\tSTATUS"); - for ((_, timeline_name), node) in cplane + println!("NODE\tADDRESS\t\tBRANCH\tLSN\t\tSTATUS"); + for ((_, node_name), node) in cplane .nodes .iter() .filter(|((node_tenantid, _), _)| node_tenantid == &tenantid) { + // FIXME: This shows the LSN at the end of the timeline. It's not the + // right thing to do for read-only nodes that might be anchored at an + // older point in time, or following but lagging behind the primary. + let lsn_str = branch_infos + .get(&node.timelineid) + .map(|bi| bi.latest_valid_lsn.to_string()) + .unwrap_or_else(|| "?".to_string()); + println!( - "{}\t{}\t{}\t{}", - timeline_name, + "{}\t{}\t{}\t{}\t{}", + node_name, node.address, - branch_infos - .get(&node.timelineid) - .map(|bi| bi.latest_valid_lsn.to_string()) - .unwrap_or_else(|| "?".to_string()), + node.timelineid, // FIXME: resolve human-friendly branch name + lsn_str, node.status(), ); } @@ -457,8 +471,8 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let tenantid: ZTenantId = create_match .value_of("tenantid") .map_or(Ok(env.tenantid), |value| value.parse())?; - let node_name = start_match.value_of("node").unwrap_or("main"); - let timeline_name = start_match.value_of("timeline"); + let node_name = create_match.value_of("node").unwrap_or("main"); + let timeline_name = create_match.value_of("timeline").unwrap_or(node_name); let port: Option = match create_match.value_of("port") { Some(p) => Some(p.parse()?), @@ -487,12 +501,11 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { None }; - println!( - "Starting {} postgres on timeline {}...", - if node.is_some() { "existing" } else { "new" }, - timeline_name - ); if let Some(node) = node { + if timeline_name.is_some() { + println!("timeline name ignored because node exists already"); + } + println!("Starting existing postgres {}...", node_name); node.start(&auth_token)?; } else { // when used with custom port this results in non obvious behaviour @@ -500,12 +513,17 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { // start --port X // stop // start <-- will also use port X even without explicit port argument + let timeline_name = timeline_name.unwrap_or(node_name); + println!( + "Starting new postgres {} on {}...", + node_name, timeline_name + ); let node = cplane.new_node(tenantid, node_name, timeline_name, port)?; node.start(&auth_token)?; } } ("stop", Some(stop_match)) => { - let timeline_name = stop_match.value_of("timeline").unwrap_or("main"); + let node_name = stop_match.value_of("node").unwrap_or("main"); let destroy = stop_match.is_present("destroy"); let tenantid: ZTenantId = stop_match .value_of("tenantid") @@ -513,8 +531,8 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { let node = cplane .nodes - .get(&(tenantid, timeline_name.to_owned())) - .ok_or_else(|| anyhow!("postgres {} is not found", timeline_name))?; + .get(&(tenantid, node_name.to_owned())) + .ok_or_else(|| anyhow!("postgres {} is not found", node_name))?; node.stop(destroy)?; } From 732d13fe062a6e91b4f9df7f916e4ae9fe9d6700 Mon Sep 17 00:00:00 2001 From: Dmitry Rodionov Date: Tue, 28 Sep 2021 14:23:10 +0300 Subject: [PATCH 43/46] use cached-property package because python<3.8 doesnt have cached_property in functools --- test_runner/Pipfile | 1 + test_runner/Pipfile.lock | 198 +++++++++++++----------- test_runner/fixtures/zenith_fixtures.py | 3 +- 3 files changed, 108 insertions(+), 94 deletions(-) diff --git a/test_runner/Pipfile b/test_runner/Pipfile index f5ff0d7e2b..e0c7102a96 100644 --- a/test_runner/Pipfile +++ b/test_runner/Pipfile @@ -11,6 +11,7 @@ pyjwt = {extras = ["crypto"], version = "*"} requests = "*" pytest-xdist = "*" asyncpg = "*" +cached-property = "*" [dev-packages] yapf = "*" diff --git a/test_runner/Pipfile.lock b/test_runner/Pipfile.lock index 3c68c0ff3a..fdaa8ecfa6 100644 --- a/test_runner/Pipfile.lock +++ b/test_runner/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3cdc048691824d0b93912b6b78a0aa01dc98f278212c1badb0cc2edbd2103c3a" + "sha256": "b3ebe8fa70f41f9f79a8727ff47131b9e30772548749c85587987dcbb7bed336" }, "pipfile-spec": 6, "requires": { @@ -43,94 +43,108 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==21.2.0" }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "index": "pypi", + "version": "==1.5.2" + }, "certifi": { "hashes": [ - "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", - "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" + "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", + "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" ], - "version": "==2021.5.30" + "version": "==2021.10.8" }, "cffi": { "hashes": [ - "sha256:06c54a68935738d206570b20da5ef2b6b6d92b38ef3ec45c5422c0ebaf338d4d", - "sha256:0c0591bee64e438883b0c92a7bed78f6290d40bf02e54c5bf0978eaf36061771", - "sha256:19ca0dbdeda3b2615421d54bef8985f72af6e0c47082a8d26122adac81a95872", - "sha256:22b9c3c320171c108e903d61a3723b51e37aaa8c81255b5e7ce102775bd01e2c", - "sha256:26bb2549b72708c833f5abe62b756176022a7b9a7f689b571e74c8478ead51dc", - "sha256:33791e8a2dc2953f28b8d8d300dde42dd929ac28f974c4b4c6272cb2955cb762", - "sha256:3c8d896becff2fa653dc4438b54a5a25a971d1f4110b32bd3068db3722c80202", - "sha256:4373612d59c404baeb7cbd788a18b2b2a8331abcc84c3ba40051fcd18b17a4d5", - "sha256:487d63e1454627c8e47dd230025780e91869cfba4c753a74fda196a1f6ad6548", - "sha256:48916e459c54c4a70e52745639f1db524542140433599e13911b2f329834276a", - "sha256:4922cd707b25e623b902c86188aca466d3620892db76c0bdd7b99a3d5e61d35f", - "sha256:55af55e32ae468e9946f741a5d51f9896da6b9bf0bbdd326843fec05c730eb20", - "sha256:57e555a9feb4a8460415f1aac331a2dc833b1115284f7ded7278b54afc5bd218", - "sha256:5d4b68e216fc65e9fe4f524c177b54964af043dde734807586cf5435af84045c", - "sha256:64fda793737bc4037521d4899be780534b9aea552eb673b9833b01f945904c2e", - "sha256:6d6169cb3c6c2ad50db5b868db6491a790300ade1ed5d1da29289d73bbe40b56", - "sha256:7bcac9a2b4fdbed2c16fa5681356d7121ecabf041f18d97ed5b8e0dd38a80224", - "sha256:80b06212075346b5546b0417b9f2bf467fea3bfe7352f781ffc05a8ab24ba14a", - "sha256:818014c754cd3dba7229c0f5884396264d51ffb87ec86e927ef0be140bfdb0d2", - "sha256:8eb687582ed7cd8c4bdbff3df6c0da443eb89c3c72e6e5dcdd9c81729712791a", - "sha256:99f27fefe34c37ba9875f224a8f36e31d744d8083e00f520f133cab79ad5e819", - "sha256:9f3e33c28cd39d1b655ed1ba7247133b6f7fc16fa16887b120c0c670e35ce346", - "sha256:a8661b2ce9694ca01c529bfa204dbb144b275a31685a075ce123f12331be790b", - "sha256:a9da7010cec5a12193d1af9872a00888f396aba3dc79186604a09ea3ee7c029e", - "sha256:aedb15f0a5a5949ecb129a82b72b19df97bbbca024081ed2ef88bd5c0a610534", - "sha256:b315d709717a99f4b27b59b021e6207c64620790ca3e0bde636a6c7f14618abb", - "sha256:ba6f2b3f452e150945d58f4badd92310449876c4c954836cfb1803bdd7b422f0", - "sha256:c33d18eb6e6bc36f09d793c0dc58b0211fccc6ae5149b808da4a62660678b156", - "sha256:c9a875ce9d7fe32887784274dd533c57909b7b1dcadcc128a2ac21331a9765dd", - "sha256:c9e005e9bd57bc987764c32a1bee4364c44fdc11a3cc20a40b93b444984f2b87", - "sha256:d2ad4d668a5c0645d281dcd17aff2be3212bc109b33814bbb15c4939f44181cc", - "sha256:d950695ae4381ecd856bcaf2b1e866720e4ab9a1498cba61c602e56630ca7195", - "sha256:e22dcb48709fc51a7b58a927391b23ab37eb3737a98ac4338e2448bef8559b33", - "sha256:e8c6a99be100371dbb046880e7a282152aa5d6127ae01783e37662ef73850d8f", - "sha256:e9dc245e3ac69c92ee4c167fbdd7428ec1956d4e754223124991ef29eb57a09d", - "sha256:eb687a11f0a7a1839719edd80f41e459cc5366857ecbed383ff376c4e3cc6afd", - "sha256:eb9e2a346c5238a30a746893f23a9535e700f8192a68c07c0258e7ece6ff3728", - "sha256:ed38b924ce794e505647f7c331b22a693bee1538fdf46b0222c4717b42f744e7", - "sha256:f0010c6f9d1a4011e429109fda55a225921e3206e7f62a0c22a35344bfd13cca", - "sha256:f0c5d1acbfca6ebdd6b1e3eded8d261affb6ddcf2186205518f1428b8569bb99", - "sha256:f10afb1004f102c7868ebfe91c28f4a712227fe4cb24974350ace1f90e1febbf", - "sha256:f174135f5609428cc6e1b9090f9268f5c8935fddb1b25ccb8255a2d50de6789e", - "sha256:f3ebe6e73c319340830a9b2825d32eb6d8475c1dac020b4f0aa774ee3b898d1c", - "sha256:f627688813d0a4140153ff532537fbe4afea5a3dffce1f9deb7f91f848a832b5", - "sha256:fd4305f86f53dfd8cd3522269ed7fc34856a8ee3709a5e28b2836b2db9d4cd69" + "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", + "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", + "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", + "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", + "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", + "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", + "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", + "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", + "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", + "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", + "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", + "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", + "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", + "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", + "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", + "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", + "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", + "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", + "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", + "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", + "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", + "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", + "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", + "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", + "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", + "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", + "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", + "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", + "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", + "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", + "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", + "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", + "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", + "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", + "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", + "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", + "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", + "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", + "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", + "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", + "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", + "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", + "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", + "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", + "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", + "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", + "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", + "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", + "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", + "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" ], - "version": "==1.14.6" + "version": "==1.15.0" }, "charset-normalizer": { "hashes": [ - "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6", - "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f" + "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0", + "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b" ], "markers": "python_version >= '3'", - "version": "==2.0.6" + "version": "==2.0.7" }, "cryptography": { "hashes": [ - "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e", - "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b", - "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7", - "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085", - "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc", - "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d", - "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a", - "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498", - "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89", - "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9", - "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c", - "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7", - "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb", - "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14", - "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af", - "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e", - "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5", - "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06", - "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7" + "sha256:07bb7fbfb5de0980590ddfc7f13081520def06dc9ed214000ad4372fb4e3c7f6", + "sha256:18d90f4711bf63e2fb21e8c8e51ed8189438e6b35a6d996201ebd98a26abbbe6", + "sha256:1ed82abf16df40a60942a8c211251ae72858b25b7421ce2497c2eb7a1cee817c", + "sha256:22a38e96118a4ce3b97509443feace1d1011d0571fae81fc3ad35f25ba3ea999", + "sha256:2d69645f535f4b2c722cfb07a8eab916265545b3475fdb34e0be2f4ee8b0b15e", + "sha256:4a2d0e0acc20ede0f06ef7aa58546eee96d2592c00f450c9acb89c5879b61992", + "sha256:54b2605e5475944e2213258e0ab8696f4f357a31371e538ef21e8d61c843c28d", + "sha256:7075b304cd567694dc692ffc9747f3e9cb393cc4aa4fb7b9f3abd6f5c4e43588", + "sha256:7b7ceeff114c31f285528ba8b390d3e9cfa2da17b56f11d366769a807f17cbaa", + "sha256:7eba2cebca600a7806b893cb1d541a6e910afa87e97acf2021a22b32da1df52d", + "sha256:928185a6d1ccdb816e883f56ebe92e975a262d31cc536429041921f8cb5a62fd", + "sha256:9933f28f70d0517686bd7de36166dda42094eac49415459d9bdf5e7df3e0086d", + "sha256:a688ebcd08250eab5bb5bca318cc05a8c66de5e4171a65ca51db6bd753ff8953", + "sha256:abb5a361d2585bb95012a19ed9b2c8f412c5d723a9836418fab7aaa0243e67d2", + "sha256:c10c797ac89c746e488d2ee92bd4abd593615694ee17b2500578b63cad6b93a8", + "sha256:ced40344e811d6abba00295ced98c01aecf0c2de39481792d87af4fa58b7b4d6", + "sha256:d57e0cdc1b44b6cdf8af1d01807db06886f10177469312fbde8f44ccbb284bc9", + "sha256:d99915d6ab265c22873f1b4d6ea5ef462ef797b4140be4c9d8b179915e0985c6", + "sha256:eb80e8a1f91e4b7ef8b33041591e6d89b2b8e122d787e87eeb2b08da71bb16ad", + "sha256:ebeddd119f526bcf323a89f853afb12e225902a24d29b55fe18dd6fcb2838a76" ], - "version": "==3.4.8" + "version": "==35.0.0" }, "execnet": { "hashes": [ @@ -142,11 +156,11 @@ }, "idna": { "hashes": [ - "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", - "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], "markers": "python_version >= '3'", - "version": "==3.2" + "version": "==3.3" }, "iniconfig": { "hashes": [ @@ -207,11 +221,11 @@ "crypto" ], "hashes": [ - "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1", - "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130" + "sha256:b888b4d56f06f6dcd777210c334e69c737be74755d3e5e9ee3fe67dc18a0ee41", + "sha256:e0c4bb8d9f0af0c7f5b1ec4c5036309617d03d56932877f2f7a0beeb5318322f" ], "index": "pypi", - "version": "==2.1.0" + "version": "==2.3.0" }, "pyparsing": { "hashes": [ @@ -272,21 +286,21 @@ }, "urllib3": { "hashes": [ - "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", - "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" + "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece", + "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.6" + "version": "==1.26.7" } }, "develop": { "flake8": { "hashes": [ - "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b", - "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907" + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" ], "index": "pypi", - "version": "==3.9.2" + "version": "==4.0.1" }, "mccabe": { "hashes": [ @@ -333,19 +347,19 @@ }, "pycodestyle": { "hashes": [ - "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068", - "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef" + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.7.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" }, "pyflakes": { "hashes": [ - "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", - "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db" + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.3.1" + "version": "==2.4.0" }, "toml": { "hashes": [ diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index eda04a8da6..7dbe61b04c 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from functools import cached_property +from cached_property import cached_property import asyncpg import os import pathlib @@ -13,7 +13,6 @@ import signal import subprocess import time import filecmp -import difflib from contextlib import closing from pathlib import Path From 798df756de4cf5bad7ddf511b098fd09a03655f6 Mon Sep 17 00:00:00 2001 From: Dmitry Rodionov Date: Tue, 19 Oct 2021 16:39:26 +0300 Subject: [PATCH 44/46] suppress FileNotFound exception instead of missing_ok=True because the latter is added in python 3.8 and we claim to support >3.6 --- test_runner/fixtures/zenith_fixtures.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index 7dbe61b04c..bd62ed4b15 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -14,7 +14,7 @@ import subprocess import time import filecmp -from contextlib import closing +from contextlib import closing, suppress from pathlib import Path from dataclasses import dataclass @@ -829,7 +829,8 @@ class WalAcceptor: def start(self) -> 'WalAcceptor': # create data directory if not exists self.data_dir.mkdir(parents=True, exist_ok=True) - self.pidfile.unlink(missing_ok=True) + with suppress(FileNotFoundError): + self.pidfile.unlink() cmd = [str(self.wa_bin_path)] cmd.extend(["-D", str(self.data_dir)]) From eb706bc9f40760668461899171d999022a1bfa28 Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Tue, 19 Oct 2021 20:13:47 +0300 Subject: [PATCH 45/46] Force yapf (Python code formatter) in CI (#772) * Add yapf run to CircleCI * Pin yapf version * Enable `SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES` setting * Reformat all existing code with slight manual adjustments * test_runner/README: note that yapf is forced --- .circleci/config.yml | 16 ++ test_runner/Pipfile | 4 +- test_runner/Pipfile.lock | 2 +- test_runner/README.md | 8 +- test_runner/batch_others/test_auth.py | 20 +- .../batch_others/test_clog_truncate.py | 9 +- test_runner/batch_others/test_createdropdb.py | 5 +- test_runner/batch_others/test_multixact.py | 8 +- .../batch_others/test_old_request_lsn.py | 14 +- .../batch_others/test_pageserver_api.py | 6 +- .../batch_others/test_pageserver_restart.py | 20 +- .../batch_others/test_readonly_node.py | 7 +- .../batch_others/test_restart_compute.py | 27 +-- test_runner/batch_others/test_snapfiles_gc.py | 36 ++-- test_runner/batch_others/test_tenants.py | 16 +- .../batch_others/test_timeline_size.py | 11 +- test_runner/batch_others/test_twophase.py | 9 +- test_runner/batch_others/test_vm_bits.py | 16 +- test_runner/batch_others/test_wal_acceptor.py | 24 ++- .../batch_others/test_wal_acceptor_async.py | 34 +-- test_runner/batch_others/test_zenith_cli.py | 8 +- .../batch_pg_regress/test_isolation.py | 10 +- .../batch_pg_regress/test_pg_regress.py | 10 +- .../batch_pg_regress/test_zenith_regress.py | 10 +- test_runner/fixtures/benchmark_fixture.py | 22 +- test_runner/fixtures/log_helper.py | 5 +- test_runner/fixtures/utils.py | 2 + test_runner/fixtures/zenith_fixtures.py | 204 ++++++++++++------ test_runner/performance/test_bulk_insert.py | 16 +- .../performance/test_bulk_tenant_create.py | 4 +- test_runner/performance/test_gist_build.py | 26 ++- test_runner/performance/test_perf_pgbench.py | 12 +- .../performance/test_write_amplification.py | 16 +- test_runner/setup.cfg | 1 + 34 files changed, 420 insertions(+), 218 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78d49cf74b..c94dd20ff0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -182,6 +182,21 @@ jobs: paths: - "*" + check-python: + executor: python/default + steps: + - checkout + - run: + name: Install pipenv & deps + working_directory: test_runner + command: | + pip install pipenv + pipenv install --dev + - run: + name: Run yapf to ensure code format + working_directory: test_runner + command: pipenv run yapf --recursive --diff . + run-pytest: #description: "Run pytest" executor: python/default @@ -333,6 +348,7 @@ workflows: build_and_test: jobs: - check-codestyle + - check-python - build-postgres: name: build-postgres-<< matrix.build_type >> matrix: diff --git a/test_runner/Pipfile b/test_runner/Pipfile index e0c7102a96..a98acc5718 100644 --- a/test_runner/Pipfile +++ b/test_runner/Pipfile @@ -14,9 +14,11 @@ asyncpg = "*" cached-property = "*" [dev-packages] -yapf = "*" flake8 = "*" mypy = "*" +# Behavior may change slightly between versions. These are run continuously, +# so we pin exact versions to avoid suprising breaks. Update if comfortable. +yapf = "==0.31.0" [requires] # we need at least 3.6, but pipenv doesn't allow to say this directly diff --git a/test_runner/Pipfile.lock b/test_runner/Pipfile.lock index fdaa8ecfa6..75fc17ffad 100644 --- a/test_runner/Pipfile.lock +++ b/test_runner/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b3ebe8fa70f41f9f79a8727ff47131b9e30772548749c85587987dcbb7bed336" + "sha256": "3645ae8d2dcf55bd2a54963c44cfeedf577f3b289d1077365214a80a7f36e643" }, "pipfile-spec": 6, "requires": { diff --git a/test_runner/README.md b/test_runner/README.md index 62a95350aa..e4bbff053d 100644 --- a/test_runner/README.md +++ b/test_runner/README.md @@ -95,11 +95,13 @@ Python destructors, e.g. `__del__()` aren't recommended for cleanup. ### Code quality +We force code formatting via yapf: + +1. Install `yapf` and other tools (`flake8`, `mypy`) with `pipenv install --dev`. +1. Reformat all your code by running `pipenv run yapf -ri .` in the `test_runner/` directory. + Before submitting a patch, please consider: * Writing a couple of docstrings to clarify the reasoning behind a new test. * Running `flake8` (or a linter of your choice, e.g. `pycodestyle`) and fixing possible defects, if any. -* Formatting the code with `yapf -r -i .` (TODO: implement an opt-in pre-commit hook for that). * (Optional) Typechecking the code with `mypy .`. Currently this mostly affects `fixtures/zenith_fixtures.py`. - -The tools can be installed with `pipenv install --dev`. diff --git a/test_runner/batch_others/test_auth.py b/test_runner/batch_others/test_auth.py index 614883d4b8..9fe7567902 100644 --- a/test_runner/batch_others/test_auth.py +++ b/test_runner/batch_others/test_auth.py @@ -1,4 +1,3 @@ - from contextlib import closing from typing import Iterator from uuid import uuid4 @@ -6,7 +5,6 @@ import psycopg2 from fixtures.zenith_fixtures import PortDistributor, Postgres, ZenithCli, ZenithPageserver, PgBin import pytest - pytest_plugins = ("fixtures.zenith_fixtures") @@ -35,7 +33,9 @@ def test_pageserver_auth(pageserver_auth_enabled: ZenithPageserver): ps.safe_psql(f"tenant_create {uuid4().hex}", password=management_token) # fail to create tenant using tenant token - with pytest.raises(psycopg2.DatabaseError, match='Attempt to access management api with tenant scope. Permission denied'): + with pytest.raises( + psycopg2.DatabaseError, + match='Attempt to access management api with tenant scope. Permission denied'): ps.safe_psql(f"tenant_create {uuid4().hex}", password=tenant_token) @@ -60,14 +60,14 @@ def test_compute_auth_to_pageserver( wa_factory.start_n_new(3, management_token) with Postgres( - zenith_cli=zenith_cli, - repo_dir=repo_dir, - pg_bin=pg_bin, - tenant_id=ps.initial_tenant, - port=port_distributor.get_port(), + zenith_cli=zenith_cli, + repo_dir=repo_dir, + pg_bin=pg_bin, + tenant_id=ps.initial_tenant, + port=port_distributor.get_port(), ).create_start( - branch, - wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None, + branch, + wal_acceptors=wa_factory.get_connstrs() if with_wal_acceptors else None, ) as pg: with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_clog_truncate.py b/test_runner/batch_others/test_clog_truncate.py index 8ad3c22732..a70e14d9a9 100644 --- a/test_runner/batch_others/test_clog_truncate.py +++ b/test_runner/batch_others/test_clog_truncate.py @@ -18,9 +18,12 @@ def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: Postg # set agressive autovacuum to make sure that truncation will happen config = [ - 'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0', - 'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0', - 'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s', + 'autovacuum_max_workers=10', + 'autovacuum_vacuum_threshold=0', + 'autovacuum_vacuum_insert_threshold=0', + 'autovacuum_vacuum_cost_delay=0', + 'autovacuum_vacuum_cost_limit=10000', + 'autovacuum_naptime =1s', 'autovacuum_freeze_max_age=100000' ] diff --git a/test_runner/batch_others/test_createdropdb.py b/test_runner/batch_others/test_createdropdb.py index 4194538556..5fe103496d 100644 --- a/test_runner/batch_others/test_createdropdb.py +++ b/test_runner/batch_others/test_createdropdb.py @@ -41,6 +41,7 @@ def test_createdb( for db in (pg, pg2): db.connect(dbname='foodb').close() + # # Test DROP DATABASE # @@ -49,7 +50,7 @@ def test_dropdb( pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, - test_output_dir + test_output_dir, ): zenith_cli.run(["branch", "test_dropdb", "empty"]) @@ -66,7 +67,6 @@ def test_dropdb( cur.execute("SELECT oid FROM pg_database WHERE datname='foodb';") dboid = cur.fetchone()[0] - with closing(pg.connect()) as conn: with conn.cursor() as cur: cur.execute('DROP DATABASE foodb') @@ -76,7 +76,6 @@ def test_dropdb( cur.execute('SELECT pg_current_wal_insert_lsn()') lsn_after_drop = cur.fetchone()[0] - # Create two branches before and after database drop. zenith_cli.run(["branch", "test_before_dropdb", "test_dropdb@" + lsn_before_drop]) pg_before = postgres.create_start('test_before_dropdb') diff --git a/test_runner/batch_others/test_multixact.py b/test_runner/batch_others/test_multixact.py index 403eee9974..78504b95ed 100644 --- a/test_runner/batch_others/test_multixact.py +++ b/test_runner/batch_others/test_multixact.py @@ -10,8 +10,12 @@ pytest_plugins = ("fixtures.zenith_fixtures") # it only checks next_multixact_id field in restored pg_control, # since we don't have functions to check multixact internals. # -def test_multixact(pageserver: ZenithPageserver, postgres: PostgresFactory, - pg_bin, zenith_cli, base_dir, test_output_dir): +def test_multixact(pageserver: ZenithPageserver, + postgres: PostgresFactory, + pg_bin, + zenith_cli, + base_dir, + test_output_dir): # Create a branch for us zenith_cli.run(["branch", "test_multixact", "empty"]) pg = postgres.create_start('test_multixact') diff --git a/test_runner/batch_others/test_old_request_lsn.py b/test_runner/batch_others/test_old_request_lsn.py index 49e87210e4..6cc5c01b83 100644 --- a/test_runner/batch_others/test_old_request_lsn.py +++ b/test_runner/batch_others/test_old_request_lsn.py @@ -5,6 +5,7 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") + # # Test where Postgres generates a lot of WAL, and it's garbage collected away, but # no pages are evicted so that Postgres uses an old LSN in a GetPage request. @@ -15,7 +16,10 @@ pytest_plugins = ("fixtures.zenith_fixtures") # just a hint that the page hasn't been modified since that LSN, and the page # server should return the latest page version regardless of the LSN. # -def test_old_request_lsn(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin): +def test_old_request_lsn(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + pg_bin): # Create a branch for us zenith_cli.run(["branch", "test_old_request_lsn", "empty"]) pg = postgres.create_start('test_old_request_lsn') @@ -47,20 +51,20 @@ def test_old_request_lsn(zenith_cli, pageserver: ZenithPageserver, postgres: Pos from pg_settings where name = 'shared_buffers' ''') row = cur.fetchone() - log.info(f'shared_buffers is {row[0]}, table size {row[1]}'); + log.info(f'shared_buffers is {row[0]}, table size {row[1]}') assert int(row[0]) < int(row[1]) - cur.execute('VACUUM foo'); + cur.execute('VACUUM foo') # Make a lot of updates on a single row, generating a lot of WAL. Trigger # garbage collections so that the page server will remove old page versions. for i in range(10): pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") for j in range(100): - cur.execute('UPDATE foo SET val = val + 1 WHERE id = 1;'); + cur.execute('UPDATE foo SET val = val + 1 WHERE id = 1;') # All (or at least most of) the updates should've been on the same page, so # that we haven't had to evict any dirty pages for a long time. Now run # a query that sends GetPage@LSN requests with the old LSN. - cur.execute("SELECT COUNT(*), SUM(val) FROM foo"); + cur.execute("SELECT COUNT(*), SUM(val) FROM foo") assert cur.fetchone() == (100000, 101000) diff --git a/test_runner/batch_others/test_pageserver_api.py b/test_runner/batch_others/test_pageserver_api.py index 8d0f92a263..95b0172e4c 100644 --- a/test_runner/batch_others/test_pageserver_api.py +++ b/test_runner/batch_others/test_pageserver_api.py @@ -63,7 +63,8 @@ def test_tenant_list_psql(pageserver: ZenithPageserver, zenith_cli): cur = conn.cursor() # check same tenant cannot be created twice - with pytest.raises(psycopg2.DatabaseError, match=f'tenant {pageserver.initial_tenant} already exists'): + with pytest.raises(psycopg2.DatabaseError, + match=f'tenant {pageserver.initial_tenant} already exists'): cur.execute(f'tenant_create {pageserver.initial_tenant}') # create one more tenant @@ -102,5 +103,6 @@ def test_pageserver_http_api_client(pageserver: ZenithPageserver): def test_pageserver_http_api_client_auth_enabled(pageserver_auth_enabled: ZenithPageserver): - client = pageserver_auth_enabled.http_client(auth_token=pageserver_auth_enabled.auth_keys.generate_management_token()) + client = pageserver_auth_enabled.http_client( + auth_token=pageserver_auth_enabled.auth_keys.generate_management_token()) check_client(client, pageserver_auth_enabled.initial_tenant) diff --git a/test_runner/batch_others/test_pageserver_restart.py b/test_runner/batch_others/test_pageserver_restart.py index 0656b7c6e8..5b4943aa27 100644 --- a/test_runner/batch_others/test_pageserver_restart.py +++ b/test_runner/batch_others/test_pageserver_restart.py @@ -9,17 +9,20 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") + # Check that dead minority doesn't prevent the commits: execute insert n_inserts # times, with fault_probability chance of getting a wal acceptor down or up # along the way. 2 of 3 are always alive, so the work keeps going. -def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory): +def test_pageserver_restart(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + wa_factory: WalAcceptorFactory): # One safekeeper is enough for this test. wa_factory.start_n_new(1) zenith_cli.run(["branch", "test_pageserver_restart", "empty"]) - pg = postgres.create_start('test_pageserver_restart', - wal_acceptors=wa_factory.get_connstrs()) + pg = postgres.create_start('test_pageserver_restart', wal_acceptors=wa_factory.get_connstrs()) pg_conn = pg.connect() cur = pg_conn.cursor() @@ -41,14 +44,14 @@ def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: from pg_settings where name = 'shared_buffers' ''') row = cur.fetchone() - log.info(f"shared_buffers is {row[0]}, table size {row[1]}"); + log.info(f"shared_buffers is {row[0]}, table size {row[1]}") assert int(row[0]) < int(row[1]) # Stop and restart pageserver. This is a more or less graceful shutdown, although # the page server doesn't currently have a shutdown routine so there's no difference # between stopping and crashing. - pageserver.stop(); - pageserver.start(); + pageserver.stop() + pageserver.start() # Stopping the pageserver breaks the connection from the postgres backend to # the page server, and causes the next query on the connection to fail. Start a new @@ -62,6 +65,5 @@ def test_pageserver_restart(zenith_cli, pageserver: ZenithPageserver, postgres: assert cur.fetchone() == (100000, ) # Stop the page server by force, and restart it - pageserver.stop(); - pageserver.start(); - + pageserver.stop() + pageserver.start() diff --git a/test_runner/batch_others/test_readonly_node.py b/test_runner/batch_others/test_readonly_node.py index e6df7df0ef..cc6c11caad 100644 --- a/test_runner/batch_others/test_readonly_node.py +++ b/test_runner/batch_others/test_readonly_node.py @@ -3,6 +3,7 @@ from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver pytest_plugins = ("fixtures.zenith_fixtures") + # # Create read-only compute nodes, anchored at historical points in time. # @@ -51,7 +52,8 @@ def test_readonly_node(zenith_cli, pageserver: ZenithPageserver, postgres: Postg print('LSN after 400100 rows: ' + lsn_c) # Create first read-only node at the point where only 100 rows were inserted - pg_hundred = postgres.create_start("test_readonly_node_hundred", branch=f'test_readonly_node@{lsn_a}') + pg_hundred = postgres.create_start("test_readonly_node_hundred", + branch=f'test_readonly_node@{lsn_a}') # And another at the point where 200100 rows were inserted pg_more = postgres.create_start("test_readonly_node_more", branch=f'test_readonly_node@{lsn_b}') @@ -73,7 +75,8 @@ def test_readonly_node(zenith_cli, pageserver: ZenithPageserver, postgres: Postg assert main_cur.fetchone() == (400100, ) # Check creating a node at segment boundary - pg = postgres.create_start("test_branch_segment_boundary", branch="test_readonly_node@0/3000000") + pg = postgres.create_start("test_branch_segment_boundary", + branch="test_readonly_node@0/3000000") cur = pg.connect().cursor() cur.execute('SELECT 1') assert cur.fetchone() == (1, ) diff --git a/test_runner/batch_others/test_restart_compute.py b/test_runner/batch_others/test_restart_compute.py index 48a19b490b..5d47d32aac 100644 --- a/test_runner/batch_others/test_restart_compute.py +++ b/test_runner/batch_others/test_restart_compute.py @@ -12,13 +12,13 @@ pytest_plugins = ("fixtures.zenith_fixtures") # @pytest.mark.parametrize('with_wal_acceptors', [False, True]) def test_restart_compute( - zenith_cli, - pageserver: ZenithPageserver, - postgres: PostgresFactory, - pg_bin, - wa_factory, - with_wal_acceptors: bool, - ): + zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + pg_bin, + wa_factory, + with_wal_acceptors: bool, +): wal_acceptor_connstrs = None zenith_cli.run(["branch", "test_restart_compute", "empty"]) @@ -26,8 +26,7 @@ def test_restart_compute( wa_factory.start_n_new(3) wal_acceptor_connstrs = wa_factory.get_connstrs() - pg = postgres.create_start('test_restart_compute', - wal_acceptors=wal_acceptor_connstrs) + pg = postgres.create_start('test_restart_compute', wal_acceptors=wal_acceptor_connstrs) log.info("postgres is running on 'test_restart_compute' branch") with closing(pg.connect()) as conn: @@ -40,9 +39,7 @@ def test_restart_compute( log.info(f"res = {r}") # Remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute', - wal_acceptors=wal_acceptor_connstrs) - + pg.stop_and_destroy().create_start('test_restart_compute', wal_acceptors=wal_acceptor_connstrs) with closing(pg.connect()) as conn: with conn.cursor() as cur: @@ -61,8 +58,7 @@ def test_restart_compute( log.info(f"res = {r}") # Again remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute', - wal_acceptors=wal_acceptor_connstrs) + pg.stop_and_destroy().create_start('test_restart_compute', wal_acceptors=wal_acceptor_connstrs) # That select causes lots of FPI's and increases probability of wakeepers # lagging behind after query completion @@ -76,8 +72,7 @@ def test_restart_compute( log.info(f"res = {r}") # And again remove data directory and restart - pg.stop_and_destroy().create_start('test_restart_compute', - wal_acceptors=wal_acceptor_connstrs) + pg.stop_and_destroy().create_start('test_restart_compute', wal_acceptors=wal_acceptor_connstrs) with closing(pg.connect()) as conn: with conn.cursor() as cur: diff --git a/test_runner/batch_others/test_snapfiles_gc.py b/test_runner/batch_others/test_snapfiles_gc.py index 9cd01ca42e..a799b34aa6 100644 --- a/test_runner/batch_others/test_snapfiles_gc.py +++ b/test_runner/batch_others/test_snapfiles_gc.py @@ -5,10 +5,15 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") + def print_gc_result(row): - log.info("GC duration {elapsed} ms".format_map(row)); - log.info(" REL total: {layer_relfiles_total}, needed_by_cutoff {layer_relfiles_needed_by_cutoff}, needed_by_branches: {layer_relfiles_needed_by_branches}, not_updated: {layer_relfiles_not_updated}, needed_as_tombstone {layer_relfiles_needed_as_tombstone}, removed: {layer_relfiles_removed}, dropped: {layer_relfiles_dropped}".format_map(row)) - log.info(" NONREL total: {layer_nonrelfiles_total}, needed_by_cutoff {layer_nonrelfiles_needed_by_cutoff}, needed_by_branches: {layer_nonrelfiles_needed_by_branches}, not_updated: {layer_nonrelfiles_not_updated}, needed_as_tombstone {layer_nonrelfiles_needed_as_tombstone}, removed: {layer_nonrelfiles_removed}, dropped: {layer_nonrelfiles_dropped}".format_map(row)) + log.info("GC duration {elapsed} ms".format_map(row)) + log.info( + " REL total: {layer_relfiles_total}, needed_by_cutoff {layer_relfiles_needed_by_cutoff}, needed_by_branches: {layer_relfiles_needed_by_branches}, not_updated: {layer_relfiles_not_updated}, needed_as_tombstone {layer_relfiles_needed_as_tombstone}, removed: {layer_relfiles_removed}, dropped: {layer_relfiles_dropped}" + .format_map(row)) + log.info( + " NONREL total: {layer_nonrelfiles_total}, needed_by_cutoff {layer_nonrelfiles_needed_by_cutoff}, needed_by_branches: {layer_nonrelfiles_needed_by_branches}, not_updated: {layer_nonrelfiles_not_updated}, needed_as_tombstone {layer_nonrelfiles_needed_as_tombstone}, removed: {layer_nonrelfiles_removed}, dropped: {layer_nonrelfiles_dropped}" + .format_map(row)) # @@ -24,7 +29,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): with closing(pg.connect()) as conn: with conn.cursor() as cur: with closing(pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory = psycopg2.extras.DictCursor) as pscur: + with psconn.cursor(cursor_factory=psycopg2.extras.DictCursor) as pscur: # Get the timeline ID of our branch. We need it for the 'do_gc' command cur.execute("SHOW zenith.zenith_timeline") @@ -34,9 +39,9 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): cur.execute("CREATE TABLE foo(x integer)") cur.execute("INSERT INTO foo VALUES (1)") - cur.execute("select relfilenode from pg_class where oid = 'foo'::regclass"); - row = cur.fetchone(); - log.info(f"relfilenode is {row[0]}"); + cur.execute("select relfilenode from pg_class where oid = 'foo'::regclass") + row = cur.fetchone() + log.info(f"relfilenode is {row[0]}") # Run GC, to clear out any garbage left behind in the catalogs by # the CREATE TABLE command. We want to have a clean slate with no garbage @@ -54,9 +59,10 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): log.info("Running GC before test") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() - print_gc_result(row); + print_gc_result(row) # remember the number of files - layer_relfiles_remain = row['layer_relfiles_total'] - row['layer_relfiles_removed'] + layer_relfiles_remain = (row['layer_relfiles_total'] - + row['layer_relfiles_removed']) assert layer_relfiles_remain > 0 # Insert a row and run GC. Checkpoint should freeze the layer @@ -66,7 +72,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): cur.execute("INSERT INTO foo VALUES (1)") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() - print_gc_result(row); + print_gc_result(row) assert row['layer_relfiles_total'] == layer_relfiles_remain + 2 assert row['layer_relfiles_removed'] == 2 assert row['layer_relfiles_dropped'] == 0 @@ -80,7 +86,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() - print_gc_result(row); + print_gc_result(row) assert row['layer_relfiles_total'] == layer_relfiles_remain + 2 assert row['layer_relfiles_removed'] == 2 assert row['layer_relfiles_dropped'] == 0 @@ -92,7 +98,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() - print_gc_result(row); + print_gc_result(row) assert row['layer_relfiles_total'] == layer_relfiles_remain + 2 assert row['layer_relfiles_removed'] == 2 assert row['layer_relfiles_dropped'] == 0 @@ -101,7 +107,7 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): log.info("Run GC again, with nothing to do") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() - print_gc_result(row); + print_gc_result(row) assert row['layer_relfiles_total'] == layer_relfiles_remain assert row['layer_relfiles_removed'] == 0 assert row['layer_relfiles_dropped'] == 0 @@ -109,12 +115,12 @@ def test_layerfiles_gc(zenith_cli, pageserver, postgres, pg_bin): # # Test DROP TABLE checks that relation data and metadata was deleted by GC from object storage # - log.info("Drop table and run GC again"); + log.info("Drop table and run GC again") cur.execute("DROP TABLE foo") pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() - print_gc_result(row); + print_gc_result(row) # We still cannot remove the latest layers # because they serve as tombstones for earlier layers. diff --git a/test_runner/batch_others/test_tenants.py b/test_runner/batch_others/test_tenants.py index b05a4a9f62..d646f10666 100644 --- a/test_runner/batch_others/test_tenants.py +++ b/test_runner/batch_others/test_tenants.py @@ -21,8 +21,18 @@ def test_tenants_normal_work( tenant_1 = tenant_factory.create() tenant_2 = tenant_factory.create() - zenith_cli.run(["branch", f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", "main", f"--tenantid={tenant_1}"]) - zenith_cli.run(["branch", f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", "main", f"--tenantid={tenant_2}"]) + zenith_cli.run([ + "branch", + f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", + "main", + f"--tenantid={tenant_1}" + ]) + zenith_cli.run([ + "branch", + f"test_tenants_normal_work_with_wal_acceptors{with_wal_acceptors}", + "main", + f"--tenantid={tenant_2}" + ]) if with_wal_acceptors: wa_factory.start_n_new(3) @@ -47,4 +57,4 @@ def test_tenants_normal_work( cur.execute("CREATE TABLE t(key int primary key, value text)") cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'") cur.execute("SELECT sum(key) FROM t") - assert cur.fetchone() == (5000050000,) + assert cur.fetchone() == (5000050000, ) diff --git a/test_runner/batch_others/test_timeline_size.py b/test_runner/batch_others/test_timeline_size.py index acc2394306..819edc26b4 100644 --- a/test_runner/batch_others/test_timeline_size.py +++ b/test_runner/batch_others/test_timeline_size.py @@ -4,9 +4,8 @@ import psycopg2.extras from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver from fixtures.log_helper import log -def test_timeline_size( - zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin -): + +def test_timeline_size(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin): # Branch at the point where only 100 rows were inserted zenith_cli.run(["branch", "test_timeline_size", "empty"]) @@ -23,13 +22,11 @@ def test_timeline_size( # Create table, and insert the first 100 rows cur.execute("CREATE TABLE foo (t text)") - cur.execute( - """ + cur.execute(""" INSERT INTO foo SELECT 'long string to consume some space' || g FROM generate_series(1, 10) g - """ - ) + """) res = client.branch_detail(UUID(pageserver.initial_tenant), "test_timeline_size") assert res["current_logical_size"] == res["current_logical_size_non_incremental"] diff --git a/test_runner/batch_others/test_twophase.py b/test_runner/batch_others/test_twophase.py index a6315fed15..bc6ee076c1 100644 --- a/test_runner/batch_others/test_twophase.py +++ b/test_runner/batch_others/test_twophase.py @@ -9,7 +9,10 @@ pytest_plugins = ("fixtures.zenith_fixtures") # # Test branching, when a transaction is in prepared state # -def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin: PgBin): +def test_twophase(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + pg_bin: PgBin): zenith_cli.run(["branch", "test_twophase", "empty"]) pg = postgres.create_start('test_twophase', config_lines=['max_prepared_transactions=5']) @@ -79,8 +82,8 @@ def test_twophase(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFa cur2.execute("ROLLBACK PREPARED 'insert_two'") cur2.execute('SELECT * FROM foo') - assert cur2.fetchall() == [('one',), ('three',)] + assert cur2.fetchall() == [('one', ), ('three', )] # Only one committed insert is visible on the original branch cur.execute('SELECT * FROM foo') - assert cur.fetchall() == [('three',)] + assert cur.fetchall() == [('three', )] diff --git a/test_runner/batch_others/test_vm_bits.py b/test_runner/batch_others/test_vm_bits.py index 86c56ddb9c..6f19940f2f 100644 --- a/test_runner/batch_others/test_vm_bits.py +++ b/test_runner/batch_others/test_vm_bits.py @@ -3,11 +3,16 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") + # # Test that the VM bit is cleared correctly at a HEAP_DELETE and # HEAP_UPDATE record. # -def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, base_dir): +def test_vm_bit_clear(pageserver: ZenithPageserver, + postgres: PostgresFactory, + pg_bin, + zenith_cli, + base_dir): # Create a branch for us zenith_cli.run(["branch", "test_vm_bit_clear", "empty"]) pg = postgres.create_start('test_vm_bit_clear') @@ -49,13 +54,12 @@ def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, p ''') cur.execute('SELECT * FROM vmtest_delete WHERE id = 1') - assert(cur.fetchall() == []); + assert (cur.fetchall() == []) cur.execute('SELECT * FROM vmtest_update WHERE id = 1') - assert(cur.fetchall() == []); + assert (cur.fetchall() == []) cur.close() - # Check the same thing on the branch that we created right after the DELETE # # As of this writing, the code in smgrwrite() creates a full-page image whenever @@ -75,6 +79,6 @@ def test_vm_bit_clear(pageserver: ZenithPageserver, postgres: PostgresFactory, p ''') cur_new.execute('SELECT * FROM vmtest_delete WHERE id = 1') - assert(cur_new.fetchall() == []); + assert (cur_new.fetchall() == []) cur_new.execute('SELECT * FROM vmtest_update WHERE id = 1') - assert(cur_new.fetchall() == []); + assert (cur_new.fetchall() == []) diff --git a/test_runner/batch_others/test_wal_acceptor.py b/test_runner/batch_others/test_wal_acceptor.py index 263757e2e7..3eaadc78a6 100644 --- a/test_runner/batch_others/test_wal_acceptor.py +++ b/test_runner/batch_others/test_wal_acceptor.py @@ -16,7 +16,10 @@ pytest_plugins = ("fixtures.zenith_fixtures") # basic test, write something in setup with wal acceptors, ensure that commits # succeed and data is written -def test_normal_work(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory): +def test_normal_work(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + wa_factory): zenith_cli.run(["branch", "test_wal_acceptors_normal_work", "empty"]) wa_factory.start_n_new(3) pg = postgres.create_start('test_wal_acceptors_normal_work', @@ -34,7 +37,10 @@ def test_normal_work(zenith_cli, pageserver: ZenithPageserver, postgres: Postgre # Run page server and multiple acceptors, and multiple compute nodes running # against different timelines. -def test_many_timelines(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory): +def test_many_timelines(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + wa_factory): n_timelines = 2 wa_factory.start_n_new(3) @@ -66,7 +72,10 @@ def test_many_timelines(zenith_cli, pageserver: ZenithPageserver, postgres: Post # Check that dead minority doesn't prevent the commits: execute insert n_inserts # times, with fault_probability chance of getting a wal acceptor down or up # along the way. 2 of 3 are always alive, so the work keeps going. -def test_restarts(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory: WalAcceptorFactory): +def test_restarts(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + wa_factory: WalAcceptorFactory): fault_probability = 0.01 n_inserts = 1000 n_acceptors = 3 @@ -177,7 +186,11 @@ def stop_value(): # do inserts while concurrently getting up/down subsets of acceptors -def test_race_conditions(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, wa_factory, stop_value): +def test_race_conditions(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, + wa_factory, + stop_value): wa_factory.start_n_new(3) @@ -318,6 +331,5 @@ def test_timeline_status(zenith_cli, pageserver, postgres, wa_factory: WalAccept pg.stop().start() pg.safe_psql("insert into t values(10)") - epoch_after_reboot = wa_http_cli.timeline_status(tenant_id, - timeline_id).acceptor_epoch + epoch_after_reboot = wa_http_cli.timeline_status(tenant_id, timeline_id).acceptor_epoch assert epoch_after_reboot > epoch diff --git a/test_runner/batch_others/test_wal_acceptor_async.py b/test_runner/batch_others/test_wal_acceptor_async.py index b2faa9b985..a5d4191375 100644 --- a/test_runner/batch_others/test_wal_acceptor_async.py +++ b/test_runner/batch_others/test_wal_acceptor_async.py @@ -19,13 +19,16 @@ class BankClient(object): async def initdb(self): await self.conn.execute('DROP TABLE IF EXISTS bank_accs') await self.conn.execute('CREATE TABLE bank_accs(uid int primary key, amount int)') - await self.conn.execute(''' + await self.conn.execute( + ''' INSERT INTO bank_accs SELECT *, $1 FROM generate_series(0, $2) - ''', self.init_amount, self.n_accounts - 1) + ''', + self.init_amount, + self.n_accounts - 1) await self.conn.execute('DROP TABLE IF EXISTS bank_log') await self.conn.execute('CREATE TABLE bank_log(from_uid int, to_uid int, amount int)') - + # TODO: Remove when https://github.com/zenithdb/zenith/issues/644 is fixed await self.conn.execute('ALTER TABLE bank_accs SET (autovacuum_enabled = false)') await self.conn.execute('ALTER TABLE bank_log SET (autovacuum_enabled = false)') @@ -34,6 +37,7 @@ class BankClient(object): row = await self.conn.fetchrow('SELECT sum(amount) AS sum FROM bank_accs') assert row['sum'] == self.n_accounts * self.init_amount + async def bank_transfer(conn: asyncpg.Connection, from_uid, to_uid, amount): # avoid deadlocks by sorting uids if from_uid > to_uid: @@ -42,16 +46,22 @@ async def bank_transfer(conn: asyncpg.Connection, from_uid, to_uid, amount): async with conn.transaction(): await conn.execute( 'UPDATE bank_accs SET amount = amount + ($1) WHERE uid = $2', - amount, to_uid, + amount, + to_uid, ) await conn.execute( 'UPDATE bank_accs SET amount = amount - ($1) WHERE uid = $2', - amount, from_uid, + amount, + from_uid, ) - await conn.execute('INSERT INTO bank_log VALUES ($1, $2, $3)', - from_uid, to_uid, amount, + await conn.execute( + 'INSERT INTO bank_log VALUES ($1, $2, $3)', + from_uid, + to_uid, + amount, ) + class WorkerStats(object): def __init__(self, n_workers): self.counters = [0] * n_workers @@ -114,7 +124,6 @@ async def run_restarts_under_load(pg: Postgres, acceptors: List[WalAcceptor], n_ worker = run_random_worker(stats, pg, worker_id, bank.n_accounts, max_transfer) workers.append(asyncio.create_task(worker)) - for it in range(iterations): victim = acceptors[it % len(acceptors)] victim.stop() @@ -122,10 +131,7 @@ async def run_restarts_under_load(pg: Postgres, acceptors: List[WalAcceptor], n_ # Wait till previous victim recovers so it is ready for the next # iteration by making any writing xact. conn = await pg.connect_async() - await conn.execute( - 'UPDATE bank_accs SET amount = amount WHERE uid = 1', - timeout=120 - ) + await conn.execute('UPDATE bank_accs SET amount = amount WHERE uid = 1', timeout=120) await conn.close() stats.reset() @@ -145,7 +151,9 @@ async def run_restarts_under_load(pg: Postgres, acceptors: List[WalAcceptor], n_ # restart acceptors one by one, while executing and validating bank transactions -def test_restarts_under_load(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, +def test_restarts_under_load(zenith_cli, + pageserver: ZenithPageserver, + postgres: PostgresFactory, wa_factory: WalAcceptorFactory): wa_factory.start_n_new(3) diff --git a/test_runner/batch_others/test_zenith_cli.py b/test_runner/batch_others/test_zenith_cli.py index be9e2b07fd..7379cf2981 100644 --- a/test_runner/batch_others/test_zenith_cli.py +++ b/test_runner/batch_others/test_zenith_cli.py @@ -23,8 +23,11 @@ def helper_compare_branch_list(page_server_cur, zenith_cli, initial_tenant: str) res = zenith_cli.run(["branch", f"--tenantid={initial_tenant}"]) res.check_returncode() - branches_cli_with_tenant_arg = sorted(map(lambda b: b.split(':')[-1].strip(), res.stdout.strip().split("\n"))) - branches_cli_with_tenant_arg = [b for b in branches_cli if b.startswith('test_cli_') or b in ('empty', 'main')] + branches_cli_with_tenant_arg = sorted( + map(lambda b: b.split(':')[-1].strip(), res.stdout.strip().split("\n"))) + branches_cli_with_tenant_arg = [ + b for b in branches_cli if b.startswith('test_cli_') or b in ('empty', 'main') + ] assert branches_api == branches_cli == branches_cli_with_tenant_arg @@ -54,6 +57,7 @@ def test_cli_branch_list(pageserver: ZenithPageserver, zenith_cli): assert 'test_cli_branch_list_main' in branches_cli assert 'test_cli_branch_list_nested' in branches_cli + def helper_compare_tenant_list(page_server_cur, zenith_cli: ZenithCli): page_server_cur.execute(f'tenant_list') tenants_api = sorted(json.loads(page_server_cur.fetchone()[0])) diff --git a/test_runner/batch_pg_regress/test_isolation.py b/test_runner/batch_pg_regress/test_isolation.py index ae654401cc..0f215337be 100644 --- a/test_runner/batch_pg_regress/test_isolation.py +++ b/test_runner/batch_pg_regress/test_isolation.py @@ -6,8 +6,14 @@ from fixtures.zenith_fixtures import ZenithPageserver, PostgresFactory pytest_plugins = ("fixtures.zenith_fixtures") -def test_isolation(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir, - base_dir, capsys): +def test_isolation(pageserver: ZenithPageserver, + postgres: PostgresFactory, + pg_bin, + zenith_cli, + test_output_dir, + pg_distrib_dir, + base_dir, + capsys): # Create a branch for us zenith_cli.run(["branch", "test_isolation", "empty"]) diff --git a/test_runner/batch_pg_regress/test_pg_regress.py b/test_runner/batch_pg_regress/test_pg_regress.py index 6f61b77ebc..2fd7fee314 100644 --- a/test_runner/batch_pg_regress/test_pg_regress.py +++ b/test_runner/batch_pg_regress/test_pg_regress.py @@ -6,8 +6,14 @@ from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver, check_re pytest_plugins = ("fixtures.zenith_fixtures") -def test_pg_regress(pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir, - base_dir, capsys): +def test_pg_regress(pageserver: ZenithPageserver, + postgres: PostgresFactory, + pg_bin, + zenith_cli, + test_output_dir, + pg_distrib_dir, + base_dir, + capsys): # Create a branch for us zenith_cli.run(["branch", "test_pg_regress", "empty"]) diff --git a/test_runner/batch_pg_regress/test_zenith_regress.py b/test_runner/batch_pg_regress/test_zenith_regress.py index ab43f511ef..ca1422388e 100644 --- a/test_runner/batch_pg_regress/test_zenith_regress.py +++ b/test_runner/batch_pg_regress/test_zenith_regress.py @@ -7,8 +7,14 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures") -def test_zenith_regress(postgres: PostgresFactory, pg_bin, zenith_cli, test_output_dir, pg_distrib_dir, - base_dir, capsys, pageserver_port: PageserverPort): +def test_zenith_regress(postgres: PostgresFactory, + pg_bin, + zenith_cli, + test_output_dir, + pg_distrib_dir, + base_dir, + capsys, + pageserver_port: PageserverPort): # Create a branch for us zenith_cli.run(["branch", "test_zenith_regress", "empty"]) diff --git a/test_runner/fixtures/benchmark_fixture.py b/test_runner/fixtures/benchmark_fixture.py index c70ee36256..f41d66674d 100644 --- a/test_runner/fixtures/benchmark_fixture.py +++ b/test_runner/fixtures/benchmark_fixture.py @@ -24,7 +24,6 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, cast from typing_extensions import Literal from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture) - """ This file contains fixtures for micro-benchmarks. @@ -55,7 +54,6 @@ in the test initialization, or measure disk usage after the test query. """ - # All the results are collected in this list, as a tuple: # (test_name: str, metric_name: str, metric_value: float, unit: str) # @@ -65,6 +63,7 @@ in the test initialization, or measure disk usage after the test query. global zenbenchmark_results zenbenchmark_results = [] + class ZenithBenchmarkResults: """ An object for recording benchmark results. """ def __init__(self): @@ -77,6 +76,7 @@ class ZenithBenchmarkResults: self.results.append((test_name, metric_name, metric_value, unit)) + # Session scope fixture that initializes the results object @pytest.fixture(autouse=True, scope='session') def zenbenchmark_global(request) -> Iterator[ZenithBenchmarkResults]: @@ -88,6 +88,7 @@ def zenbenchmark_global(request) -> Iterator[ZenithBenchmarkResults]: yield zenbenchmark_results + class ZenithBenchmarker: """ An object for recording benchmark results. This is created for each test @@ -103,7 +104,6 @@ class ZenithBenchmarker: """ self.results.record(self.request.node.name, metric_name, metric_value, unit) - @contextmanager def record_duration(self, metric_name): """ @@ -134,7 +134,8 @@ class ZenithBenchmarker: # The metric should be an integer, as it's a number of bytes. But in general # all prometheus metrics are floats. So to be pedantic, read it as a float # and round to integer. - matches = re.search(r'^pageserver_disk_io_bytes{io_operation="write"} (\S+)$', all_metrics, + matches = re.search(r'^pageserver_disk_io_bytes{io_operation="write"} (\S+)$', + all_metrics, re.MULTILINE) return int(round(float(matches.group(1)))) @@ -145,8 +146,7 @@ class ZenithBenchmarker: # Fetch all the exposed prometheus metrics from page server all_metrics = pageserver.http_client().get_metrics() # See comment in get_io_writes() - matches = re.search(r'^pageserver_maxrss_kb (\S+)$', all_metrics, - re.MULTILINE) + matches = re.search(r'^pageserver_maxrss_kb (\S+)$', all_metrics, re.MULTILINE) return int(round(float(matches.group(1)))) def get_timeline_size(self, repo_dir: str, tenantid: str, timelineid: str): @@ -171,7 +171,11 @@ class ZenithBenchmarker: yield after = self.get_io_writes(pageserver) - self.results.record(self.request.node.name, metric_name, round((after - before) / (1024 * 1024)), 'MB') + self.results.record(self.request.node.name, + metric_name, + round((after - before) / (1024 * 1024)), + 'MB') + @pytest.fixture(scope='function') def zenbenchmark(zenbenchmark_global, request) -> Iterator[ZenithBenchmarker]: @@ -185,9 +189,7 @@ def zenbenchmark(zenbenchmark_global, request) -> Iterator[ZenithBenchmarker]: # Hook to print the results at the end @pytest.hookimpl(hookwrapper=True) -def pytest_terminal_summary( - terminalreporter: TerminalReporter, exitstatus: int, config: Config -): +def pytest_terminal_summary(terminalreporter: TerminalReporter, exitstatus: int, config: Config): yield global zenbenchmark_results diff --git a/test_runner/fixtures/log_helper.py b/test_runner/fixtures/log_helper.py index f253576e7b..cab7462a51 100644 --- a/test_runner/fixtures/log_helper.py +++ b/test_runner/fixtures/log_helper.py @@ -1,6 +1,5 @@ import logging import logging.config - """ This file configures logging to use in python tests. Logs are automatically captured and shown in their @@ -27,17 +26,19 @@ LOGGING = { "level": "INFO" }, "root.wal_acceptor_async": { - "level": "INFO" # a lot of logs on DEBUG level + "level": "INFO" # a lot of logs on DEBUG level } } } + def getLogger(name='root') -> logging.Logger: """Method to get logger for tests. Should be used to get correctly initialized logger. """ return logging.getLogger(name) + # default logger for tests log = getLogger() diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index dd90a36dbb..dbb1809a2b 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -4,6 +4,7 @@ import subprocess from typing import Any, List from fixtures.log_helper import log + def get_self_dir() -> str: """ Get the path to the directory where this script lives. """ return os.path.dirname(os.path.abspath(__file__)) @@ -58,6 +59,7 @@ def global_counter() -> int: _global_counter += 1 return _global_counter + def lsn_to_hex(num: int) -> str: """ Convert lsn from int to standard hex notation. """ return "{:X}/{:X}".format(num >> 32, num & 0xffffffff) diff --git a/test_runner/fixtures/zenith_fixtures.py b/test_runner/fixtures/zenith_fixtures.py index bd62ed4b15..868f14ab29 100644 --- a/test_runner/fixtures/zenith_fixtures.py +++ b/test_runner/fixtures/zenith_fixtures.py @@ -27,7 +27,6 @@ import requests from .utils import (get_self_dir, mkdir_if_needed, subprocess_capture) from fixtures.log_helper import log - """ This file contains pytest fixtures. A fixture is a test resource that can be summoned by placing its name in the test's arguments. @@ -55,14 +54,15 @@ DEFAULT_POSTGRES_DIR = 'tmp_install' BASE_PORT = 15000 WORKER_PORT_NUM = 100 + def pytest_configure(config): """ Ensure that no unwanted daemons are running before we start testing. Check that we do not owerflow available ports range. """ numprocesses = config.getoption('numprocesses') - if numprocesses is not None and BASE_PORT + numprocesses * WORKER_PORT_NUM > 32768: # do not use ephemeral ports - raise Exception('Too many workers configured. Cannot distrubute ports for services.') + if numprocesses is not None and BASE_PORT + numprocesses * WORKER_PORT_NUM > 32768: # do not use ephemeral ports + raise Exception('Too many workers configured. Cannot distrubute ports for services.') # does not use -c as it is not supported on macOS cmd = ['pgrep', 'pageserver|postgres|safekeeper'] @@ -106,7 +106,11 @@ class PgProtocol: self.port = port self.username = username or "zenith_admin" - def connstr(self, *, dbname: str = 'postgres', username: Optional[str] = None, password: Optional[str] = None) -> str: + def connstr(self, + *, + dbname: str = 'postgres', + username: Optional[str] = None, + password: Optional[str] = None) -> str: """ Build a libpq connection string for the Postgres instance. """ @@ -118,7 +122,12 @@ class PgProtocol: return f'{res} password={password}' # autocommit=True here by default because that's what we need most of the time - def connect(self, *, autocommit=True, dbname: str = 'postgres', username: Optional[str] = None, password: Optional[str] = None) -> PgConnection: + def connect(self, + *, + autocommit=True, + dbname: str = 'postgres', + username: Optional[str] = None, + password: Optional[str] = None) -> PgConnection: """ Connect to the node. Returns psycopg2's connection object. @@ -134,7 +143,11 @@ class PgProtocol: conn.autocommit = autocommit return conn - async def connect_async(self, *, dbname: str = 'postgres', username: Optional[str] = None, password: Optional[str] = None) -> asyncpg.Connection: + async def connect_async(self, + *, + dbname: str = 'postgres', + username: Optional[str] = None, + password: Optional[str] = None) -> asyncpg.Connection: """ Connect to the node from async python. Returns asyncpg's connection object. @@ -200,11 +213,11 @@ class ZenithCli: # Interceipt CalledProcessError and print more info try: res = subprocess.run(args, - env=self.env, - check=True, - universal_newlines=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + env=self.env, + check=True, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) except subprocess.CalledProcessError as exc: # this way command output will be in recorded and shown in CI in failure message msg = f"""\ @@ -242,21 +255,17 @@ class ZenithPageserverHttpClient(requests.Session): return res.json() def branch_create(self, tenant_id: uuid.UUID, name: str, start_point: str) -> Dict: - res = self.post( - f"http://localhost:{self.port}/v1/branch", - json={ - 'tenant_id': tenant_id.hex, - 'name': name, - 'start_point': start_point, - } - ) + res = self.post(f"http://localhost:{self.port}/v1/branch", + json={ + 'tenant_id': tenant_id.hex, + 'name': name, + 'start_point': start_point, + }) res.raise_for_status() return res.json() def branch_detail(self, tenant_id: uuid.UUID, name: str) -> Dict: - res = self.get( - f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}/{name}", - ) + res = self.get(f"http://localhost:{self.port}/v1/branch/{tenant_id.hex}/{name}", ) res.raise_for_status() return res.json() @@ -298,7 +307,11 @@ class AuthKeys: return token def generate_tenant_token(self, tenant_id): - token = jwt.encode({"scope": "tenant", "tenant_id": tenant_id}, self.priv, algorithm="RS256") + token = jwt.encode({ + "scope": "tenant", "tenant_id": tenant_id + }, + self.priv, + algorithm="RS256") if isinstance(token, bytes): token = token.decode() @@ -323,6 +336,7 @@ def worker_base_port(worker_seq_no: int): # so workers have disjoint set of ports for services return BASE_PORT + worker_seq_no * WORKER_PORT_NUM + class PortDistributor: def __init__(self, base_port: int, port_number: int) -> None: self.iterator = iter(range(base_port, base_port + port_number)) @@ -331,13 +345,15 @@ class PortDistributor: try: return next(self.iterator) except StopIteration: - raise RuntimeError('port range configured for test is exhausted, consider enlarging the range') + raise RuntimeError( + 'port range configured for test is exhausted, consider enlarging the range') @zenfixture def port_distributor(worker_base_port): return PortDistributor(base_port=worker_base_port, port_number=WORKER_PORT_NUM) + @dataclass class PageserverPort: pg: int @@ -352,14 +368,18 @@ class ZenithPageserver(PgProtocol): self.running = False self.initial_tenant = None self.repo_dir = repo_dir - self.service_port = port # do not shadow PgProtocol.port which is just int + self.service_port = port # do not shadow PgProtocol.port which is just int def init(self, enable_auth: bool = False) -> 'ZenithPageserver': """ Initialize the repository, i.e. run "zenith init". Returns self. """ - cmd = ['init', f'--pageserver-pg-port={self.service_port.pg}', f'--pageserver-http-port={self.service_port.http}'] + cmd = [ + 'init', + f'--pageserver-pg-port={self.service_port.pg}', + f'--pageserver-http-port={self.service_port.http}' + ] if enable_auth: cmd.append('--enable-auth') self.zenith_cli.run(cmd) @@ -419,8 +439,6 @@ class ZenithPageserver(PgProtocol): ) - - @zenfixture def pageserver_port(port_distributor: PortDistributor) -> PageserverPort: pg = port_distributor.get_port() @@ -430,7 +448,8 @@ def pageserver_port(port_distributor: PortDistributor) -> PageserverPort: @zenfixture -def pageserver(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: PageserverPort) -> Iterator[ZenithPageserver]: +def pageserver(zenith_cli: ZenithCli, repo_dir: str, + pageserver_port: PageserverPort) -> Iterator[ZenithPageserver]: """ The 'pageserver' fixture provides a Page Server that's up and running. @@ -442,7 +461,8 @@ def pageserver(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: Pageserver By convention, the test branches are named after the tests. For example, test called 'test_foo' would create and use branches with the 'test_foo' prefix. """ - ps = ZenithPageserver(zenith_cli=zenith_cli, repo_dir=repo_dir, port=pageserver_port).init().start() + ps = ZenithPageserver(zenith_cli=zenith_cli, repo_dir=repo_dir, + port=pageserver_port).init().start() # For convenience in tests, create a branch from the freshly-initialized cluster. zenith_cli.run(["branch", "empty", "main"]) @@ -452,6 +472,7 @@ def pageserver(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: Pageserver log.info('Starting pageserver cleanup') ps.stop(True) + class PgBin: """ A helper class for executing postgres binaries """ def __init__(self, log_dir: str, pg_distrib_dir: str): @@ -513,9 +534,11 @@ class PgBin: def pg_bin(test_output_dir: str, pg_distrib_dir: str) -> PgBin: return PgBin(test_output_dir, pg_distrib_dir) + @pytest.fixture def pageserver_auth_enabled(zenith_cli: ZenithCli, repo_dir: str, pageserver_port: PageserverPort): - with ZenithPageserver(zenith_cli=zenith_cli, repo_dir=repo_dir, port=pageserver_port).init(enable_auth=True).start() as ps: + with ZenithPageserver(zenith_cli=zenith_cli, repo_dir=repo_dir, + port=pageserver_port).init(enable_auth=True).start() as ps: # For convenience in tests, create a branch from the freshly-initialized cluster. zenith_cli.run(["branch", "empty", "main"]) yield ps @@ -523,14 +546,19 @@ def pageserver_auth_enabled(zenith_cli: ZenithCli, repo_dir: str, pageserver_por class Postgres(PgProtocol): """ An object representing a running postgres daemon. """ - def __init__(self, zenith_cli: ZenithCli, repo_dir: str, pg_bin: PgBin, tenant_id: str, port: int): + def __init__(self, + zenith_cli: ZenithCli, + repo_dir: str, + pg_bin: PgBin, + tenant_id: str, + port: int): super().__init__(host='localhost', port=port) self.zenith_cli = zenith_cli self.running = False self.repo_dir = repo_dir self.node_name: Optional[str] = None # dubious, see asserts below - self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA + self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA self.tenant_id = tenant_id self.pg_bin = pg_bin # path to conf is /pgdatadirs/tenants///postgresql.conf @@ -555,7 +583,14 @@ class Postgres(PgProtocol): if branch is None: branch = node_name - self.zenith_cli.run(['pg', 'create', f'--tenantid={self.tenant_id}', f'--port={self.port}', node_name, branch]) + self.zenith_cli.run([ + 'pg', + 'create', + f'--tenantid={self.tenant_id}', + f'--port={self.port}', + node_name, + branch + ]) self.node_name = node_name path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.node_name self.pgdata_dir = os.path.join(self.repo_dir, path) @@ -578,7 +613,8 @@ class Postgres(PgProtocol): log.info(f"Starting postgres node {self.node_name}") - run_result = self.zenith_cli.run(['pg', 'start', f'--tenantid={self.tenant_id}', f'--port={self.port}', self.node_name]) + run_result = self.zenith_cli.run( + ['pg', 'start', f'--tenantid={self.tenant_id}', f'--port={self.port}', self.node_name]) self.running = True log.info(f"stdout: {run_result.stdout}") @@ -658,7 +694,8 @@ class Postgres(PgProtocol): assert self.node_name is not None assert self.tenant_id is not None - self.zenith_cli.run(['pg', 'stop', '--destroy', self.node_name, f'--tenantid={self.tenant_id}']) + self.zenith_cli.run( + ['pg', 'stop', '--destroy', self.node_name, f'--tenantid={self.tenant_id}']) return self @@ -690,9 +727,15 @@ class Postgres(PgProtocol): def __exit__(self, exc_type, exc, tb): self.stop() + class PostgresFactory: """ An object representing multiple running postgres daemons. """ - def __init__(self, zenith_cli: ZenithCli, repo_dir: str, pg_bin: PgBin, initial_tenant: str, port_distributor: PortDistributor): + def __init__(self, + zenith_cli: ZenithCli, + repo_dir: str, + pg_bin: PgBin, + initial_tenant: str, + port_distributor: PortDistributor): self.zenith_cli = zenith_cli self.repo_dir = repo_dir self.num_instances = 0 @@ -701,14 +744,12 @@ class PostgresFactory: self.port_distributor = port_distributor self.pg_bin = pg_bin - def create_start( - self, - node_name: str = "main", - branch: Optional[str] = None, - tenant_id: Optional[str] = None, - wal_acceptors: Optional[str] = None, - config_lines: Optional[List[str]] = None - ) -> Postgres: + def create_start(self, + node_name: str = "main", + branch: Optional[str] = None, + tenant_id: Optional[str] = None, + wal_acceptors: Optional[str] = None, + config_lines: Optional[List[str]] = None) -> Postgres: pg = Postgres( zenith_cli=self.zenith_cli, @@ -727,14 +768,12 @@ class PostgresFactory: config_lines=config_lines, ) - def create( - self, - node_name: str = "main", - branch: Optional[str] = None, - tenant_id: Optional[str] = None, - wal_acceptors: Optional[str] = None, - config_lines: Optional[List[str]] = None - ) -> Postgres: + def create(self, + node_name: str = "main", + branch: Optional[str] = None, + tenant_id: Optional[str] = None, + wal_acceptors: Optional[str] = None, + config_lines: Optional[List[str]] = None) -> Postgres: pg = Postgres( zenith_cli=self.zenith_cli, @@ -754,13 +793,11 @@ class PostgresFactory: config_lines=config_lines, ) - def config( - self, - node_name: str = "main", - tenant_id: Optional[str] = None, - wal_acceptors: Optional[str] = None, - config_lines: Optional[List[str]] = None - ) -> Postgres: + def config(self, + node_name: str = "main", + tenant_id: Optional[str] = None, + wal_acceptors: Optional[str] = None, + config_lines: Optional[List[str]] = None) -> Postgres: pg = Postgres( zenith_cli=self.zenith_cli, @@ -785,13 +822,18 @@ class PostgresFactory: return self + @zenfixture def initial_tenant(pageserver: ZenithPageserver): return pageserver.initial_tenant @zenfixture -def postgres(zenith_cli: ZenithCli, initial_tenant: str, repo_dir: str, pg_bin: PgBin, port_distributor: PortDistributor) -> Iterator[PostgresFactory]: +def postgres(zenith_cli: ZenithCli, + initial_tenant: str, + repo_dir: str, + pg_bin: PgBin, + port_distributor: PortDistributor) -> Iterator[PostgresFactory]: pgfactory = PostgresFactory( zenith_cli=zenith_cli, repo_dir=repo_dir, @@ -806,6 +848,7 @@ def postgres(zenith_cli: ZenithCli, initial_tenant: str, repo_dir: str, pg_bin: log.info('Starting postgres cleanup') pgfactory.stop_all() + def read_pid(path: Path): """ Read content of file into number """ return int(path.read_text()) @@ -816,13 +859,14 @@ class WalAcceptorPort: pg: int http: int + @dataclass class WalAcceptor: """ An object representing a running wal acceptor daemon. """ wa_bin_path: Path data_dir: Path port: WalAcceptorPort - num: int # identifier for logging + num: int # identifier for logging pageserver_port: int auth_token: Optional[str] = None @@ -887,10 +931,11 @@ class WalAcceptor: os.kill(pid, signal.SIGTERM) except Exception: # TODO: cleanup pid file on exit in wal acceptor - pass # pidfile might be obsolete + pass # pidfile might be obsolete return self - def append_logical_message(self, tenant_id: str, timeline_id: str, request: Dict[str, Any]) -> Dict[str, Any]: + def append_logical_message(self, tenant_id: str, timeline_id: str, + request: Dict[str, Any]) -> Dict[str, Any]: """ Send JSON_CTRL query to append LogicalMessage to WAL and modify safekeeper state. It will construct LogicalMessage from provided @@ -918,7 +963,11 @@ class WalAcceptor: class WalAcceptorFactory: """ An object representing multiple running wal acceptors. """ - def __init__(self, zenith_binpath: Path, data_dir: Path, pageserver_port: int, port_distributor: PortDistributor): + def __init__(self, + zenith_binpath: Path, + data_dir: Path, + pageserver_port: int, + port_distributor: PortDistributor): self.wa_bin_path = zenith_binpath / 'safekeeper' self.data_dir = data_dir self.instances: List[WalAcceptor] = [] @@ -964,7 +1013,10 @@ class WalAcceptorFactory: @zenfixture -def wa_factory(zenith_binpath: str, repo_dir: str, pageserver_port: PageserverPort, port_distributor: PortDistributor) -> Iterator[WalAcceptorFactory]: +def wa_factory(zenith_binpath: str, + repo_dir: str, + pageserver_port: PageserverPort, + port_distributor: PortDistributor) -> Iterator[WalAcceptorFactory]: """ Gives WalAcceptorFactory providing wal acceptors. """ wafactory = WalAcceptorFactory( zenith_binpath=Path(zenith_binpath), @@ -977,10 +1029,12 @@ def wa_factory(zenith_binpath: str, repo_dir: str, pageserver_port: PageserverPo log.info('Starting wal acceptors cleanup') wafactory.stop_all() + @dataclass class PageserverTimelineStatus: acceptor_epoch: int + class WalAcceptorHttpClient(requests.Session): def __init__(self, port: int) -> None: super().__init__() @@ -1094,6 +1148,7 @@ class TenantFactory: def tenant_factory(zenith_cli: ZenithCli): return TenantFactory(zenith_cli) + # # Test helpers # @@ -1104,8 +1159,15 @@ def list_files_to_compare(pgdata_dir: str): rel_dir = os.path.relpath(root, pgdata_dir) # Skip some dirs and files we don't want to compare skip_dirs = ['pg_wal', 'pg_stat', 'pg_stat_tmp', 'pg_subtrans', 'pg_logical'] - skip_files = ['pg_internal.init', 'pg.log', 'zenith.signal', 'postgresql.conf', - 'postmaster.opts', 'postmaster.pid', 'pg_control'] + skip_files = [ + 'pg_internal.init', + 'pg.log', + 'zenith.signal', + 'postgresql.conf', + 'postmaster.opts', + 'postmaster.pid', + 'pg_control' + ] if rel_dir not in skip_dirs and filename not in skip_files: rel_file = os.path.join(rel_dir, filename) pgdata_files.append(rel_file) @@ -1114,8 +1176,12 @@ def list_files_to_compare(pgdata_dir: str): log.info(pgdata_files) return pgdata_files + # pg is the existing and running compute node, that we want to compare with a basebackup -def check_restored_datadir_content(zenith_cli: ZenithCli, test_output_dir: str, pg: Postgres, pageserver_pg_port: int): +def check_restored_datadir_content(zenith_cli: ZenithCli, + test_output_dir: str, + pg: Postgres, + pageserver_pg_port: int): # Get the timeline ID of our branch. We need it for the 'basebackup' command with closing(pg.connect()) as conn: diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py index 1effa56ee9..cf6fa03703 100644 --- a/test_runner/performance/test_bulk_insert.py +++ b/test_runner/performance/test_bulk_insert.py @@ -5,6 +5,7 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") + # # Run bulk INSERT test. # @@ -15,7 +16,12 @@ pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") # 3. Disk space used # 4. Peak memory usage # -def test_bulk_insert(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str): +def test_bulk_insert(postgres: PostgresFactory, + pageserver: ZenithPageserver, + pg_bin, + zenith_cli, + zenbenchmark, + repo_dir: str): # Create a branch for us zenith_cli.run(["branch", "test_bulk_insert", "empty"]) @@ -24,7 +30,7 @@ def test_bulk_insert(postgres: PostgresFactory, pageserver: ZenithPageserver, pg # Open a connection directly to the page server that we'll use to force # flushing the layers to disk - psconn = pageserver.connect(); + psconn = pageserver.connect() pscur = psconn.cursor() # Get the timeline ID of our branch. We need it for the 'do_gc' command @@ -48,5 +54,7 @@ def test_bulk_insert(postgres: PostgresFactory, pageserver: ZenithPageserver, pg zenbenchmark.record("peak_mem", zenbenchmark.get_peak_mem(pageserver) / 1024, 'MB') # Report disk space used by the repository - timeline_size = zenbenchmark.get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) - zenbenchmark.record('size', timeline_size / (1024*1024), 'MB') + timeline_size = zenbenchmark.get_timeline_size(repo_dir, + pageserver.initial_tenant, + timeline) + zenbenchmark.record('size', timeline_size / (1024 * 1024), 'MB') diff --git a/test_runner/performance/test_bulk_tenant_create.py b/test_runner/performance/test_bulk_tenant_create.py index 3612189544..1e2a17c2c9 100644 --- a/test_runner/performance/test_bulk_tenant_create.py +++ b/test_runner/performance/test_bulk_tenant_create.py @@ -37,7 +37,9 @@ def test_bulk_tenant_create( tenant = tenant_factory.create() zenith_cli.run([ - "branch", f"test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}", "main", + "branch", + f"test_bulk_tenant_create_{tenants_count}_{i}_{use_wal_acceptors}", + "main", f"--tenantid={tenant}" ]) diff --git a/test_runner/performance/test_gist_build.py b/test_runner/performance/test_gist_build.py index b9ef0a3d4b..5a80978cf0 100644 --- a/test_runner/performance/test_gist_build.py +++ b/test_runner/performance/test_gist_build.py @@ -5,12 +5,18 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") + # # Test buffering GisT build. It WAL-logs the whole relation, in 32-page chunks. # As of this writing, we're duplicate those giant WAL records for each page, # which makes the delta layer about 32x larger than it needs to be. # -def test_gist_buffering_build(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str): +def test_gist_buffering_build(postgres: PostgresFactory, + pageserver: ZenithPageserver, + pg_bin, + zenith_cli, + zenbenchmark, + repo_dir: str): # Create a branch for us zenith_cli.run(["branch", "test_gist_buffering_build", "empty"]) @@ -19,7 +25,7 @@ def test_gist_buffering_build(postgres: PostgresFactory, pageserver: ZenithPages # Open a connection directly to the page server that we'll use to force # flushing the layers to disk - psconn = pageserver.connect(); + psconn = pageserver.connect() pscur = psconn.cursor() # Get the timeline ID of our branch. We need it for the 'do_gc' command @@ -29,13 +35,17 @@ def test_gist_buffering_build(postgres: PostgresFactory, pageserver: ZenithPages timeline = cur.fetchone()[0] # Create test table. - cur.execute("create table gist_point_tbl(id int4, p point)"); - cur.execute("insert into gist_point_tbl select g, point(g, g) from generate_series(1, 1000000) g;"); + cur.execute("create table gist_point_tbl(id int4, p point)") + cur.execute( + "insert into gist_point_tbl select g, point(g, g) from generate_series(1, 1000000) g;" + ) # Build the index. with zenbenchmark.record_pageserver_writes(pageserver, 'pageserver_writes'): with zenbenchmark.record_duration('build'): - cur.execute("create index gist_pointidx2 on gist_point_tbl using gist(p) with (buffering = on)"); + cur.execute( + "create index gist_pointidx2 on gist_point_tbl using gist(p) with (buffering = on)" + ) # Flush the layers from memory to disk. This is included in the reported # time and I/O @@ -45,5 +55,7 @@ def test_gist_buffering_build(postgres: PostgresFactory, pageserver: ZenithPages zenbenchmark.record("peak_mem", zenbenchmark.get_peak_mem(pageserver) / 1024, 'MB') # Report disk space used by the repository - timeline_size = zenbenchmark.get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) - zenbenchmark.record('size', timeline_size / (1024*1024), 'MB') + timeline_size = zenbenchmark.get_timeline_size(repo_dir, + pageserver.initial_tenant, + timeline) + zenbenchmark.record('size', timeline_size / (1024 * 1024), 'MB') diff --git a/test_runner/performance/test_perf_pgbench.py b/test_runner/performance/test_perf_pgbench.py index 22e8f2aee3..388ac4314c 100644 --- a/test_runner/performance/test_perf_pgbench.py +++ b/test_runner/performance/test_perf_pgbench.py @@ -5,6 +5,7 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") + # # Run a very short pgbench test. # @@ -14,7 +15,12 @@ pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") # 2. Time to run 5000 pgbench transactions # 3. Disk space used # -def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str): +def test_pgbench(postgres: PostgresFactory, + pageserver: ZenithPageserver, + pg_bin, + zenith_cli, + zenbenchmark, + repo_dir: str): # Create a branch for us zenith_cli.run(["branch", "test_pgbench_perf", "empty"]) @@ -23,7 +29,7 @@ def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin # Open a connection directly to the page server that we'll use to force # flushing the layers to disk - psconn = pageserver.connect(); + psconn = pageserver.connect() pscur = psconn.cursor() # Get the timeline ID of our branch. We need it for the 'do_gc' command @@ -53,4 +59,4 @@ def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin # Report disk space used by the repository timeline_size = zenbenchmark.get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) - zenbenchmark.record('size', timeline_size / (1024*1024), 'MB') + zenbenchmark.record('size', timeline_size / (1024 * 1024), 'MB') diff --git a/test_runner/performance/test_write_amplification.py b/test_runner/performance/test_write_amplification.py index 8410499bd2..1a1cc7bf21 100644 --- a/test_runner/performance/test_write_amplification.py +++ b/test_runner/performance/test_write_amplification.py @@ -17,7 +17,13 @@ from fixtures.log_helper import log pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") -def test_write_amplification(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin, zenith_cli, zenbenchmark, repo_dir: str): + +def test_write_amplification(postgres: PostgresFactory, + pageserver: ZenithPageserver, + pg_bin, + zenith_cli, + zenbenchmark, + repo_dir: str): # Create a branch for us zenith_cli.run(["branch", "test_write_amplification", "empty"]) @@ -26,7 +32,7 @@ def test_write_amplification(postgres: PostgresFactory, pageserver: ZenithPagese # Open a connection directly to the page server that we'll use to force # flushing the layers to disk - psconn = pageserver.connect(); + psconn = pageserver.connect() pscur = psconn.cursor() with closing(pg.connect()) as conn: @@ -71,5 +77,7 @@ def test_write_amplification(postgres: PostgresFactory, pageserver: ZenithPagese pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") # Report disk space used by the repository - timeline_size = zenbenchmark.get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) - zenbenchmark.record('size', timeline_size / (1024*1024), 'MB') + timeline_size = zenbenchmark.get_timeline_size(repo_dir, + pageserver.initial_tenant, + timeline) + zenbenchmark.record('size', timeline_size / (1024 * 1024), 'MB') diff --git a/test_runner/setup.cfg b/test_runner/setup.cfg index 578cb28efc..cff4c7f86e 100644 --- a/test_runner/setup.cfg +++ b/test_runner/setup.cfg @@ -10,6 +10,7 @@ max-line-length = 100 [yapf] based_on_style = pep8 column_limit = 100 +split_all_top_level_comma_separated_values = true [mypy] # some tests don't typecheck when this flag is set From e42c884c2bb782ae890b62711185a81ca1c0dc72 Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Wed, 20 Oct 2021 01:55:49 +0300 Subject: [PATCH 46/46] test_runner/README: add note on capturing logs (#778) Became actual after #674 --- test_runner/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/README.md b/test_runner/README.md index e4bbff053d..cdbf7e988d 100644 --- a/test_runner/README.md +++ b/test_runner/README.md @@ -53,8 +53,8 @@ Useful environment variables: should go. `TEST_SHARED_FIXTURES`: Try to re-use a single pageserver for all the tests. -Let stdout and stderr go to the terminal instead of capturing them: -`pytest -s ...` +Let stdout, stderr and `INFO` log messages go to the terminal instead of capturing them: +`pytest -s --log-cli-level=INFO ...` (Note many tests capture subprocess outputs separately, so this may not show much.)