From b08a0ee186896c33fdc6907e05175bcee99d6cdb Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 17 Oct 2023 09:55:39 +0200 Subject: [PATCH 01/49] walredo: fix race condition where shutdown kills the wrong process (#5557) Before this PR, the following race condition existed: ``` T1: does the apply_wal_records() call and gets back an error T2: does the apply_wal_records() call and gets back an error T2: does the kill_and_shutdown T2: new loop iteration T2: launches new walredo process T1: does the kill_and_shutdown of the new process ``` That last step is wrong, T2 already did the kill_and_shutdown. The symptom of this race condition was that T2 would observe an error when it tried to do something with the process after T1 killed it. For example, but not limited to: `POLLHUP` / `"WAL redo process closed its stderr unexpectedly"`. The fix in this PR is the following: * Use Arc to represent walredo processes. The Arc lives at least as long as the walredo process. * Use Arc::ptr_eq to determine whether to kill the process or not. The price is an additional RwLock to protect the new `redo_process` field that holds the Arc. I guess that could perhaps be an atomic pointer swap some day. But, let's get one race fixed without risking introducing a new one. The use of Arc/drop is also not super great here because it now allows for an unlimited number of to-be-killed processes to exist concurrently. See the various `NB` comments above `drop(proc)` for why it's "ok" right now due to the blocking `wait` inside `drop`. Note: an earlier fix attempt was https://github.com/neondatabase/neon/pull/5545 where we apply_batch_postgres would compare stdout_fd for equality. That's incorrect because the kernel can reuse the file descriptor when T2 launches the new process. Details: https://github.com/neondatabase/neon/pull/5545#pullrequestreview-1676589373 --- pageserver/src/walredo.rs | 219 +++++++++++++++++++++----------------- 1 file changed, 122 insertions(+), 97 deletions(-) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 28e5f997bb..7056ef4f90 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -26,13 +26,12 @@ use serde::Serialize; use std::collections::VecDeque; use std::io; use std::io::prelude::*; -use std::io::{Error, ErrorKind}; use std::ops::{Deref, DerefMut}; use std::os::unix::io::{AsRawFd, RawFd}; use std::os::unix::prelude::CommandExt; use std::process::Stdio; use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; -use std::sync::{Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use std::time::Duration; use std::time::Instant; use tracing::*; @@ -93,7 +92,6 @@ pub trait WalRedoManager: Send + Sync { } struct ProcessInput { - child: NoLeakChild, stdin: ChildStdin, stderr_fd: RawFd, stdout_fd: RawFd, @@ -116,13 +114,7 @@ struct ProcessOutput { pub struct PostgresRedoManager { tenant_id: TenantId, conf: &'static PageServerConf, - /// Counter to separate same sized walredo inputs failing at the same millisecond. - #[cfg(feature = "testing")] - dump_sequence: AtomicUsize, - - stdout: Mutex>, - stdin: Mutex>, - stderr: Mutex>, + redo_process: RwLock>>, } /// Can this request be served by neon redo functions @@ -215,11 +207,7 @@ impl PostgresRedoManager { PostgresRedoManager { tenant_id, conf, - #[cfg(feature = "testing")] - dump_sequence: AtomicUsize::default(), - stdin: Mutex::new(None), - stdout: Mutex::new(None), - stderr: Mutex::new(None), + redo_process: RwLock::new(None), } } @@ -242,20 +230,38 @@ impl PostgresRedoManager { let start_time = Instant::now(); let mut n_attempts = 0u32; loop { - let mut proc = self.stdin.lock().unwrap(); let lock_time = Instant::now(); // launch the WAL redo process on first use - if proc.is_none() { - self.launch(&mut proc, pg_version) - .context("launch process")?; - } + let proc: Arc = { + let proc_guard = self.redo_process.read().unwrap(); + match &*proc_guard { + None => { + // "upgrade" to write lock to launch the process + drop(proc_guard); + let mut proc_guard = self.redo_process.write().unwrap(); + match &*proc_guard { + None => { + let proc = Arc::new( + WalRedoProcess::launch(self.conf, self.tenant_id, pg_version) + .context("launch walredo process")?, + ); + *proc_guard = Some(Arc::clone(&proc)); + proc + } + Some(proc) => Arc::clone(proc), + } + } + Some(proc) => Arc::clone(proc), + } + }; + WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64()); // Relational WAL records are applied using wal-redo-postgres let buf_tag = BufferTag { rel, blknum }; - let result = self - .apply_wal_records(proc, buf_tag, &base_img, records, wal_redo_timeout) + let result = proc + .apply_wal_records(buf_tag, &base_img, records, wal_redo_timeout) .context("apply_wal_records"); let end_time = Instant::now(); @@ -296,22 +302,34 @@ impl PostgresRedoManager { n_attempts, e, ); - // self.stdin only holds stdin & stderr as_raw_fd(). - // Dropping it as part of take() doesn't close them. - // The owning objects (ChildStdout and ChildStderr) are stored in - // self.stdout and self.stderr, respsectively. - // We intentionally keep them open here to avoid a race between - // currently running `apply_wal_records()` and a `launch()` call - // after we return here. - // The currently running `apply_wal_records()` must not read from - // the newly launched process. - // By keeping self.stdout and self.stderr open here, `launch()` will - // get other file descriptors for the new child's stdout and stderr, - // and hence the current `apply_wal_records()` calls will observe - // `output.stdout.as_raw_fd() != stdout_fd` . - if let Some(proc) = self.stdin.lock().unwrap().take() { - proc.child.kill_and_wait(); + // Avoid concurrent callers hitting the same issue. + // We can't prevent it from happening because we want to enable parallelism. + let mut guard = self.redo_process.write().unwrap(); + match &*guard { + Some(current_field_value) => { + if Arc::ptr_eq(current_field_value, &proc) { + // We're the first to observe an error from `proc`, it's our job to take it out of rotation. + *guard = None; + } + } + None => { + // Another thread was faster to observe the error, and already took the process out of rotation. + } } + drop(guard); + // NB: there may still be other concurrent threads using `proc`. + // The last one will send SIGKILL when the underlying Arc reaches refcount 0. + // NB: it's important to drop(proc) after drop(guard). Otherwise we'd keep + // holding the lock while waiting for the process to exit. + // NB: the drop impl blocks the current threads with a wait() system call for + // the child process. We dropped the `guard` above so that other threads aren't + // affected. But, it's good that the current thread _does_ block to wait. + // If we instead deferred the waiting into the background / to tokio, it could + // happen that if walredo always fails immediately, we spawn processes faster + // than we can SIGKILL & `wait` for them to exit. By doing it the way we do here, + // we limit this risk of run-away to at most $num_runtimes * $num_executor_threads. + // This probably needs revisiting at some later point. + drop(proc); } else if n_attempts != 0 { info!(n_attempts, "retried walredo succeeded"); } @@ -614,24 +632,32 @@ impl CloseFileDescriptors for C { } } -impl PostgresRedoManager { +struct WalRedoProcess { + #[allow(dead_code)] + conf: &'static PageServerConf, + tenant_id: TenantId, + // Some() on construction, only becomes None on Drop. + child: Option, + stdout: Mutex, + stdin: Mutex, + stderr: Mutex, + /// Counter to separate same sized walredo inputs failing at the same millisecond. + #[cfg(feature = "testing")] + dump_sequence: AtomicUsize, +} + +impl WalRedoProcess { // // Start postgres binary in special WAL redo mode. // - #[instrument(skip_all,fields(tenant_id=%self.tenant_id, pg_version=pg_version))] + #[instrument(skip_all,fields(tenant_id=%tenant_id, pg_version=pg_version))] fn launch( - &self, - input: &mut MutexGuard>, + conf: &'static PageServerConf, + tenant_id: TenantId, pg_version: u32, - ) -> Result<(), Error> { - let pg_bin_dir_path = self - .conf - .pg_bin_dir(pg_version) - .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_bin_dir path: {e}")))?; - let pg_lib_dir_path = self - .conf - .pg_lib_dir(pg_version) - .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_lib_dir path: {e}")))?; + ) -> anyhow::Result { + let pg_bin_dir_path = conf.pg_bin_dir(pg_version).context("pg_bin_dir")?; // TODO these should be infallible. + let pg_lib_dir_path = conf.pg_lib_dir(pg_version).context("pg_lib_dir")?; // Start postgres itself let child = Command::new(pg_bin_dir_path.join("postgres")) @@ -652,13 +678,8 @@ impl PostgresRedoManager { // as close-on-exec by default, but that's not enough, since we use // libraries that directly call libc open without setting that flag. .close_fds() - .spawn_no_leak_child(self.tenant_id) - .map_err(|e| { - Error::new( - e.kind(), - format!("postgres --wal-redo command failed to start: {}", e), - ) - })?; + .spawn_no_leak_child(tenant_id) + .context("spawn process")?; let mut child = scopeguard::guard(child, |child| { error!("killing wal-redo-postgres process due to a problem during launch"); @@ -685,36 +706,47 @@ impl PostgresRedoManager { // all fallible operations post-spawn are complete, so get rid of the guard let child = scopeguard::ScopeGuard::into_inner(child); - **input = Some(ProcessInput { - child, - stdout_fd: stdout.as_raw_fd(), - stderr_fd: stderr.as_raw_fd(), - stdin, - n_requests: 0, - }); + Ok(Self { + conf, + tenant_id, + child: Some(child), + stdin: Mutex::new(ProcessInput { + stdout_fd: stdout.as_raw_fd(), + stderr_fd: stderr.as_raw_fd(), + stdin, + n_requests: 0, + }), + stdout: Mutex::new(ProcessOutput { + stdout, + pending_responses: VecDeque::new(), + n_processed_responses: 0, + }), + stderr: Mutex::new(stderr), + #[cfg(feature = "testing")] + dump_sequence: AtomicUsize::default(), + }) + } - *self.stdout.lock().unwrap() = Some(ProcessOutput { - stdout, - pending_responses: VecDeque::new(), - n_processed_responses: 0, - }); - *self.stderr.lock().unwrap() = Some(stderr); - - Ok(()) + fn id(&self) -> u32 { + self.child + .as_ref() + .expect("must not call this during Drop") + .id() } // Apply given WAL records ('records') over an old page image. Returns // new page image. // - #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%input.as_ref().unwrap().child.id()))] + #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%self.id()))] fn apply_wal_records( &self, - input: MutexGuard>, tag: BufferTag, base_img: &Option, records: &[(Lsn, NeonWalRecord)], wal_redo_timeout: Duration, ) -> anyhow::Result { + let input = self.stdin.lock().unwrap(); + // Serialize all the messages to send the WAL redo process first. // // This could be problematic if there are millions of records to replay, @@ -757,18 +789,17 @@ impl PostgresRedoManager { fn apply_wal_records0( &self, writebuf: &[u8], - mut input: MutexGuard>, + input: MutexGuard, wal_redo_timeout: Duration, ) -> anyhow::Result { - let proc = input.as_mut().unwrap(); + let mut proc = { input }; // TODO: remove this legacy rename, but this keep the patch small. let mut nwrite = 0usize; - let stdout_fd = proc.stdout_fd; // Prepare for calling poll() let mut pollfds = [ PollFd::new(proc.stdin.as_raw_fd(), PollFlags::POLLOUT), PollFd::new(proc.stderr_fd, PollFlags::POLLIN), - PollFd::new(stdout_fd, PollFlags::POLLIN), + PollFd::new(proc.stdout_fd, PollFlags::POLLIN), ]; // We do two things simultaneously: send the old base image and WAL records to @@ -790,8 +821,7 @@ impl PostgresRedoManager { let err_revents = pollfds[1].revents().unwrap(); if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { let mut errbuf: [u8; 16384] = [0; 16384]; - let mut stderr_guard = self.stderr.lock().unwrap(); - let stderr = stderr_guard.as_mut().unwrap(); + let mut stderr = self.stderr.lock().unwrap(); let len = stderr.read(&mut errbuf)?; // The message might not be split correctly into lines here. But this is @@ -821,7 +851,7 @@ impl PostgresRedoManager { } let request_no = proc.n_requests; proc.n_requests += 1; - drop(input); + drop(proc); // To improve walredo performance we separate sending requests and receiving // responses. Them are protected by different mutexes (output and input). @@ -835,20 +865,7 @@ impl PostgresRedoManager { // pending responses ring buffer and truncate all empty elements from the front, // advancing processed responses number. - let mut output_guard = self.stdout.lock().unwrap(); - let output = output_guard.as_mut().unwrap(); - if output.stdout.as_raw_fd() != stdout_fd { - // If stdout file descriptor is changed then it means that walredo process is crashed and restarted. - // As far as ProcessInput and ProcessOutout are protected by different mutexes, - // it can happen that we send request to one process and waiting response from another. - // To prevent such situation we compare stdout file descriptors. - // As far as old stdout pipe is destroyed only after new one is created, - // it can not reuse the same file descriptor, so this check is safe. - // - // Cross-read this with the comment in apply_batch_postgres if result.is_err(). - // That's where we kill the child process. - anyhow::bail!("WAL redo process closed its stdout unexpectedly"); - } + let mut output = self.stdout.lock().unwrap(); let n_processed_responses = output.n_processed_responses; while n_processed_responses + output.pending_responses.len() <= request_no { // We expect the WAL redo process to respond with an 8k page image. We read it @@ -873,8 +890,7 @@ impl PostgresRedoManager { let err_revents = pollfds[1].revents().unwrap(); if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { let mut errbuf: [u8; 16384] = [0; 16384]; - let mut stderr_guard = self.stderr.lock().unwrap(); - let stderr = stderr_guard.as_mut().unwrap(); + let mut stderr = self.stderr.lock().unwrap(); let len = stderr.read(&mut errbuf)?; // The message might not be split correctly into lines here. But this is @@ -984,6 +1000,15 @@ impl PostgresRedoManager { fn record_and_log(&self, _: &[u8]) {} } +impl Drop for WalRedoProcess { + fn drop(&mut self) { + self.child + .take() + .expect("we only do this once") + .kill_and_wait(); + } +} + /// Wrapper type around `std::process::Child` which guarantees that the child /// will be killed and waited-for by this process before being dropped. struct NoLeakChild { From b06dffe3dc46330cd3a86a13b8ffd66cd75cdb49 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 17 Oct 2023 09:21:31 +0100 Subject: [PATCH 02/49] pageserver: fixes to `/location_config` API (#5548) ## Problem I found some issues with the `/location_config` API when writing new tests. ## Summary of changes - Calling the API with the "Detached" state is now idempotent. - `Tenant::spawn_attach` now takes a boolean to indicate whether to expect a marker file. Marker files are used in the old attach path, but not in the new location conf API. They aren't needed because in the New World, the choice of whether to attach via remote state ("attach") or to trust local state ("load") will be revised to cope with the transitions between secondary & attached (see https://github.com/neondatabase/neon/issues/5550). It is okay to merge this change ahead of that ticket, because the API is not used in the wild yet. - Instead of using `schedule_local_tenant_processing`, the location conf API handler does its own directory creation and calls `spawn_attach` directly. - A new `unsafe_create_dir_all` is added. This differs from crashsafe::create_dir_all in two ways: - It is intentionally not crashsafe, because in the location conf API we are no longer using directory or config existence as the signal for any important business logic. - It is async and uses `tokio::fs`. --- pageserver/src/http/routes.rs | 12 ++- pageserver/src/tenant.rs | 41 +++++++---- pageserver/src/tenant/delete.rs | 5 +- pageserver/src/tenant/mgr.rs | 126 +++++++++++++++++++++++++------- 4 files changed, 143 insertions(+), 41 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 38304eb16b..c00dad50ac 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1034,9 +1034,17 @@ async fn put_tenant_location_config_handler( // The `Detached` state is special, it doesn't upsert a tenant, it removes // its local disk content and drops it from memory. if let LocationConfigMode::Detached = request_data.config.mode { - mgr::detach_tenant(conf, tenant_id, true, &state.deletion_queue_client) + if let Err(e) = mgr::detach_tenant(conf, tenant_id, true, &state.deletion_queue_client) .instrument(info_span!("tenant_detach", %tenant_id)) - .await?; + .await + { + match e { + TenantStateError::NotFound(_) => { + // This API is idempotent: a NotFound on a detach is fine. + } + _ => return Err(e.into()), + } + } return json_response(StatusCode::OK, ()); } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 88b2423eec..185d7954ea 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -420,6 +420,12 @@ pub enum CreateTimelineError { Other(#[from] anyhow::Error), } +/// spawn_attach argument for whether the caller is using attachment markers +pub(super) enum AttachMarkerMode { + Expect, + Ignore, +} + struct TenantDirectoryScan { sorted_timelines_to_load: Vec<(TimelineId, TimelineMetadata)>, timelines_to_resume_deletion: Vec<(TimelineId, Option)>, @@ -567,6 +573,7 @@ impl Tenant { resources: TenantSharedResources, attached_conf: AttachedTenantConf, tenants: &'static tokio::sync::RwLock, + expect_marker: AttachMarkerMode, ctx: &RequestContext, ) -> anyhow::Result> { // TODO dedup with spawn_load @@ -648,7 +655,7 @@ impl Tenant { } } - match tenant_clone.attach(&ctx).await { + match tenant_clone.attach(&ctx, expect_marker).await { Ok(()) => { info!("attach finished, activating"); tenant_clone.activate(broker_client, None, &ctx); @@ -673,17 +680,23 @@ impl Tenant { /// /// No background tasks are started as part of this routine. /// - async fn attach(self: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { + async fn attach( + self: &Arc, + ctx: &RequestContext, + expect_marker: AttachMarkerMode, + ) -> anyhow::Result<()> { span::debug_assert_current_span_has_tenant_id(); let marker_file = self.conf.tenant_attaching_mark_file_path(&self.tenant_id); - if !tokio::fs::try_exists(&marker_file) - .await - .context("check for existence of marker file")? - { - anyhow::bail!( - "implementation error: marker file should exist at beginning of this function" - ); + if let AttachMarkerMode::Expect = expect_marker { + if !tokio::fs::try_exists(&marker_file) + .await + .context("check for existence of marker file")? + { + anyhow::bail!( + "implementation error: marker file should exist at beginning of this function" + ); + } } // Get list of remote timelines @@ -805,10 +818,12 @@ impl Tenant { .map_err(LoadLocalTimelineError::ResumeDeletion)?; } - std::fs::remove_file(&marker_file) - .with_context(|| format!("unlink attach marker file {marker_file}"))?; - crashsafe::fsync(marker_file.parent().expect("marker file has parent dir")) - .context("fsync tenant directory after unlinking attach marker file")?; + if let AttachMarkerMode::Expect = expect_marker { + std::fs::remove_file(&marker_file) + .with_context(|| format!("unlink attach marker file {marker_file}"))?; + crashsafe::fsync(marker_file.parent().expect("marker file has parent dir")) + .context("fsync tenant directory after unlinking attach marker file")?; + } crate::failpoint_support::sleep_millis_async!("attach-before-activate"); diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 0db6213bbf..989f2cd779 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -458,7 +458,10 @@ impl DeleteTenantFlow { .await .expect("cant be stopping or broken"); - tenant.attach(ctx).await.context("attach")?; + tenant + .attach(ctx, super::AttachMarkerMode::Expect) + .await + .context("attach")?; Self::background( guard, diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 83ba49e1f5..3b6039db93 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -27,7 +27,8 @@ use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; use crate::tenant::delete::DeleteTenantFlow; use crate::tenant::{ - create_tenant_files, AttachedTenantConf, CreateTenantFilesMode, Tenant, TenantState, + create_tenant_files, AttachMarkerMode, AttachedTenantConf, CreateTenantFilesMode, Tenant, + TenantState, }; use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME, TEMP_FILE_SUFFIX}; @@ -151,6 +152,49 @@ async fn safe_rename_tenant_dir(path: impl AsRef) -> std::io::Result> = Lazy::new(|| RwLock::new(TenantsMap::Initializing)); +/// Create a directory, including parents. This does no fsyncs and makes +/// no guarantees about the persistence of the resulting metadata: for +/// use when creating dirs for use as cache. +async fn unsafe_create_dir_all(path: &Utf8PathBuf) -> std::io::Result<()> { + let mut dirs_to_create = Vec::new(); + let mut path: &Utf8Path = path.as_ref(); + + // Figure out which directories we need to create. + loop { + let meta = tokio::fs::metadata(path).await; + match meta { + Ok(metadata) if metadata.is_dir() => break, + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("non-directory found in path: {path}"), + )); + } + Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + + dirs_to_create.push(path); + + match path.parent() { + Some(parent) => path = parent, + None => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("can't find parent of path '{path}'"), + )); + } + } + } + + // Create directories from parent to child. + for &path in dirs_to_create.iter().rev() { + tokio::fs::create_dir(path).await?; + } + + Ok(()) +} + fn emergency_generations( tenant_confs: &HashMap>, ) -> HashMap { @@ -446,7 +490,15 @@ pub(crate) fn schedule_local_tenant_processing( "attaching mark file present but no remote storage configured".to_string(), ) } else { - match Tenant::spawn_attach(conf, tenant_id, resources, location_conf, tenants, ctx) { + match Tenant::spawn_attach( + conf, + tenant_id, + resources, + location_conf, + tenants, + AttachMarkerMode::Expect, + ctx, + ) { Ok(tenant) => tenant, Err(e) => { error!("Failed to spawn_attach tenant {tenant_id}, reason: {e:#}"); @@ -655,7 +707,7 @@ pub(crate) async fn set_new_tenant_config( Ok(()) } -#[instrument(skip_all, fields(tenant_id, new_location_config))] +#[instrument(skip_all, fields(%tenant_id))] pub(crate) async fn upsert_location( conf: &'static PageServerConf, tenant_id: TenantId, @@ -734,36 +786,61 @@ pub(crate) async fn upsert_location( } let new_slot = match &new_location_config.mode { - LocationMode::Secondary(_) => TenantSlot::Secondary, - LocationMode::Attached(_attach_config) => { - // Do a schedule_local_tenant_processing - // FIXME: should avoid doing this disk I/O inside the TenantsMap lock, - // we have the same problem in load_tenant/attach_tenant. Probably - // need a lock in TenantSlot to fix this. + LocationMode::Secondary(_) => { + let tenant_path = conf.tenant_path(&tenant_id); + // Directory doesn't need to be fsync'd because if we crash it can + // safely be recreated next time this tenant location is configured. + unsafe_create_dir_all(&tenant_path) + .await + .with_context(|| format!("Creating {tenant_path}"))?; + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) .await .map_err(SetNewTenantConfigError::Persist)?; - let tenant_path = conf.tenant_path(&tenant_id); - let resources = TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client, - }; - let new_tenant = schedule_local_tenant_processing( + + TenantSlot::Secondary + } + LocationMode::Attached(_attach_config) => { + // FIXME: should avoid doing this disk I/O inside the TenantsMap lock, + // we have the same problem in load_tenant/attach_tenant. Probably + // need a lock in TenantSlot to fix this. + let timelines_path = conf.timelines_path(&tenant_id); + + // Directory doesn't need to be fsync'd because we do not depend on + // it to exist after crashes: it may be recreated when tenant is + // re-attached, see https://github.com/neondatabase/neon/issues/5550 + unsafe_create_dir_all(&timelines_path) + .await + .with_context(|| format!("Creating {timelines_path}"))?; + + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; + + let tenant = match Tenant::spawn_attach( conf, tenant_id, - &tenant_path, + TenantSharedResources { + broker_client, + remote_storage, + deletion_queue_client, + }, AttachedTenantConf::try_from(new_location_config)?, - resources, - None, &TENANTS, + // The LocationConf API does not use marker files, because we have Secondary + // locations where the directory's existence is not a signal that it contains + // all timelines. See https://github.com/neondatabase/neon/issues/5550 + AttachMarkerMode::Ignore, ctx, - ) - .with_context(|| { - format!("Failed to schedule tenant processing in path {tenant_path:?}") - })?; + ) { + Ok(tenant) => tenant, + Err(e) => { + error!("Failed to spawn_attach tenant {tenant_id}, reason: {e:#}"); + Tenant::create_broken_tenant(conf, tenant_id, format!("{e:#}")) + } + }; - TenantSlot::Attached(new_tenant) + TenantSlot::Attached(tenant) } }; @@ -771,7 +848,6 @@ pub(crate) async fn upsert_location( }) .await?; } - Ok(()) } From 9e1449353da1a5def821a35fec800f39946e5925 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 17 Oct 2023 12:04:56 +0300 Subject: [PATCH 03/49] crash-consistent layer map through index_part.json (#5198) Fixes #5172 as it: - removes recoinciliation with remote index_part.json and accepts remote index_part.json as the truth, deleting any local progress which is yet to be reflected in remote - moves to prefer remote metadata Additionally: - tests with single LOCAL_FS parametrization are cleaned up - adds a test case for branched (non-bootstrap) local only timeline availability after restart --------- Co-authored-by: Christian Schwarz Co-authored-by: John Spray --- pageserver/src/tenant.rs | 225 +++++++++--------- .../src/tenant/storage_layer/filename.rs | 8 + pageserver/src/tenant/timeline.rs | 36 ++- pageserver/src/tenant/timeline/init.rs | 60 +++-- test_runner/regress/test_branching.py | 67 +++++- test_runner/regress/test_duplicate_layers.py | 41 ++-- test_runner/regress/test_gc_aggressive.py | 6 +- test_runner/regress/test_layer_eviction.py | 5 +- test_runner/regress/test_ondemand_download.py | 25 +- test_runner/regress/test_remote_storage.py | 46 ++-- test_runner/regress/test_tenant_detach.py | 34 +-- .../test_tenants_with_remote_storage.py | 7 +- 12 files changed, 289 insertions(+), 271 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 185d7954ea..0d987cfbfb 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -77,6 +77,7 @@ use crate::tenant::remote_timeline_client::MaybeDeletedIndexPart; use crate::tenant::storage_layer::DeltaLayer; use crate::tenant::storage_layer::ImageLayer; use crate::InitializationOrder; +use crate::METADATA_FILE_NAME; use crate::tenant::timeline::delete::DeleteTimelineFlow; use crate::tenant::timeline::uninit::cleanup_timeline_directory; @@ -246,72 +247,6 @@ pub struct Tenant { pub(crate) delete_progress: Arc>, } -// We should not blindly overwrite local metadata with remote one. -// For example, consider the following case: -// Image layer is flushed to disk as a new delta layer, we update local metadata and start upload task but after that -// pageserver crashes. During startup we'll load new metadata, and then reset it -// to the state of remote one. But current layermap will have layers from the old -// metadata which is inconsistent. -// And with current logic it wont disgard them during load because during layermap -// load it sees local disk consistent lsn which is ahead of layer lsns. -// If we treat remote as source of truth we need to completely sync with it, -// i e delete local files which are missing on the remote. This will add extra work, -// wal for these layers needs to be reingested for example -// -// So the solution is to take remote metadata only when we're attaching. -pub fn merge_local_remote_metadata<'a>( - local: Option<&'a TimelineMetadata>, - remote: Option<&'a TimelineMetadata>, -) -> anyhow::Result<(&'a TimelineMetadata, bool)> { - match (local, remote) { - (None, None) => anyhow::bail!("we should have either local metadata or remote"), - (Some(local), None) => Ok((local, true)), - // happens if we crash during attach, before writing out the metadata file - (None, Some(remote)) => Ok((remote, false)), - // This is the regular case where we crash/exit before finishing queued uploads. - // Also, it happens if we crash during attach after writing the metadata file - // but before removing the attaching marker file. - (Some(local), Some(remote)) => { - let consistent_lsn_cmp = local - .disk_consistent_lsn() - .cmp(&remote.disk_consistent_lsn()); - let gc_cutoff_lsn_cmp = local - .latest_gc_cutoff_lsn() - .cmp(&remote.latest_gc_cutoff_lsn()); - use std::cmp::Ordering::*; - match (consistent_lsn_cmp, gc_cutoff_lsn_cmp) { - // It wouldn't matter, but pick the local one so that we don't rewrite the metadata file. - (Equal, Equal) => Ok((local, true)), - // Local state is clearly ahead of the remote. - (Greater, Greater) => Ok((local, true)), - // We have local layer files that aren't on the remote, but GC horizon is on par. - (Greater, Equal) => Ok((local, true)), - // Local GC started running but we couldn't sync it to the remote. - (Equal, Greater) => Ok((local, true)), - - // We always update the local value first, so something else must have - // updated the remote value, probably a different pageserver. - // The control plane is supposed to prevent this from happening. - // Bail out. - (Less, Less) - | (Less, Equal) - | (Equal, Less) - | (Less, Greater) - | (Greater, Less) => { - anyhow::bail!( - r#"remote metadata appears to be ahead of local metadata: -local: - {local:#?} -remote: - {remote:#?} -"# - ); - } - } - } - } -} - #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GetTimelineError { #[error("Timeline {tenant_id}/{timeline_id} is not active, state: {state:?}")] @@ -375,11 +310,6 @@ impl Debug for SetStoppingError { } } -struct RemoteStartupData { - index_part: IndexPart, - remote_metadata: TimelineMetadata, -} - #[derive(Debug, thiserror::Error)] pub(crate) enum WaitToBecomeActiveError { WillNotBecomeActive { @@ -452,24 +382,17 @@ impl Tenant { &self, timeline_id: TimelineId, resources: TimelineResources, - remote_startup_data: Option, - local_metadata: Option, + index_part: Option, + metadata: TimelineMetadata, ancestor: Option>, init_order: Option<&InitializationOrder>, _ctx: &RequestContext, ) -> anyhow::Result<()> { let tenant_id = self.tenant_id; - let (up_to_date_metadata, picked_local) = merge_local_remote_metadata( - local_metadata.as_ref(), - remote_startup_data.as_ref().map(|r| &r.remote_metadata), - ) - .context("merge_local_remote_metadata")? - .to_owned(); - let timeline = self.create_timeline_struct( timeline_id, - up_to_date_metadata, + &metadata, ancestor.clone(), resources, init_order, @@ -482,20 +405,11 @@ impl Tenant { ); assert_eq!( disk_consistent_lsn, - up_to_date_metadata.disk_consistent_lsn(), + metadata.disk_consistent_lsn(), "these are used interchangeably" ); - // Save the metadata file to local disk. - if !picked_local { - save_metadata(self.conf, &tenant_id, &timeline_id, up_to_date_metadata) - .await - .context("save_metadata")?; - } - - let index_part = remote_startup_data.as_ref().map(|x| &x.index_part); - - if let Some(index_part) = index_part { + if let Some(index_part) = index_part.as_ref() { timeline .remote_client .as_ref() @@ -508,15 +422,12 @@ impl Tenant { // If control plane retries timeline creation in the meantime, the mgmt API handler // for timeline creation will coalesce on the upload we queue here. let rtc = timeline.remote_client.as_ref().unwrap(); - rtc.init_upload_queue_for_empty_remote(up_to_date_metadata)?; - rtc.schedule_index_upload_for_metadata_update(up_to_date_metadata)?; + rtc.init_upload_queue_for_empty_remote(&metadata)?; + rtc.schedule_index_upload_for_metadata_update(&metadata)?; } timeline - .load_layer_map( - disk_consistent_lsn, - remote_startup_data.map(|x| x.index_part), - ) + .load_layer_map(disk_consistent_lsn, index_part) .await .with_context(|| { format!("Failed to load layermap for timeline {tenant_id}/{timeline_id}") @@ -876,21 +787,23 @@ impl Tenant { None }; - // Even if there is local metadata it cannot be ahead of the remote one - // since we're attaching. Even if we resume interrupted attach remote one - // cannot be older than the local one - let local_metadata = None; + // we can load remote timelines during init, but they are assumed to be so rare that + // initialization order is not passed to here. + let init_order = None; + + // timeline loading after attach expects to find metadata file for each metadata + save_metadata(self.conf, &self.tenant_id, &timeline_id, &remote_metadata) + .await + .context("save_metadata") + .map_err(LoadLocalTimelineError::Load)?; self.timeline_init_and_sync( timeline_id, resources, - Some(RemoteStartupData { - index_part, - remote_metadata, - }), - local_metadata, + Some(index_part), + remote_metadata, ancestor, - None, + init_order, ctx, ) .await @@ -1364,8 +1277,8 @@ impl Tenant { LoadLocalTimelineError::Load(source) => { // We tried to load deleted timeline, this is a bug. return Err(anyhow::anyhow!(source).context( - "This is a bug. We tried to load deleted timeline which is wrong and loading failed. Timeline: {timeline_id}" - )); + format!("This is a bug. We tried to load deleted timeline which is wrong and loading failed. Timeline: {timeline_id}") + )); } LoadLocalTimelineError::ResumeDeletion(source) => { // Make sure resumed deletion wont fail loading for entire tenant. @@ -1399,6 +1312,11 @@ impl Tenant { let mut resources = self.build_timeline_resources(timeline_id); + struct RemoteStartupData { + index_part: IndexPart, + remote_metadata: TimelineMetadata, + } + let (remote_startup_data, remote_client) = match preload { Some(preload) => { let TimelinePreload { @@ -1454,7 +1372,7 @@ impl Tenant { ) } Err(DownloadError::NotFound) => { - info!("no index file was found on the remote, found_delete_mark: {found_delete_mark}"); + info!(found_delete_mark, "no index file was found on the remote, resuming deletion or cleaning unuploaded up"); if found_delete_mark { // We could've resumed at a point where remote index was deleted, but metadata file wasnt. @@ -1468,14 +1386,73 @@ impl Tenant { .map_err(LoadLocalTimelineError::ResumeDeletion); } - // We're loading fresh timeline that didnt yet make it into remote. - (None, Some(remote_client)) + // as the remote index_part.json did not exist, this timeline is a + // not-yet-uploaded one. it should be deleted now, because the branching might + // not have been valid as it's ancestor may have been restored to earlier state + // as well. in practice, control plane will keep retrying. + // + // first ensure that the un-uploaded timeline looks like it should, as in we + // are not accidentially deleting a timeline which was ever active: + // - root timelines have metadata and one possibly partial layer + // - branched timelines have metadata + // + // if the timeline does not look like expected, fail loading of the tenant. + // cleaning the timeline up manually and reloading the tenant is possible via + // the above log message. + let path = self.conf.timeline_path(&self.tenant_id, &timeline_id); + + let span = tracing::Span::current(); + + return tokio::task::spawn_blocking({ + move || { + use std::str::FromStr; + use crate::tenant::storage_layer::LayerFileName; + + let _e = span.entered(); + let mut metadata = false; + let mut layers = 0; + let mut others = 0; + for dentry in path.read_dir_utf8()? { + let dentry = dentry?; + let file_name = dentry.file_name(); + + if file_name == METADATA_FILE_NAME { + metadata = true; + continue; + } + + if LayerFileName::from_str(file_name).is_ok() + { + layers += 1; + continue; + } + + others += 1; + } + + // bootstrapped have the one image layer file, or one partial temp + // file, branched have just the metadata + if !(metadata && layers + others <= 1) { + anyhow::bail!("unexpected assumed unuploaded, never been active timeline: found metadata={}, layers={}, others={}", metadata, layers, others); + } + + let tmp_path = + path.with_file_name(format!("{timeline_id}{}", TEMP_FILE_SUFFIX)); + std::fs::rename(path, &tmp_path)?; + std::fs::remove_dir_all(&tmp_path)?; + Ok(()) + } + }) + .await + .map_err(anyhow::Error::new) + .and_then(|x| x) + .context("delete assumed unuploaded fresh timeline") + .map_err(LoadLocalTimelineError::Load); } Err(e) => return Err(LoadLocalTimelineError::Load(anyhow::Error::new(e))), } } None => { - // No remote client if found_delete_mark { // There is no remote client, we found local metadata. // Continue cleaning up local disk. @@ -1507,11 +1484,27 @@ impl Tenant { None }; + let (index_part, metadata) = match remote_startup_data { + Some(RemoteStartupData { + index_part, + remote_metadata, + }) => { + // always choose the remote metadata to be crash consistent (see RFC 27) + save_metadata(self.conf, &self.tenant_id, &timeline_id, &remote_metadata) + .await + .context("save_metadata") + .map_err(LoadLocalTimelineError::Load)?; + + (Some(index_part), remote_metadata) + } + None => (None, local_metadata), + }; + self.timeline_init_and_sync( timeline_id, resources, - remote_startup_data, - Some(local_metadata), + index_part, + metadata, ancestor, init_order, ctx, diff --git a/pageserver/src/tenant/storage_layer/filename.rs b/pageserver/src/tenant/storage_layer/filename.rs index 9fb0c23dd7..a98be0842b 100644 --- a/pageserver/src/tenant/storage_layer/filename.rs +++ b/pageserver/src/tenant/storage_layer/filename.rs @@ -226,6 +226,14 @@ impl LayerFileName { _ => false, } } + + pub(crate) fn kind(&self) -> &'static str { + use LayerFileName::*; + match self { + Delta(_) => "delta", + Image(_) => "image", + } + } } impl fmt::Display for LayerFileName { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 537f776176..6776d16d60 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1719,7 +1719,7 @@ impl Timeline { disk_consistent_lsn: Lsn, index_part: Option, ) -> anyhow::Result<()> { - use init::{Decision::*, Discovered, FutureLayer}; + use init::{Decision::*, Discovered, DismissedLayer}; use LayerFileName::*; let mut guard = self.layers.write().await; @@ -1735,7 +1735,7 @@ impl Timeline { // Copy to move into the task we're about to spawn let generation = self.generation; - let (loaded_layers, to_sync, total_physical_size) = tokio::task::spawn_blocking({ + let (loaded_layers, needs_cleanup, total_physical_size) = tokio::task::spawn_blocking({ move || { let _g = span.entered(); let discovered = init::scan_timeline_dir(&timeline_path)?; @@ -1784,7 +1784,6 @@ impl Timeline { ); let mut loaded_layers = Vec::new(); - let mut needs_upload = Vec::new(); let mut needs_cleanup = Vec::new(); let mut total_physical_size = 0; @@ -1805,7 +1804,7 @@ impl Timeline { } } Ok(decision) => decision, - Err(FutureLayer { local }) => { + Err(DismissedLayer::Future { local }) => { if local.is_some() { path.push(name.file_name()); init::cleanup_future_layer(&path, &name, disk_consistent_lsn)?; @@ -1814,6 +1813,13 @@ impl Timeline { needs_cleanup.push(name); continue; } + Err(DismissedLayer::LocalOnly(local)) => { + path.push(name.file_name()); + init::cleanup_local_only_file(&path, &name, &local)?; + path.pop(); + // this file never existed remotely, we will have to do rework + continue; + } }; match &name { @@ -1822,14 +1828,16 @@ impl Timeline { } let status = match &decision { - UseLocal(_) | NeedsUpload(_) => LayerResidenceStatus::Resident, + UseLocal(_) => LayerResidenceStatus::Resident, Evicted(_) | UseRemote { .. } => LayerResidenceStatus::Evicted, }; + tracing::debug!(layer=%name, ?decision, ?status, "applied"); + let stats = LayerAccessStats::for_loading_layer(status); let layer: Arc = match (name, &decision) { - (Delta(d), UseLocal(m) | NeedsUpload(m)) => { + (Delta(d), UseLocal(m)) => { total_physical_size += m.file_size(); Arc::new(DeltaLayer::new( conf, @@ -1840,7 +1848,7 @@ impl Timeline { stats, )) } - (Image(i), UseLocal(m) | NeedsUpload(m)) => { + (Image(i), UseLocal(m)) => { total_physical_size += m.file_size(); Arc::new(ImageLayer::new( conf, @@ -1859,17 +1867,9 @@ impl Timeline { ), }; - if let NeedsUpload(m) = decision { - needs_upload.push((layer.clone(), m)); - } - loaded_layers.push(layer); } - Ok(( - loaded_layers, - (needs_upload, needs_cleanup), - total_physical_size, - )) + Ok((loaded_layers, needs_cleanup, total_physical_size)) } }) .await @@ -1881,10 +1881,6 @@ impl Timeline { guard.initialize_local_layers(loaded_layers, disk_consistent_lsn + 1); if let Some(rtc) = self.remote_client.as_ref() { - let (needs_upload, needs_cleanup) = to_sync; - for (layer, m) in needs_upload { - rtc.schedule_layer_file_upload(&layer.layer_desc().filename(), &m)?; - } rtc.schedule_layer_file_deletion(needs_cleanup)?; rtc.schedule_index_upload_for_file_changes()?; // Tenant::create_timeline will wait for these uploads to happen before returning, or diff --git a/pageserver/src/tenant/timeline/init.rs b/pageserver/src/tenant/timeline/init.rs index 3902afe89a..96bf847fbb 100644 --- a/pageserver/src/tenant/timeline/init.rs +++ b/pageserver/src/tenant/timeline/init.rs @@ -72,7 +72,7 @@ pub(super) fn scan_timeline_dir(path: &Utf8Path) -> anyhow::Result, +pub(super) enum DismissedLayer { + /// The related layer is is in future compared to disk_consistent_lsn, it must not be loaded. + Future { + /// The local metadata. `None` if the layer is only known through [`IndexPart`]. + local: Option, + }, + /// The layer only exists locally. + /// + /// In order to make crash safe updates to layer map, we must dismiss layers which are only + /// found locally or not yet included in the remote `index_part.json`. + LocalOnly(LayerFileMetadata), } /// Merges local discoveries and remote [`IndexPart`] to a collection of decisions. -/// -/// This function should not gain additional reasons to fail than [`FutureLayer`], consider adding -/// the checks earlier to [`scan_timeline_dir`]. pub(super) fn reconcile( discovered: Vec<(LayerFileName, u64)>, index_part: Option<&IndexPart>, disk_consistent_lsn: Lsn, generation: Generation, -) -> Vec<(LayerFileName, Result)> { +) -> Vec<(LayerFileName, Result)> { use Decision::*; // name => (local, remote) @@ -142,17 +145,19 @@ pub(super) fn reconcile( .into_iter() .map(|(name, (local, remote))| { let decision = if name.is_in_future(disk_consistent_lsn) { - Err(FutureLayer { local }) + Err(DismissedLayer::Future { local }) } else { - Ok(match (local, remote) { - (Some(local), Some(remote)) if local != remote => UseRemote { local, remote }, - (Some(x), Some(_)) => UseLocal(x), - (None, Some(x)) => Evicted(x), - (Some(x), None) => NeedsUpload(x), + match (local, remote) { + (Some(local), Some(remote)) if local != remote => { + Ok(UseRemote { local, remote }) + } + (Some(x), Some(_)) => Ok(UseLocal(x)), + (None, Some(x)) => Ok(Evicted(x)), + (Some(x), None) => Err(DismissedLayer::LocalOnly(x)), (None, None) => { unreachable!("there must not be any non-local non-remote files") } - }) + } }; (name, decision) @@ -192,14 +197,21 @@ pub(super) fn cleanup_future_layer( name: &LayerFileName, disk_consistent_lsn: Lsn, ) -> anyhow::Result<()> { - use LayerFileName::*; - let kind = match name { - Delta(_) => "delta", - Image(_) => "image", - }; // future image layers are allowed to be produced always for not yet flushed to disk // lsns stored in InMemoryLayer. + let kind = name.kind(); tracing::info!("found future {kind} layer {name} disk_consistent_lsn is {disk_consistent_lsn}"); - crate::tenant::timeline::rename_to_backup(path)?; + std::fs::remove_file(path)?; + Ok(()) +} + +pub(super) fn cleanup_local_only_file( + path: &Utf8Path, + name: &LayerFileName, + local: &LayerFileMetadata, +) -> anyhow::Result<()> { + let kind = name.kind(); + tracing::info!("found local-only {kind} layer {name}, metadata {local:?}"); + std::fs::remove_file(path)?; Ok(()) } diff --git a/test_runner/regress/test_branching.py b/test_runner/regress/test_branching.py index 2541d5d475..32b1466c90 100644 --- a/test_runner/regress/test_branching.py +++ b/test_runner/regress/test_branching.py @@ -311,8 +311,8 @@ def test_competing_branchings_from_loading_race_to_ok_or_err(neon_env_builder: N assert isinstance(failed, Exception) assert isinstance(succeeded, Dict) - # FIXME: there's probably multiple valid status codes: - # - Timeline 62505b9a9f6b1d29117b1b74eaf07b12/56cd19d3b2dbcc65e9d53ec6ca304f24 already exists + # there's multiple valid status codes: + # - Timeline x/y already exists # - whatever 409 response says, but that is a subclass of PageserverApiException assert isinstance(failed, PageserverApiException) assert succeeded["state"] == "Active" @@ -320,17 +320,14 @@ def test_competing_branchings_from_loading_race_to_ok_or_err(neon_env_builder: N # we might still have the failpoint active env.pageserver.stop(immediate=True) - # pytest should nag if we leave threads unjoined for t in threads: t.join() create_root.join() -def test_non_uploaded_branch_availability_after_restart(neon_env_builder: NeonEnvBuilder): +def test_non_uploaded_root_timeline_is_deleted_after_restart(neon_env_builder: NeonEnvBuilder): """ - Currently before RFC#27 we keep and continue uploading branches which were not successfully uploaded before shutdown. - - This test likely duplicates some other test, but it's easier to write one than to make sure there will be a failing test when the rfc is implemented. + Check that a timeline is deleted locally on subsequent restart if it never successfully uploaded during creation. """ env = neon_env_builder.init_configs() @@ -366,9 +363,59 @@ def test_non_uploaded_branch_availability_after_restart(neon_env_builder: NeonEn wait_until_tenant_active(ps_http, env.initial_tenant) - # currently it lives on and will get eventually uploaded, but this will change - detail = ps_http.timeline_detail(env.initial_tenant, env.initial_timeline) - assert detail["state"] == "Active" + with pytest.raises(PageserverApiException, match="not found"): + ps_http.timeline_detail(env.initial_tenant, env.initial_timeline) + + +def test_non_uploaded_branch_is_deleted_after_restart(neon_env_builder: NeonEnvBuilder): + """ + Check that a timeline is deleted locally on subsequent restart if it never successfully uploaded during creation. + """ + + env = neon_env_builder.init_configs() + env.start() + + env.pageserver.allowed_errors.append( + ".*request{method=POST path=/v1/tenant/.*/timeline request_id=.*}: request was dropped before completing.*" + ) + ps_http = env.pageserver.http_client() + + ps_http.tenant_create(env.initial_tenant) + ps_http.timeline_create(env.pg_version, env.initial_tenant, env.initial_timeline) + + # pause all uploads + ps_http.configure_failpoints(("before-upload-index-pausable", "pause")) + branch_id = TimelineId.generate() + + def start_creating_timeline(): + with pytest.raises(RequestException): + ps_http.timeline_create( + env.pg_version, + env.initial_tenant, + branch_id, + ancestor_timeline_id=env.initial_timeline, + timeout=60, + ) + + t = threading.Thread(target=start_creating_timeline) + try: + t.start() + + wait_until_paused(env, "before-upload-index-pausable") + finally: + # FIXME: paused uploads bother shutdown + env.pageserver.stop(immediate=True) + t.join() + + # now without a failpoint + env.pageserver.start() + + wait_until_tenant_active(ps_http, env.initial_tenant) + + ps_http.timeline_detail(env.initial_tenant, env.initial_timeline) + + with pytest.raises(PageserverApiException, match="not found"): + ps_http.timeline_detail(env.initial_tenant, branch_id) def wait_until_paused(env: NeonEnv, failpoint: str): diff --git a/test_runner/regress/test_duplicate_layers.py b/test_runner/regress/test_duplicate_layers.py index ec5a2f7473..bcf99cae7c 100644 --- a/test_runner/regress/test_duplicate_layers.py +++ b/test_runner/regress/test_duplicate_layers.py @@ -1,9 +1,9 @@ import time import pytest -from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder, PgBin, wait_for_last_flush_lsn from fixtures.pageserver.utils import ( + wait_for_last_record_lsn, wait_for_upload_queue_empty, wait_until_tenant_active, ) @@ -41,9 +41,12 @@ def test_duplicate_layers(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): """ - This test sets fail point at the end of first compaction phase: - after flushing new L1 layers but before deletion of L0 layers - it should cause generation of duplicate L1 layer by compaction after restart. + Test sets fail point at the end of first compaction phase: after + flushing new L1 layer but before deletion of L0 layers. + + The L1 used to be overwritten, but with crash-consistency via remote + index_part.json, we end up deleting the not yet uploaded L1 layer on + startup. """ neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) @@ -65,7 +68,8 @@ def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) connstr = endpoint.connstr(options="-csynchronous_commit=off") pg_bin.run_capture(["pgbench", "-i", "-s1", connstr]) - wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + lsn = wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop() # make sure we receive no new wal after this, so that we'll write over the same L1 file. endpoint.stop() @@ -74,7 +78,7 @@ def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) # hit the exit failpoint with pytest.raises(ConnectionError, match="Remote end closed connection without response"): - pageserver_http.timeline_compact(tenant_id, timeline_id) + pageserver_http.timeline_checkpoint(tenant_id, timeline_id) env.pageserver.stop() # now the duplicate L1 has been created, but is not yet uploaded @@ -107,33 +111,32 @@ def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) l1_found = path assert l1_found is not None, "failed to find L1 locally" - original_created_at = l1_found.stat()[8] uploaded = env.pageserver_remote_storage.timeline_path(tenant_id, timeline_id) / l1_found.name assert not uploaded.exists(), "to-be-overwritten should not yet be uploaded" - # give room for fs timestamps - time.sleep(1) - env.pageserver.start() wait_until_tenant_active(pageserver_http, tenant_id) - message = f".*duplicated L1 layer layer={l1_found.name}" - env.pageserver.allowed_errors.append(message) + assert not l1_found.exists(), "partial compaction result should had been removed during startup" + + # wait for us to catch up again + wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, lsn) pageserver_http.timeline_compact(tenant_id, timeline_id) + # give time for log flush time.sleep(1) + message = f".*duplicated L1 layer layer={l1_found.name}" found_msg = env.pageserver.log_contains(message) - assert found_msg is not None, "no layer was duplicated, has this been fixed already?" + # resident or evicted, it should not be overwritten, however it should had been non-existing at startup + assert ( + found_msg is None + ), "layer should had been removed during startup, did it live on as evicted?" - log.info(f"found log line: {found_msg}") - - overwritten_at = l1_found.stat()[8] - assert original_created_at < overwritten_at, "expected the L1 to be overwritten" + assert l1_found.exists(), "the L1 reappears" wait_for_upload_queue_empty(pageserver_http, tenant_id, timeline_id) - uploaded_at = uploaded.stat()[8] - assert overwritten_at <= uploaded_at, "expected the L1 to finally be uploaded" + assert uploaded.exists(), "the L1 is uploaded" diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py index 017a38f85c..ef68049ee7 100644 --- a/test_runner/regress/test_gc_aggressive.py +++ b/test_runner/regress/test_gc_aggressive.py @@ -2,7 +2,6 @@ import asyncio import concurrent.futures import random -import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( Endpoint, @@ -95,13 +94,12 @@ def test_gc_aggressive(neon_env_builder: NeonEnvBuilder): # -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_gc_index_upload(neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind): +def test_gc_index_upload(neon_env_builder: NeonEnvBuilder): # Disable time-based pitr, we will use LSN-based thresholds in the manual GC calls neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}" num_index_uploads = 0 - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() tenant_id = env.initial_tenant diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index 3e23a8e165..3ab3700866 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -1,6 +1,5 @@ import time -import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, @@ -14,12 +13,10 @@ from fixtures.utils import query_scalar # Crates a few layers, ensures that we can evict them (removing locally but keeping track of them anyway) # and then download them back. -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_basic_eviction( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start( initial_tenant_conf={ diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index a38a517100..33de7445a4 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -306,12 +306,10 @@ def test_ondemand_download_timetravel( # # Ensure that the `download_remote_layers` API works # -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_download_remote_layers_api( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) ##### First start, insert data and upload it to the remote storage env = neon_env_builder.init_start( @@ -465,14 +463,11 @@ def test_download_remote_layers_api( assert query_scalar(cur, "select count(*) from testtab") == table_len -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.MOCK_S3]) -def test_compaction_downloads_on_demand_without_image_creation( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): +def test_compaction_downloads_on_demand_without_image_creation(neon_env_builder: NeonEnvBuilder): """ Create a few layers, then evict, then make sure compaction runs successfully. """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) conf = { # Disable background GC & compaction @@ -547,17 +542,14 @@ def test_compaction_downloads_on_demand_without_image_creation( assert post_compact[1] >= 3, "should had downloaded the three layers" -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.MOCK_S3]) -def test_compaction_downloads_on_demand_with_image_creation( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): +def test_compaction_downloads_on_demand_with_image_creation(neon_env_builder: NeonEnvBuilder): """ Create layers, compact with high image_creation_threshold, then run final compaction with all layers evicted. Due to current implementation, this will make image creation on-demand download layers, but we cannot really directly test for it. """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) conf = { # Disable background GC & compaction @@ -645,17 +637,14 @@ def test_compaction_downloads_on_demand_with_image_creation( assert dict(kinds_after) == {"Delta": 4, "Image": 1} -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_ondemand_download_failure_to_replace( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): +def test_ondemand_download_failure_to_replace(neon_env_builder: NeonEnvBuilder): """ Make sure that we fail on being unable to replace a RemoteLayer instead of for example livelocking. See: https://github.com/neondatabase/neon/issues/3533 """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) # disable gc and compaction via default tenant config because config is lost while detaching # so that compaction will not be the one to download the layer but the http handler is diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index f316b42d1c..ecd02c4c16 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -216,12 +216,10 @@ def test_remote_storage_backup_and_restore( # - Disable failpoints # - Wait for all uploads to finish # - Verify that remote is consistent and up-to-date (=all retries were done and succeeded) -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_remote_storage_upload_queue_retries( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() @@ -369,12 +367,10 @@ def test_remote_storage_upload_queue_retries( assert query_scalar(cur, "SELECT COUNT(*) FROM foo WHERE val = 'd'") == 20000 -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_remote_timeline_client_calls_started_metric( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) # thinking about using a shared environment? the test assumes that global # metrics are for single tenant. @@ -509,12 +505,10 @@ def test_remote_timeline_client_calls_started_metric( # Test that we correctly handle timeline with layers stuck in upload queue -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_timeline_deletion_with_files_stuck_in_upload_queue( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start( initial_tenant_conf={ @@ -624,12 +618,8 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( # Branches off a root branch, but does not write anything to the new branch, so it has a metadata file only. # Ensures that such branch is still persisted on the remote storage, and can be restored during tenant (re)attach. -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_empty_branch_remote_storage_upload( - neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_empty_branch_remote_storage_upload(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() client = env.pageserver.http_client() @@ -665,11 +655,7 @@ def test_empty_branch_remote_storage_upload( ), f"Expected to have same timelines after reattach, but got {timelines_after_detach}" -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_empty_branch_remote_storage_upload_on_restart( - neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, -): +def test_empty_branch_remote_storage_upload_on_restart(neon_env_builder: NeonEnvBuilder): """ Branches off a root branch, but does not write anything to the new branch, so it has a metadata file only. @@ -678,7 +664,7 @@ def test_empty_branch_remote_storage_upload_on_restart( — the upload should be scheduled by load, and create_timeline should await for it even though it gets 409 Conflict. """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() client = env.pageserver.http_client() @@ -731,6 +717,10 @@ def test_empty_branch_remote_storage_upload_on_restart( def create_in_background(): barrier.wait() try: + # retrying this kind of query makes no sense in real life as we do + # not lock in the lsn. with the immediate stop, we could in real + # life revert back the ancestor in startup, but most likely the lsn + # would still be branchable. client.timeline_create( tenant_id=env.initial_tenant, ancestor_timeline_id=env.initial_timeline, @@ -751,13 +741,13 @@ def test_empty_branch_remote_storage_upload_on_restart( assert not new_branch_on_remote_storage.exists(), "failpoint should had stopped uploading" client.configure_failpoints(("before-upload-index", "off")) - conflict = q.get() + exception = q.get() - assert conflict, "create_timeline should not have succeeded" assert ( - conflict.status_code == 409 - ), "timeline was created before restart, and uploads scheduled during initial load, so we expect 409 conflict" + exception is None + ), "create_timeline should have succeeded, because we deleted unuploaded local state" + # this is because creating a timeline always awaits for the uploads to complete assert_nothing_to_upload(client, env.initial_tenant, new_branch_timeline_id) assert ( @@ -767,15 +757,13 @@ def test_empty_branch_remote_storage_upload_on_restart( create_thread.join() -# Regression test for a race condition where files are compactified before the upload, +# Regression test for a race condition where L0 layers are compacted before the upload, # resulting in the uploading complaining about the file not being found # https://github.com/neondatabase/neon/issues/4526 -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_compaction_delete_before_upload( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start( initial_tenant_conf={ diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 519af1cbde..ec6f1258e5 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -18,7 +18,10 @@ from fixtures.pageserver.utils import ( wait_for_upload, wait_until_tenant_state, ) -from fixtures.remote_storage import RemoteStorageKind, available_remote_storages +from fixtures.remote_storage import ( + RemoteStorageKind, + available_remote_storages, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until from prometheus_client.samples import Sample @@ -585,11 +588,8 @@ def test_ignored_tenant_reattach(neon_env_builder: NeonEnvBuilder): # * `load` the same tenant # * ensure that it's status is `Active` # * check that timeline data is restored -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_ignored_tenant_download_missing_layers( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_ignored_tenant_download_missing_layers(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() endpoint = env.endpoints.create_start("main") @@ -648,11 +648,8 @@ def test_ignored_tenant_download_missing_layers( # * removes its `metadata` file locally # * `load` the same tenant # * ensure that it's status is `Broken` -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_ignored_tenant_stays_broken_without_metadata( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_ignored_tenant_stays_broken_without_metadata(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() env.endpoints.create_start("main") @@ -689,11 +686,8 @@ def test_ignored_tenant_stays_broken_without_metadata( # Tests that attach is never working on a tenant, ignored or not, as long as it's not absent locally # Similarly, tests that it's not possible to schedule a `load` for tenat that's not ignored. -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_load_attach_negatives( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_load_attach_negatives(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() env.endpoints.create_start("main") @@ -730,12 +724,10 @@ def test_load_attach_negatives( pageserver_http.tenant_attach(tenant_id) -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_ignore_while_attaching( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() @@ -834,12 +826,10 @@ def ensure_test_data(data_id: int, data: str, endpoint: Endpoint): ), "Should have timeline data back" -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_metrics_while_ignoring_broken_tenant_and_reloading( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index d42e566c36..30a36d0eca 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -108,11 +108,8 @@ def test_tenants_many(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Rem wait_for_upload(pageserver_http, tenant_id, timeline_id, current_lsn) -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_tenants_attached_after_download( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_tenants_attached_after_download(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) data_id = 1 data_secret = "very secret secret" From 9256788273d5661ced0b2661a8751e2aa86fbb59 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 17 Oct 2023 11:29:48 +0200 Subject: [PATCH 04/49] limit imitate accesses concurrency, using same semaphore as compactions (#5578) Before this PR, when we restarted pageserver, we'd see a rush of `$number_of_tenants` concurrent eviction tasks starting to do imitate accesses building up in the period of `[init_order allows activations, $random_access_delay + EvictionPolicyLayerAccessThreshold::period]`. We simply cannot handle that degree of concurrent IO. We already solved the problem for compactions by adding a semaphore. So, this PR shares that semaphore for use by evictions. Part of https://github.com/neondatabase/neon/issues/5479 Which is again part of https://github.com/neondatabase/neon/issues/4743 Risks / Changes In System Behavior ================================== * we don't do evictions as timely as we currently do * we log a bunch of warnings about eviction taking too long * imitate accesses and compactions compete for the same concurrency limit, so, they'll slow each other down through this shares semaphore Changes ======= - Move the `CONCURRENT_COMPACTIONS` semaphore into `tasks.rs` - Rename it to `CONCURRENT_BACKGROUND_TASKS` - Use it also for the eviction imitate accesses: - Imitate acceses are both per-TIMELINE and per-TENANT - The per-TENANT is done through coalescing all the per-TIMELINE tasks via a tokio mutex `eviction_task_tenant_state`. - We acquire the CONCURRENT_BACKGROUND_TASKS permit early, at the beginning of the eviction iteration, much before the imitate acesses start (and they may not even start at all in the given iteration, as they happen only every $threshold). - Acquiring early is **sub-optimal** because when the per-timline tasks coalesce on the `eviction_task_tenant_state` mutex, they are already holding a CONCURRENT_BACKGROUND_TASKS permit. - It's also unfair because tenants with many timelines win the CONCURRENT_BACKGROUND_TASKS more often. - I don't think there's another way though, without refactoring more of the imitate accesses logic, e.g, making it all per-tenant. - Add metrics for queue depth behind the semaphore. I found these very useful to understand what work is queued in the system. - The metrics are tagged by the new `BackgroundLoopKind`. - On a green slate, I would have used `TaskKind`, but we already had pre-existing labels whose names didn't map exactly to task kind. Also the task kind is kind of a lower-level detail, so, I think it's fine to have a separate enum to identify background work kinds. Future Work =========== I guess I could move the eviction tasks from a ticker to "sleep for $period". The benefit would be that the semaphore automatically "smears" the eviction task scheduling over time, so, we only have the rush on restart but a smeared-out rush afterward. The downside is that this perverts the meaning of "$period", as we'd actually not run the eviction at a fixed period. It also means the the "took to long" warning & metric becomes meaningless. Then again, that is already the case for the compaction and gc tasks, which do sleep for `$period` instead of using a ticker. --- pageserver/src/consumption_metrics.rs | 10 ++- pageserver/src/metrics.rs | 20 +++++ pageserver/src/tenant/tasks.rs | 81 +++++++++++++++++-- pageserver/src/tenant/timeline.rs | 39 +++------ .../src/tenant/timeline/eviction_task.rs | 18 ++++- 5 files changed, 131 insertions(+), 37 deletions(-) diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 72a2099d92..13f7977946 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -2,6 +2,7 @@ //! and push them to a HTTP endpoint. use crate::context::{DownloadBehavior, RequestContext}; use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}; +use crate::tenant::tasks::BackgroundLoopKind; use crate::tenant::{mgr, LogicalSizeCalculationCause}; use camino::Utf8PathBuf; use consumption_metrics::EventType; @@ -143,7 +144,7 @@ pub async fn collect_metrics( crate::tenant::tasks::warn_when_period_overrun( tick_at.elapsed(), metric_collection_interval, - "consumption_metrics_collect_metrics", + BackgroundLoopKind::ConsumptionMetricsCollectMetrics, ); } } @@ -268,6 +269,11 @@ async fn calculate_synthetic_size_worker( } if let Ok(tenant) = mgr::get_tenant(tenant_id, true).await { + // TODO should we use concurrent_background_tasks_rate_limit() here, like the other background tasks? + // We can put in some prioritization for consumption metrics. + // Same for the loop that fetches computed metrics. + // By using the same limiter, we centralize metrics collection for "start" and "finished" counters, + // which turns out is really handy to understand the system. if let Err(e) = tenant.calculate_synthetic_size(cause, ctx).await { error!("failed to calculate synthetic size for tenant {tenant_id}: {e:#}"); } @@ -277,7 +283,7 @@ async fn calculate_synthetic_size_worker( crate::tenant::tasks::warn_when_period_overrun( tick_at.elapsed(), synthetic_size_calculation_interval, - "consumption_metrics_synthetic_size_worker", + BackgroundLoopKind::ConsumptionMetricsSyntheticSizeWorker, ); } } diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index c154b4a4ca..eea3de0583 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1067,6 +1067,26 @@ pub(crate) static TENANT_TASK_EVENTS: Lazy = Lazy::new(|| { .expect("Failed to register tenant_task_events metric") }); +pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_START_COUNT: Lazy = + Lazy::new(|| { + register_int_counter_vec!( + "pageserver_background_loop_semaphore_wait_start_count", + "Counter for background loop concurrency-limiting semaphore acquire calls started", + &["task"], + ) + .unwrap() + }); + +pub(crate) static BACKGROUND_LOOP_SEMAPHORE_WAIT_FINISH_COUNT: Lazy = + Lazy::new(|| { + register_int_counter_vec!( + "pageserver_background_loop_semaphore_wait_finish_count", + "Counter for background loop concurrency-limiting semaphore acquire calls finished", + &["task"], + ) + .unwrap() + }); + pub(crate) static BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT: Lazy = Lazy::new(|| { register_int_counter_vec!( "pageserver_background_loop_period_overrun_count", diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index df3ffd08d3..00c8ced68a 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -14,6 +14,73 @@ use tokio_util::sync::CancellationToken; use tracing::*; use utils::completion; +static CONCURRENT_BACKGROUND_TASKS: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(|| { + let total_threads = *task_mgr::BACKGROUND_RUNTIME_WORKER_THREADS; + let permits = usize::max( + 1, + // while a lot of the work is done on spawn_blocking, we still do + // repartitioning in the async context. this should give leave us some workers + // unblocked to be blocked on other work, hopefully easing any outside visible + // effects of restarts. + // + // 6/8 is a guess; previously we ran with unlimited 8 and more from + // spawn_blocking. + (total_threads * 3).checked_div(4).unwrap_or(0), + ); + assert_ne!(permits, 0, "we will not be adding in permits later"); + assert!( + permits < total_threads, + "need threads avail for shorter work" + ); + tokio::sync::Semaphore::new(permits) + }); + +#[derive(Debug, PartialEq, Eq, Clone, Copy, strum_macros::IntoStaticStr)] +#[strum(serialize_all = "snake_case")] +pub(crate) enum BackgroundLoopKind { + Compaction, + Gc, + Eviction, + ConsumptionMetricsCollectMetrics, + ConsumptionMetricsSyntheticSizeWorker, +} + +impl BackgroundLoopKind { + fn as_static_str(&self) -> &'static str { + let s: &'static str = self.into(); + s + } +} + +pub(crate) enum RateLimitError { + Cancelled, +} + +pub(crate) async fn concurrent_background_tasks_rate_limit( + loop_kind: BackgroundLoopKind, + _ctx: &RequestContext, + cancel: &CancellationToken, +) -> Result { + crate::metrics::BACKGROUND_LOOP_SEMAPHORE_WAIT_START_COUNT + .with_label_values(&[loop_kind.as_static_str()]) + .inc(); + scopeguard::defer!( + crate::metrics::BACKGROUND_LOOP_SEMAPHORE_WAIT_FINISH_COUNT.with_label_values(&[loop_kind.as_static_str()]).inc(); + ); + tokio::select! { + permit = CONCURRENT_BACKGROUND_TASKS.acquire() => { + match permit { + Ok(permit) => Ok(permit), + Err(_closed) => unreachable!("we never close the semaphore"), + } + }, + _ = cancel.cancelled() => { + Err(RateLimitError::Cancelled) + } + } +} + /// Start per tenant background loops: compaction and gc. pub fn start_background_loops( tenant: &Arc, @@ -116,7 +183,7 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { } }; - warn_when_period_overrun(started_at.elapsed(), period, "compaction"); + warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Compaction); // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) @@ -184,7 +251,7 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { } }; - warn_when_period_overrun(started_at.elapsed(), period, "gc"); + warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Gc); // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) @@ -258,7 +325,11 @@ pub(crate) async fn random_init_delay( } /// Attention: the `task` and `period` beocme labels of a pageserver-wide prometheus metric. -pub(crate) fn warn_when_period_overrun(elapsed: Duration, period: Duration, task: &str) { +pub(crate) fn warn_when_period_overrun( + elapsed: Duration, + period: Duration, + task: BackgroundLoopKind, +) { // Duration::ZERO will happen because it's the "disable [bgtask]" value. if elapsed >= period && period != Duration::ZERO { // humantime does no significant digits clamping whereas Duration's debug is a bit more @@ -267,11 +338,11 @@ pub(crate) fn warn_when_period_overrun(elapsed: Duration, period: Duration, task warn!( ?elapsed, period = %humantime::format_duration(period), - task, + ?task, "task iteration took longer than the configured period" ); crate::metrics::BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT - .with_label_values(&[task, &format!("{}", period.as_secs())]) + .with_label_values(&[task.as_static_str(), &format!("{}", period.as_secs())]) .inc(); } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 6776d16d60..250047823e 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -44,6 +44,7 @@ use crate::tenant::storage_layer::delta_layer::DeltaEntry; use crate::tenant::storage_layer::{ DeltaLayerWriter, ImageLayerWriter, InMemoryLayer, LayerAccessStats, LayerFileName, RemoteLayer, }; +use crate::tenant::tasks::{BackgroundLoopKind, RateLimitError}; use crate::tenant::timeline::logical_size::CurrentLogicalSize; use crate::tenant::{ layer_map::{LayerMap, SearchResult}, @@ -684,37 +685,17 @@ impl Timeline { ) -> anyhow::Result<()> { const ROUNDS: usize = 2; - static CONCURRENT_COMPACTIONS: once_cell::sync::Lazy = - once_cell::sync::Lazy::new(|| { - let total_threads = *task_mgr::BACKGROUND_RUNTIME_WORKER_THREADS; - let permits = usize::max( - 1, - // while a lot of the work is done on spawn_blocking, we still do - // repartitioning in the async context. this should give leave us some workers - // unblocked to be blocked on other work, hopefully easing any outside visible - // effects of restarts. - // - // 6/8 is a guess; previously we ran with unlimited 8 and more from - // spawn_blocking. - (total_threads * 3).checked_div(4).unwrap_or(0), - ); - assert_ne!(permits, 0, "we will not be adding in permits later"); - assert!( - permits < total_threads, - "need threads avail for shorter work" - ); - tokio::sync::Semaphore::new(permits) - }); - // this wait probably never needs any "long time spent" logging, because we already nag if // compaction task goes over it's period (20s) which is quite often in production. - let _permit = tokio::select! { - permit = CONCURRENT_COMPACTIONS.acquire() => { - permit - }, - _ = cancel.cancelled() => { - return Ok(()); - } + let _permit = match super::tasks::concurrent_background_tasks_rate_limit( + BackgroundLoopKind::Compaction, + ctx, + cancel, + ) + .await + { + Ok(permit) => permit, + Err(RateLimitError::Cancelled) => return Ok(()), }; let last_record_lsn = self.get_last_record_lsn(); diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 9bf31d85d4..38da993deb 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -30,6 +30,7 @@ use crate::{ tenant::{ config::{EvictionPolicy, EvictionPolicyLayerAccessThreshold}, storage_layer::PersistentLayer, + tasks::{BackgroundLoopKind, RateLimitError}, timeline::EvictionError, LogicalSizeCalculationCause, Tenant, }, @@ -129,7 +130,11 @@ impl Timeline { ControlFlow::Continue(()) => (), } let elapsed = start.elapsed(); - crate::tenant::tasks::warn_when_period_overrun(elapsed, p.period, "eviction"); + crate::tenant::tasks::warn_when_period_overrun( + elapsed, + p.period, + BackgroundLoopKind::Eviction, + ); crate::metrics::EVICTION_ITERATION_DURATION .get_metric_with_label_values(&[ &format!("{}", p.period.as_secs()), @@ -150,6 +155,17 @@ impl Timeline { ) -> ControlFlow<()> { let now = SystemTime::now(); + let _permit = match crate::tenant::tasks::concurrent_background_tasks_rate_limit( + BackgroundLoopKind::Eviction, + ctx, + cancel, + ) + .await + { + Ok(permit) => permit, + Err(RateLimitError::Cancelled) => return ControlFlow::Break(()), + }; + // If we evict layers but keep cached values derived from those layers, then // we face a storm of on-demand downloads after pageserver restart. // The reason is that the restart empties the caches, and so, the values From 00c71bb93a0ffc34b564b39222fa0a3164517c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 17 Oct 2023 12:59:57 +0200 Subject: [PATCH 05/49] Also try to login to Azure via SDK provided methods (#5573) ## Problem We ideally use the Azure SDK's way of obtaining authorization, as pointed out in https://github.com/neondatabase/neon/pull/5546#discussion_r1360619178 . ## Summary of changes This PR adds support for Azure SDK based authentication, using [DefaultAzureCredential](https://docs.rs/azure_identity/0.16.1/azure_identity/struct.DefaultAzureCredential.html), which tries the following credentials: * [EnvironmentCredential](https://docs.rs/azure_identity/0.16.1/azure_identity/struct.EnvironmentCredential.html), reading from various env vars * [ImdsManagedIdentityCredential](https://docs.rs/azure_identity/0.16.1/azure_identity/struct.ImdsManagedIdentityCredential.html), using managed identity * [AzureCliCredential](https://docs.rs/azure_identity/0.16.1/azure_identity/struct.AzureCliCredential.html), using Azure CLI closes #5566. --- Cargo.lock | 63 +++++++++++++++++++++++++++ Cargo.toml | 5 ++- libs/remote_storage/Cargo.toml | 1 + libs/remote_storage/src/azure_blob.rs | 17 +++++--- workspace_hack/Cargo.toml | 4 +- 5 files changed, 81 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aacf4e53d7..5665a9ef88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -853,6 +853,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "azure_identity" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b67b337346da8739e91ea1e9400a6ebc9bc54e0b2af1d23c9bcd565950588f9" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "futures", + "log", + "oauth2", + "pin-project", + "serde", + "serde_json", + "time 0.3.21", + "tz-rs", + "url", + "uuid", +] + [[package]] name = "azure_storage" version = "0.16.0" @@ -1131,9 +1152,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", + "js-sys", "num-integer", "num-traits", "serde", + "wasm-bindgen", "winapi", ] @@ -2949,6 +2972,34 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + +[[package]] +name = "oauth2" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" +dependencies = [ + "base64 0.13.1", + "chrono", + "getrandom 0.2.9", + "http", + "rand 0.8.5", + "serde", + "serde_json", + "serde_path_to_error", + "sha2 0.10.6", + "thiserror", + "url", +] + [[package]] name = "object" version = "0.30.3" @@ -4021,6 +4072,7 @@ dependencies = [ "aws-smithy-http", "aws-types", "azure_core", + "azure_identity", "azure_storage", "azure_storage_blobs", "bytes", @@ -5219,6 +5271,8 @@ checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", "js-sys", + "libc", + "num_threads", "serde", "time-core", "time-macros 0.2.9", @@ -5770,6 +5824,15 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "tz-rs" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" +dependencies = [ + "const_fn", +] + [[package]] name = "uname" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index 4827652229..621b7af564 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,9 +36,10 @@ license = "Apache-2.0" [workspace.dependencies] anyhow = { version = "1.0", features = ["backtrace"] } async-compression = { version = "0.4.0", features = ["tokio", "gzip"] } -azure_core = "0.16.0" +azure_core = "0.16" +azure_identity = "0.16" azure_storage = "0.16" -azure_storage_blobs = "0.16.0" +azure_storage_blobs = "0.16" flate2 = "1.0.26" async-stream = "0.3" async-trait = "0.1" diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index 5d3bda70af..65b5392389 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -28,6 +28,7 @@ utils.workspace = true pin-project-lite.workspace = true workspace_hack.workspace = true azure_core.workspace = true +azure_identity.workspace = true azure_storage.workspace = true azure_storage_blobs.workspace = true futures-util.workspace = true diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index 4f7c8ce6f4..b81cb47606 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -1,12 +1,15 @@ //! Azure Blob Storage wrapper +use std::env; use std::num::NonZeroU32; +use std::sync::Arc; use std::{borrow::Cow, collections::HashMap, io::Cursor}; use super::REMOTE_STORAGE_PREFIX_SEPARATOR; use anyhow::Result; use azure_core::request_options::{MaxResults, Metadata, Range}; use azure_core::Header; +use azure_identity::DefaultAzureCredential; use azure_storage::StorageCredentials; use azure_storage_blobs::prelude::ClientBuilder; use azure_storage_blobs::{ @@ -38,12 +41,16 @@ impl AzureBlobStorage { azure_config.container_name ); - let account = - std::env::var("AZURE_STORAGE_ACCOUNT").expect("missing AZURE_STORAGE_ACCOUNT"); - let access_key = - std::env::var("AZURE_STORAGE_ACCESS_KEY").expect("missing AZURE_STORAGE_ACCESS_KEY"); + let account = env::var("AZURE_STORAGE_ACCOUNT").expect("missing AZURE_STORAGE_ACCOUNT"); - let credentials = StorageCredentials::access_key(account.clone(), access_key); + // If the `AZURE_STORAGE_ACCESS_KEY` env var has an access key, use that, + // otherwise try the token based credentials. + let credentials = if let Ok(access_key) = env::var("AZURE_STORAGE_ACCESS_KEY") { + StorageCredentials::access_key(account.clone(), access_key) + } else { + let token_credential = DefaultAzureCredential::default(); + StorageCredentials::token_credential(Arc::new(token_credential)) + }; let builder = ClientBuilder::new(account, credentials); diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 4b250822e5..11e583084c 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -21,7 +21,7 @@ aws-smithy-http = { version = "0.56", default-features = false, features = ["eve axum = { version = "0.6", features = ["ws"] } base64 = { version = "0.21", features = ["alloc"] } bytes = { version = "1", features = ["serde"] } -chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } +chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } clap = { version = "4", features = ["derive", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8" } @@ -57,7 +57,7 @@ serde_json = { version = "1", features = ["raw_value"] } smallvec = { version = "1", default-features = false, features = ["write"] } socket2 = { version = "0.4", default-features = false, features = ["all"] } standback = { version = "0.2", default-features = false, features = ["std"] } -time = { version = "0.3", features = ["macros", "serde-well-known"] } +time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-rustls = { version = "0.24" } tokio-util = { version = "0.7", features = ["codec", "io"] } From 093f8c5f45e8a5d86730ad4363b7eb45045314ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 17 Oct 2023 14:13:12 +0200 Subject: [PATCH 06/49] Update rust to 1.73.0 (#5574) [Release notes](https://blog.rust-lang.org/2023/10/05/Rust-1.73.0.html) --- libs/metrics/src/wrappers.rs | 16 ++++++++-------- pageserver/src/tenant/ephemeral_file.rs | 3 +-- .../src/tenant/storage_layer/remote_layer.rs | 2 +- proxy/src/sasl/messages.rs | 2 +- rust-toolchain.toml | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/libs/metrics/src/wrappers.rs b/libs/metrics/src/wrappers.rs index 1bf1ea0753..c3959cbf16 100644 --- a/libs/metrics/src/wrappers.rs +++ b/libs/metrics/src/wrappers.rs @@ -1,6 +1,6 @@ use std::io::{Read, Result, Write}; -/// A wrapper for an object implementing [Read](std::io::Read) +/// A wrapper for an object implementing [Read] /// which allows a closure to observe the amount of bytes read. /// This is useful in conjunction with metrics (e.g. [IntCounter](crate::IntCounter)). /// @@ -51,17 +51,17 @@ impl<'a, T> CountedReader<'a, T> { } } - /// Get an immutable reference to the underlying [Read](std::io::Read) implementor + /// Get an immutable reference to the underlying [Read] implementor pub fn inner(&self) -> &T { &self.reader } - /// Get a mutable reference to the underlying [Read](std::io::Read) implementor + /// Get a mutable reference to the underlying [Read] implementor pub fn inner_mut(&mut self) -> &mut T { &mut self.reader } - /// Consume the wrapper and return the underlying [Read](std::io::Read) implementor + /// Consume the wrapper and return the underlying [Read] implementor pub fn into_inner(self) -> T { self.reader } @@ -75,7 +75,7 @@ impl Read for CountedReader<'_, T> { } } -/// A wrapper for an object implementing [Write](std::io::Write) +/// A wrapper for an object implementing [Write] /// which allows a closure to observe the amount of bytes written. /// This is useful in conjunction with metrics (e.g. [IntCounter](crate::IntCounter)). /// @@ -122,17 +122,17 @@ impl<'a, T> CountedWriter<'a, T> { } } - /// Get an immutable reference to the underlying [Write](std::io::Write) implementor + /// Get an immutable reference to the underlying [Write] implementor pub fn inner(&self) -> &T { &self.writer } - /// Get a mutable reference to the underlying [Write](std::io::Write) implementor + /// Get a mutable reference to the underlying [Write] implementor pub fn inner_mut(&mut self) -> &mut T { &mut self.writer } - /// Consume the wrapper and return the underlying [Write](std::io::Write) implementor + /// Consume the wrapper and return the underlying [Write] implementor pub fn into_inner(self) -> T { self.writer } diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index 5b99a1dd03..9a06d9df61 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -354,8 +354,7 @@ mod tests { } // Test a large blob that spans multiple pages - let mut large_data = Vec::new(); - large_data.resize(20000, 0); + let mut large_data = vec![0; 20000]; thread_rng().fill_bytes(&mut large_data); let pos_large = file.write_blob(&large_data, &ctx).await?; let result = file.block_cursor().read_blob(pos_large, &ctx).await?; diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index cafe5f6bb6..94bebb5f6e 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -25,7 +25,7 @@ use super::{ }; /// RemoteLayer is a not yet downloaded [`ImageLayer`] or -/// [`DeltaLayer`](super::DeltaLayer). +/// [`DeltaLayer`]. /// /// RemoteLayer might be downloaded on-demand during operations which are /// allowed download remote layers and during which, it gets replaced with a diff --git a/proxy/src/sasl/messages.rs b/proxy/src/sasl/messages.rs index fb3833c8b6..b9208f6f1f 100644 --- a/proxy/src/sasl/messages.rs +++ b/proxy/src/sasl/messages.rs @@ -31,7 +31,7 @@ impl<'a> FirstMessage<'a> { /// A single SASL message. /// This struct is deliberately decoupled from lower-level -/// [`BeAuthenticationSaslMessage`](pq_proto::BeAuthenticationSaslMessage). +/// [`BeAuthenticationSaslMessage`]. #[derive(Debug)] pub(super) enum ServerMessage { /// We expect to see more steps. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 9cc47ec039..ca12f0dee5 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.72.1" +channel = "1.73.0" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html From ea648cfbc66e4406006cba9de2f72e30bd8e6f5b Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 17 Oct 2023 13:26:11 +0100 Subject: [PATCH 07/49] tests: fix test_eviction_across_generations trying to evict temp files (#5579) This test is listing files in a timeline and then evicting them: if the test ran slowly this could encounter temp files for unfinished downloads: fix by filtering these out in evict_all_layers. --- test_runner/regress/test_pageserver_generations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 6dc8582755..994d09665c 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -492,7 +492,7 @@ def evict_all_layers(env: NeonEnv, tenant_id: TenantId, timeline_id: TimelineId) ) client = env.pageserver.http_client() for layer in initial_local_layers: - if "ephemeral" in layer.name: + if "ephemeral" in layer.name or "temp" in layer.name: continue log.info(f"Evicting layer {tenant_id}/{timeline_id} {layer.name}") client.evict_layer(tenant_id=tenant_id, timeline_id=timeline_id, layer_name=layer.name) From f775928dfc7049fe6b5eba9145554fc67f2c8ccf Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Tue, 17 Oct 2023 14:55:52 +0100 Subject: [PATCH 08/49] proxy: refactor how and when connections are returned to the pool (#5095) ## Problem Transactions break connections in the pool fixes #4698 ## Summary of changes * Pool `Client`s are smart object that return themselves to the pool * Pool `Client`s can be 'discard'ed * Pool `Client`s are discarded when certain errors are encountered. * Pool `Client`s are discarded when ReadyForQuery returns a non-idle state. --- Cargo.lock | 13 +- Cargo.toml | 12 +- proxy/src/http/conn_pool.rs | 129 +++++++++++++--- proxy/src/http/sql_over_http.rs | 242 +++++++++++++++++------------- test_runner/regress/test_proxy.py | 23 ++- workspace_hack/Cargo.toml | 1 - 6 files changed, 284 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5665a9ef88..8c641bd36c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3561,7 +3561,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" dependencies = [ "bytes", "fallible-iterator", @@ -3574,7 +3574,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" dependencies = [ "native-tls", "tokio", @@ -3585,7 +3585,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" dependencies = [ "base64 0.20.0", "byteorder", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" dependencies = [ "bytes", "fallible-iterator", @@ -5407,7 +5407,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" dependencies = [ "async-trait", "byteorder", @@ -5422,7 +5422,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2 0.4.9", + "socket2 0.5.3", "tokio", "tokio-util", ] @@ -6497,7 +6497,6 @@ dependencies = [ "serde", "serde_json", "smallvec", - "socket2 0.4.9", "standback", "syn 1.0.109", "syn 2.0.28", diff --git a/Cargo.toml b/Cargo.toml index 621b7af564..727fcbd5a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,11 +160,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -200,7 +200,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } ################# Binary contents sections diff --git a/proxy/src/http/conn_pool.rs b/proxy/src/http/conn_pool.rs index b268c4073e..ed84b0f2bf 100644 --- a/proxy/src/http/conn_pool.rs +++ b/proxy/src/http/conn_pool.rs @@ -8,14 +8,17 @@ use pbkdf2::{ Params, Pbkdf2, }; use pq_proto::StartupMessageParams; -use std::sync::atomic::{self, AtomicUsize}; use std::{collections::HashMap, sync::Arc}; use std::{ fmt, task::{ready, Poll}, }; +use std::{ + ops::Deref, + sync::atomic::{self, AtomicUsize}, +}; use tokio::time; -use tokio_postgres::AsyncMessage; +use tokio_postgres::{AsyncMessage, ReadyForQueryStatus}; use crate::{ auth, console, @@ -26,13 +29,13 @@ use crate::{compute, config}; use crate::proxy::ConnectMechanism; -use tracing::{error, warn}; +use tracing::{error, warn, Span}; use tracing::{info, info_span, Instrument}; pub const APP_NAME: &str = "sql_over_http"; const MAX_CONNS_PER_ENDPOINT: usize = 20; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ConnInfo { pub username: String, pub dbname: String, @@ -55,7 +58,7 @@ impl fmt::Display for ConnInfo { } struct ConnPoolEntry { - conn: Client, + conn: ClientInner, _last_access: std::time::Instant, } @@ -133,14 +136,20 @@ impl GlobalConnPool { } pub async fn get( - &self, + self: &Arc, conn_info: &ConnInfo, force_new: bool, session_id: uuid::Uuid, ) -> anyhow::Result { - let mut client: Option = None; + let mut client: Option = None; let mut latency_timer = LatencyTimer::new("http"); + let pool = if force_new { + None + } else { + Some((conn_info.clone(), self.clone())) + }; + let mut hash_valid = false; if !force_new { let pool = self.get_or_create_endpoint_pool(&conn_info.hostname); @@ -188,7 +197,11 @@ impl GlobalConnPool { latency_timer.pool_hit(); info!("pool: reusing connection '{conn_info}'"); client.session.send(session_id)?; - return Ok(client); + return Ok(Client { + inner: Some(client), + span: Span::current(), + pool, + }); } } else { info!("pool: opening a new connection '{conn_info}'"); @@ -228,10 +241,14 @@ impl GlobalConnPool { _ => {} } - new_client + new_client.map(|inner| Client { + inner: Some(inner), + span: Span::current(), + pool, + }) } - pub fn put(&self, conn_info: &ConnInfo, client: Client) -> anyhow::Result<()> { + fn put(&self, conn_info: &ConnInfo, client: ClientInner) -> anyhow::Result<()> { // We want to hold this open while we return. This ensures that the pool can't close // while we are in the middle of returning the connection. let closed = self.closed.read(); @@ -326,7 +343,7 @@ struct TokioMechanism<'a> { #[async_trait] impl ConnectMechanism for TokioMechanism<'_> { - type Connection = Client; + type Connection = ClientInner; type ConnectError = tokio_postgres::Error; type Error = anyhow::Error; @@ -350,7 +367,7 @@ async fn connect_to_compute( conn_info: &ConnInfo, session_id: uuid::Uuid, latency_timer: LatencyTimer, -) -> anyhow::Result { +) -> anyhow::Result { let tls = config.tls_config.as_ref(); let common_names = tls.and_then(|tls| tls.common_names.clone()); @@ -399,7 +416,7 @@ async fn connect_to_compute_once( conn_info: &ConnInfo, timeout: time::Duration, mut session: uuid::Uuid, -) -> Result { +) -> Result { let mut config = (*node_info.config).clone(); let (client, mut connection) = config @@ -462,21 +479,99 @@ async fn connect_to_compute_once( .instrument(span) ); - Ok(Client { + Ok(ClientInner { inner: client, session: tx, ids, }) } -pub struct Client { - pub inner: tokio_postgres::Client, +struct ClientInner { + inner: tokio_postgres::Client, session: tokio::sync::watch::Sender, ids: Ids, } impl Client { pub fn metrics(&self) -> Arc { - USAGE_METRICS.register(self.ids.clone()) + USAGE_METRICS.register(self.inner.as_ref().unwrap().ids.clone()) + } +} + +pub struct Client { + span: Span, + inner: Option, + pool: Option<(ConnInfo, Arc)>, +} + +pub struct Discard<'a> { + pool: &'a mut Option<(ConnInfo, Arc)>, +} + +impl Client { + pub fn inner(&mut self) -> (&mut tokio_postgres::Client, Discard<'_>) { + let Self { + inner, + pool, + span: _, + } = self; + ( + &mut inner + .as_mut() + .expect("client inner should not be removed") + .inner, + Discard { pool }, + ) + } + + pub fn check_idle(&mut self, status: ReadyForQueryStatus) { + self.inner().1.check_idle(status) + } + pub fn discard(&mut self) { + self.inner().1.discard() + } +} + +impl Discard<'_> { + pub fn check_idle(&mut self, status: ReadyForQueryStatus) { + if status != ReadyForQueryStatus::Idle { + if let Some((conn_info, _)) = self.pool.take() { + info!("pool: throwing away connection '{conn_info}' because connection is not idle") + } + } + } + pub fn discard(&mut self) { + if let Some((conn_info, _)) = self.pool.take() { + info!("pool: throwing away connection '{conn_info}' because connection is potentially in a broken state") + } + } +} + +impl Deref for Client { + type Target = tokio_postgres::Client; + + fn deref(&self) -> &Self::Target { + &self + .inner + .as_ref() + .expect("client inner should not be removed") + .inner + } +} + +impl Drop for Client { + fn drop(&mut self) { + let client = self + .inner + .take() + .expect("client inner should not be removed"); + if let Some((conn_info, conn_pool)) = self.pool.take() { + let current_span = self.span.clone(); + // return connection to the pool + tokio::task::spawn_blocking(move || { + let _span = current_span.enter(); + let _ = conn_pool.put(&conn_info, client); + }); + } } } diff --git a/proxy/src/http/sql_over_http.rs b/proxy/src/http/sql_over_http.rs index 02ce3aa9dc..93df86cfc4 100644 --- a/proxy/src/http/sql_over_http.rs +++ b/proxy/src/http/sql_over_http.rs @@ -17,7 +17,9 @@ use tokio_postgres::types::Kind; use tokio_postgres::types::Type; use tokio_postgres::GenericClient; use tokio_postgres::IsolationLevel; +use tokio_postgres::ReadyForQueryStatus; use tokio_postgres::Row; +use tokio_postgres::Transaction; use tracing::error; use tracing::instrument; use url::Url; @@ -64,20 +66,18 @@ static HEADER_VALUE_TRUE: HeaderValue = HeaderValue::from_static("true"); // Convert json non-string types to strings, so that they can be passed to Postgres // as parameters. // -fn json_to_pg_text(json: Vec) -> Result>, serde_json::Error> { +fn json_to_pg_text(json: Vec) -> Vec> { json.iter() .map(|value| { match value { // special care for nulls - Value::Null => Ok(None), + Value::Null => None, // convert to text with escaping - Value::Bool(_) => serde_json::to_string(value).map(Some), - Value::Number(_) => serde_json::to_string(value).map(Some), - Value::Object(_) => serde_json::to_string(value).map(Some), + v @ (Value::Bool(_) | Value::Number(_) | Value::Object(_)) => Some(v.to_string()), // avoid escaping here, as we pass this as a parameter - Value::String(s) => Ok(Some(s.to_string())), + Value::String(s) => Some(s.to_string()), // special care for arrays Value::Array(_) => json_array_to_pg_array(value), @@ -94,29 +94,26 @@ fn json_to_pg_text(json: Vec) -> Result>, serde_json:: // // Example of the same escaping in node-postgres: packages/pg/lib/utils.js // -fn json_array_to_pg_array(value: &Value) -> Result, serde_json::Error> { +fn json_array_to_pg_array(value: &Value) -> Option { match value { // special care for nulls - Value::Null => Ok(None), + Value::Null => None, // convert to text with escaping - Value::Bool(_) => serde_json::to_string(value).map(Some), - Value::Number(_) => serde_json::to_string(value).map(Some), - // here string needs to be escaped, as it is part of the array - Value::Object(_) => json_array_to_pg_array(&Value::String(serde_json::to_string(value)?)), - Value::String(_) => serde_json::to_string(value).map(Some), + v @ (Value::Bool(_) | Value::Number(_) | Value::String(_)) => Some(v.to_string()), + v @ Value::Object(_) => json_array_to_pg_array(&Value::String(v.to_string())), // recurse into array Value::Array(arr) => { let vals = arr .iter() .map(json_array_to_pg_array) - .map(|r| r.map(|v| v.unwrap_or_else(|| "NULL".to_string()))) - .collect::, _>>()? + .map(|v| v.unwrap_or_else(|| "NULL".to_string())) + .collect::>() .join(","); - Ok(Some(format!("{{{}}}", vals))) + Some(format!("{{{}}}", vals)) } } } @@ -315,83 +312,119 @@ async fn handle_inner( // Now execute the query and return the result // let mut size = 0; - let result = match payload { - Payload::Single(query) => { - query_to_json(&client.inner, query, &mut size, raw_output, array_mode).await - } - Payload::Batch(batch_query) => { - let mut results = Vec::new(); - let mut builder = client.inner.build_transaction(); - if let Some(isolation_level) = txn_isolation_level { - builder = builder.isolation_level(isolation_level); + let result = + match payload { + Payload::Single(stmt) => { + let (status, results) = + query_to_json(&*client, stmt, &mut 0, raw_output, array_mode) + .await + .map_err(|e| { + client.discard(); + e + })?; + client.check_idle(status); + results } - if txn_read_only { - builder = builder.read_only(true); - } - if txn_deferrable { - builder = builder.deferrable(true); - } - let transaction = builder.start().await?; - for query in batch_query.queries { - let result = - query_to_json(&transaction, query, &mut size, raw_output, array_mode).await; - match result { - Ok(r) => results.push(r), - Err(e) => { - transaction.rollback().await?; - return Err(e); - } + Payload::Batch(statements) => { + let (inner, mut discard) = client.inner(); + let mut builder = inner.build_transaction(); + if let Some(isolation_level) = txn_isolation_level { + builder = builder.isolation_level(isolation_level); } + if txn_read_only { + builder = builder.read_only(true); + } + if txn_deferrable { + builder = builder.deferrable(true); + } + + let transaction = builder.start().await.map_err(|e| { + // if we cannot start a transaction, we should return immediately + // and not return to the pool. connection is clearly broken + discard.discard(); + e + })?; + + let results = + match query_batch(&transaction, statements, &mut size, raw_output, array_mode) + .await + { + Ok(results) => { + let status = transaction.commit().await.map_err(|e| { + // if we cannot commit - for now don't return connection to pool + // TODO: get a query status from the error + discard.discard(); + e + })?; + discard.check_idle(status); + results + } + Err(err) => { + let status = transaction.rollback().await.map_err(|e| { + // if we cannot rollback - for now don't return connection to pool + // TODO: get a query status from the error + discard.discard(); + e + })?; + discard.check_idle(status); + return Err(err); + } + }; + + if txn_read_only { + response = response.header( + TXN_READ_ONLY.clone(), + HeaderValue::try_from(txn_read_only.to_string())?, + ); + } + if txn_deferrable { + response = response.header( + TXN_DEFERRABLE.clone(), + HeaderValue::try_from(txn_deferrable.to_string())?, + ); + } + if let Some(txn_isolation_level) = txn_isolation_level_raw { + response = response.header(TXN_ISOLATION_LEVEL.clone(), txn_isolation_level); + } + json!({ "results": results }) } - transaction.commit().await?; - if txn_read_only { - response = response.header( - TXN_READ_ONLY.clone(), - HeaderValue::try_from(txn_read_only.to_string())?, - ); - } - if txn_deferrable { - response = response.header( - TXN_DEFERRABLE.clone(), - HeaderValue::try_from(txn_deferrable.to_string())?, - ); - } - if let Some(txn_isolation_level) = txn_isolation_level_raw { - response = response.header(TXN_ISOLATION_LEVEL.clone(), txn_isolation_level); - } - Ok(json!({ "results": results })) - } - }; + }; let metrics = client.metrics(); - if allow_pool { - let current_span = tracing::Span::current(); - // return connection to the pool - tokio::task::spawn_blocking(move || { - let _span = current_span.enter(); - let _ = conn_pool.put(&conn_info, client); - }); - } + // how could this possibly fail + let body = serde_json::to_string(&result).expect("json serialization should not fail"); + let len = body.len(); + let response = response + .body(Body::from(body)) + // only fails if invalid status code or invalid header/values are given. + // these are not user configurable so it cannot fail dynamically + .expect("building response payload should not fail"); - match result { - Ok(value) => { - // how could this possibly fail - let body = serde_json::to_string(&value).expect("json serialization should not fail"); - let len = body.len(); - let response = response - .body(Body::from(body)) - // only fails if invalid status code or invalid header/values are given. - // these are not user configurable so it cannot fail dynamically - .expect("building response payload should not fail"); + // count the egress bytes - we miss the TLS and header overhead but oh well... + // moving this later in the stack is going to be a lot of effort and ehhhh + metrics.record_egress(len as u64); - // count the egress bytes - we miss the TLS and header overhead but oh well... - // moving this later in the stack is going to be a lot of effort and ehhhh - metrics.record_egress(len as u64); - Ok(response) - } - Err(e) => Err(e), + Ok(response) +} + +async fn query_batch( + transaction: &Transaction<'_>, + queries: BatchQueryData, + total_size: &mut usize, + raw_output: bool, + array_mode: bool, +) -> anyhow::Result> { + let mut results = Vec::with_capacity(queries.queries.len()); + let mut current_size = 0; + for stmt in queries.queries { + // TODO: maybe we should check that the transaction bit is set here + let (_, values) = + query_to_json(transaction, stmt, &mut current_size, raw_output, array_mode).await?; + results.push(values); } + *total_size += current_size; + Ok(results) } async fn query_to_json( @@ -400,11 +433,9 @@ async fn query_to_json( current_size: &mut usize, raw_output: bool, array_mode: bool, -) -> anyhow::Result { - let query_params = json_to_pg_text(data.params)?; - let row_stream = client - .query_raw_txt::(data.query, query_params) - .await?; +) -> anyhow::Result<(ReadyForQueryStatus, Value)> { + let query_params = json_to_pg_text(data.params); + let row_stream = client.query_raw_txt(&data.query, query_params).await?; // Manually drain the stream into a vector to leave row_stream hanging // around to get a command tag. Also check that the response is not too @@ -424,6 +455,8 @@ async fn query_to_json( } } + let ready = row_stream.ready_status(); + // grab the command tag and number of rows affected let command_tag = row_stream.command_tag().unwrap_or_default(); let mut command_tag_split = command_tag.split(' '); @@ -464,13 +497,16 @@ async fn query_to_json( .collect::, _>>()?; // resulting JSON format is based on the format of node-postgres result - Ok(json!({ - "command": command_tag_name, - "rowCount": command_tag_count, - "rows": rows, - "fields": fields, - "rowAsArray": array_mode, - })) + Ok(( + ready, + json!({ + "command": command_tag_name, + "rowCount": command_tag_count, + "rows": rows, + "fields": fields, + "rowAsArray": array_mode, + }), + )) } // @@ -655,22 +691,22 @@ mod tests { #[test] fn test_atomic_types_to_pg_params() { let json = vec![Value::Bool(true), Value::Bool(false)]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!( pg_params, vec![Some("true".to_owned()), Some("false".to_owned())] ); let json = vec![Value::Number(serde_json::Number::from(42))]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!(pg_params, vec![Some("42".to_owned())]); let json = vec![Value::String("foo\"".to_string())]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!(pg_params, vec![Some("foo\"".to_owned())]); let json = vec![Value::Null]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!(pg_params, vec![None]); } @@ -679,7 +715,7 @@ mod tests { // atoms and escaping let json = "[true, false, null, \"NULL\", 42, \"foo\", \"bar\\\"-\\\\\"]"; let json: Value = serde_json::from_str(json).unwrap(); - let pg_params = json_to_pg_text(vec![json]).unwrap(); + let pg_params = json_to_pg_text(vec![json]); assert_eq!( pg_params, vec![Some( @@ -690,7 +726,7 @@ mod tests { // nested arrays let json = "[[true, false], [null, 42], [\"foo\", \"bar\\\"-\\\\\"]]"; let json: Value = serde_json::from_str(json).unwrap(); - let pg_params = json_to_pg_text(vec![json]).unwrap(); + let pg_params = json_to_pg_text(vec![json]); assert_eq!( pg_params, vec![Some( @@ -700,7 +736,7 @@ mod tests { // array of objects let json = r#"[{"foo": 1},{"bar": 2}]"#; let json: Value = serde_json::from_str(json).unwrap(); - let pg_params = json_to_pg_text(vec![json]).unwrap(); + let pg_params = json_to_pg_text(vec![json]); assert_eq!( pg_params, vec![Some(r#"{"{\"foo\":1}","{\"bar\":2}"}"#.to_owned())] diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index f57b15c9b9..a33b29549c 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -7,6 +7,8 @@ import pytest import requests from fixtures.neon_fixtures import PSQL, NeonProxy, VanillaPostgres +GET_CONNECTION_PID_QUERY = "SELECT pid FROM pg_stat_activity WHERE state = 'active'" + def test_proxy_select_1(static_proxy: NeonProxy): """ @@ -353,7 +355,7 @@ def test_sql_over_http_pool(static_proxy: NeonProxy): def get_pid(status: int, pw: str) -> Any: return static_proxy.http_query( - "SELECT pid FROM pg_stat_activity WHERE state = 'active'", + GET_CONNECTION_PID_QUERY, [], user="http_auth", password=pw, @@ -387,7 +389,6 @@ def test_sql_over_http_pool(static_proxy: NeonProxy): # Beginning a transaction should not impact the next query, # which might come from a completely different client. -@pytest.mark.xfail(reason="not implemented") def test_http_pool_begin(static_proxy: NeonProxy): static_proxy.safe_psql("create user http_auth with password 'http' superuser") @@ -403,3 +404,21 @@ def test_http_pool_begin(static_proxy: NeonProxy): query(200, "BEGIN;") query(400, "garbage-lol(&(&(&(&") # Intentional error to break the transaction query(200, "SELECT 1;") # Query that should succeed regardless of the transaction + + +def test_sql_over_http_pool_idle(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth2 with password 'http' superuser") + + def query(status: int, query: str) -> Any: + return static_proxy.http_query( + query, + [], + user="http_auth2", + password="http", + expected_code=status, + ) + + pid1 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] + query(200, "BEGIN") + pid2 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] + assert pid1 != pid2 diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 11e583084c..e2a65ad150 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -55,7 +55,6 @@ scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } smallvec = { version = "1", default-features = false, features = ["write"] } -socket2 = { version = "0.4", default-features = false, features = ["all"] } standback = { version = "0.2", default-features = false, features = ["std"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } From 4a5048386199484236ef40d118f68990b71fc88d Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 17 Oct 2023 16:41:47 +0200 Subject: [PATCH 09/49] docs: error handling: document preferred anyhow context & logging style (#5178) We already had strong support for this many months ago on Slack: https://neondb.slack.com/archives/C0277TKAJCA/p1673453329770429 --- docs/error-handling.md | 55 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/docs/error-handling.md b/docs/error-handling.md index c2231274a1..531c80906b 100644 --- a/docs/error-handling.md +++ b/docs/error-handling.md @@ -188,11 +188,60 @@ that. ## Error message style +### PostgreSQL extensions + PostgreSQL has a style guide for writing error messages: https://www.postgresql.org/docs/current/error-style-guide.html Follow that guide when writing error messages in the PostgreSQL -extension. We don't follow it strictly in the pageserver and -safekeeper, but the advice in the PostgreSQL style guide is generally -good, and you can't go wrong by following it. +extensions. + +### Neon Rust code + +#### Anyhow Context + +When adding anyhow `context()`, use form `present-tense-verb+action`. + +Example: +- Bad: `file.metadata().context("could not get file metadata")?;` +- Good: `file.metadata().context("get file metadata")?;` + +#### Logging Errors + +When logging any error `e`, use `could not {e:#}` or `failed to {e:#}`. + +If `e` is an `anyhow` error and you want to log the backtrace that it contains, +use `{e:?}` instead of `{e:#}`. + +#### Rationale + +The `{:#}` ("alternate Display") of an `anyhow` error chain is concatenation fo the contexts, using `: `. + +For example, the following Rust code will result in output +``` +ERROR failed to list users: load users from server: parse response: invalid json +``` + +This is more concise / less noisy than what happens if you do `.context("could not ...")?` at each level, i.e.: + +``` +ERROR could not list users: could not load users from server: could not parse response: invalid json +``` + + +```rust +fn main() { + match list_users().context("list users") else { + Ok(_) => ..., + Err(e) => tracing::error!("failed to {e:#}"), + } +} +fn list_users() { + http_get_users().context("load users from server")?; +} +fn http_get_users() { + let response = client....?; + response.parse().context("parse response")?; // fails with serde error "invalid json" +} +``` From 3a8959a4c4da213d170184cdbafb7f237865912e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 17 Oct 2023 16:56:16 +0200 Subject: [PATCH 10/49] page_cache: remove dead code (#5493) --- pageserver/src/page_cache.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index f6acf64f22..0702057766 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -318,15 +318,6 @@ impl std::ops::Deref for PageWriteGuard<'_> { } } -impl AsMut<[u8; PAGE_SZ]> for PageWriteGuard<'_> { - fn as_mut(&mut self) -> &mut [u8; PAGE_SZ] { - match &mut self.state { - PageWriteGuardState::Invalid { inner, _permit } => inner.buf, - PageWriteGuardState::Downgraded => unreachable!(), - } - } -} - impl<'a> PageWriteGuard<'a> { /// Mark that the buffer contents are now valid. #[must_use] From 543b8153c6231e809d8716b5343ac99f6b086f9e Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Tue, 17 Oct 2023 16:59:35 +0100 Subject: [PATCH 11/49] proxy: add flag to reject requests without proxy protocol client ip (#5417) ## Problem We need a flag to require proxy protocol (prerequisite for #5416) ## Summary of changes Add a cli flag to require client IP addresses. Error if IP address is missing when the flag is active. --- proxy/src/bin/proxy.rs | 5 +++++ proxy/src/config.rs | 1 + proxy/src/http/websocket.rs | 12 +++++++++--- proxy/src/proxy.rs | 2 ++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 6bccdf6e5a..b5970b22c9 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -83,6 +83,10 @@ struct ProxyCliArgs { /// timeout for http connections #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] sql_over_http_timeout: tokio::time::Duration, + + /// Require that all incoming requests have a Proxy Protocol V2 packet **and** have an IP address associated. + #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] + require_client_ip: bool, } #[tokio::main] @@ -233,6 +237,7 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { metric_collection, allow_self_signed_compute: args.allow_self_signed_compute, http_config, + require_client_ip: args.require_client_ip, })); Ok(config) diff --git a/proxy/src/config.rs b/proxy/src/config.rs index b37c1736bd..b5ff3b64bd 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -14,6 +14,7 @@ pub struct ProxyConfig { pub metric_collection: Option, pub allow_self_signed_compute: bool, pub http_config: HttpConfig, + pub require_client_ip: bool, } #[derive(Debug)] diff --git a/proxy/src/http/websocket.rs b/proxy/src/http/websocket.rs index 656635b299..689a84969c 100644 --- a/proxy/src/http/websocket.rs +++ b/proxy/src/http/websocket.rs @@ -8,6 +8,7 @@ use crate::{ NUM_CLIENT_CONNECTION_OPENED_COUNTER, }, }; +use anyhow::bail; use bytes::{Buf, Bytes}; use futures::{Sink, Stream, StreamExt}; use hyper::{ @@ -22,7 +23,6 @@ use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, WebSocketStream}; use pin_project_lite::pin_project; use std::{ - convert::Infallible, future::ready, pin::Pin, sync::Arc, @@ -280,12 +280,18 @@ pub async fn task_main( let make_svc = hyper::service::make_service_fn( |stream: &tokio_rustls::server::TlsStream>| { let (io, tls) = stream.get_ref(); - let peer_addr = io.client_addr().unwrap_or(io.inner.remote_addr()); + let client_addr = io.client_addr(); + let remote_addr = io.inner.remote_addr(); let sni_name = tls.server_name().map(|s| s.to_string()); let conn_pool = conn_pool.clone(); async move { - Ok::<_, Infallible>(MetricService::new(hyper::service::service_fn( + let peer_addr = match client_addr { + Some(addr) => addr, + None if config.require_client_ip => bail!("missing required client ip"), + None => remote_addr, + }; + Ok(MetricService::new(hyper::service::service_fn( move |req: Request| { let sni_name = sni_name.clone(); let conn_pool = conn_pool.clone(); diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 6a928547a9..907a061f96 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -200,6 +200,8 @@ pub async fn task_main( let mut socket = WithClientIp::new(socket); if let Some(ip) = socket.wait_for_addr().await? { tracing::Span::current().record("peer_addr", &tracing::field::display(ip)); + } else if config.require_client_ip { + bail!("missing required client IP"); } socket From 9fe5cc6a8238c15ee7387c834292c3cc434723ca Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Tue, 17 Oct 2023 15:30:40 -0700 Subject: [PATCH 12/49] vm-monitor: Switch from memory.high to polling memory.stat (#5524) tl;dr it's really hard to avoid throttling from memory.high, and it counts tmpfs & page cache usage, so it's also hard to make sense of. In the interest of fixing things quickly with something that should be *good enough*, this PR switches to instead periodically fetch memory statistics from the cgroup's memory.stat and use that data to determine if and when we should upscale. This PR fixes #5444, which has a lot more detail on the difficulties we've hit with memory.high. This PR also supersedes #5488. --- libs/vm_monitor/README.md | 4 +- libs/vm_monitor/src/cgroup.rs | 786 ++++++++---------------------- libs/vm_monitor/src/dispatcher.rs | 29 +- libs/vm_monitor/src/runner.rs | 285 ++++++----- 4 files changed, 366 insertions(+), 738 deletions(-) diff --git a/libs/vm_monitor/README.md b/libs/vm_monitor/README.md index 53cdecd9f3..fdd943077d 100644 --- a/libs/vm_monitor/README.md +++ b/libs/vm_monitor/README.md @@ -27,8 +27,8 @@ and old one if it exists. * the filecache: a struct that allows communication with the Postgres file cache. On startup, we connect to the filecache and hold on to the connection for the entire monitor lifetime. -* the cgroup watcher: the `CgroupWatcher` manages the `neon-postgres` cgroup by -listening for `memory.high` events and setting its `memory.{high,max}` values. +* the cgroup watcher: the `CgroupWatcher` polls the `neon-postgres` cgroup's memory +usage and sends rolling aggregates to the runner. * the runner: the runner marries the filecache and cgroup watcher together, communicating with the agent throught the `Dispatcher`, and then calling filecache and cgroup watcher functions as needed to upscale and downscale diff --git a/libs/vm_monitor/src/cgroup.rs b/libs/vm_monitor/src/cgroup.rs index 15e972505e..7160a42df2 100644 --- a/libs/vm_monitor/src/cgroup.rs +++ b/libs/vm_monitor/src/cgroup.rs @@ -1,161 +1,38 @@ -use std::{ - fmt::{Debug, Display}, - fs, - pin::pin, - sync::atomic::{AtomicU64, Ordering}, -}; +use std::fmt::{self, Debug, Formatter}; +use std::time::{Duration, Instant}; -use anyhow::{anyhow, bail, Context}; +use anyhow::{anyhow, Context}; use cgroups_rs::{ - freezer::FreezerController, - hierarchies::{self, is_cgroup2_unified_mode, UNIFIED_MOUNTPOINT}, + hierarchies::{self, is_cgroup2_unified_mode}, memory::MemController, - MaxValue, - Subsystem::{Freezer, Mem}, + Subsystem, }; -use inotify::{EventStream, Inotify, WatchMask}; -use tokio::sync::mpsc::{self, error::TryRecvError}; -use tokio::time::{Duration, Instant}; -use tokio_stream::{Stream, StreamExt}; +use tokio::sync::watch; use tracing::{info, warn}; -use crate::protocol::Resources; -use crate::MiB; - -/// Monotonically increasing counter of the number of memory.high events -/// the cgroup has experienced. -/// -/// We use this to determine if a modification to the `memory.events` file actually -/// changed the `high` field. If not, we don't care about the change. When we -/// read the file, we check the `high` field in the file against `MEMORY_EVENT_COUNT` -/// to see if it changed since last time. -pub static MEMORY_EVENT_COUNT: AtomicU64 = AtomicU64::new(0); - -/// Monotonically increasing counter that gives each cgroup event a unique id. -/// -/// This allows us to answer questions like "did this upscale arrive before this -/// memory.high?". This static is also used by the `Sequenced` type to "tag" values -/// with a sequence number. As such, prefer to used the `Sequenced` type rather -/// than this static directly. -static EVENT_SEQUENCE_NUMBER: AtomicU64 = AtomicU64::new(0); - -/// A memory event type reported in memory.events. -#[derive(Debug, Eq, PartialEq, Copy, Clone)] -pub enum MemoryEvent { - Low, - High, - Max, - Oom, - OomKill, - OomGroupKill, -} - -impl MemoryEvent { - fn as_str(&self) -> &str { - match self { - MemoryEvent::Low => "low", - MemoryEvent::High => "high", - MemoryEvent::Max => "max", - MemoryEvent::Oom => "oom", - MemoryEvent::OomKill => "oom_kill", - MemoryEvent::OomGroupKill => "oom_group_kill", - } - } -} - -impl Display for MemoryEvent { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - /// Configuration for a `CgroupWatcher` #[derive(Debug, Clone)] pub struct Config { - // The target difference between the total memory reserved for the cgroup - // and the value of the cgroup's memory.high. - // - // In other words, memory.high + oom_buffer_bytes will equal the total memory that the cgroup may - // use (equal to system memory, minus whatever's taken out for the file cache). - oom_buffer_bytes: u64, + /// Interval at which we should be fetching memory statistics + memory_poll_interval: Duration, - // The amount of memory, in bytes, below a proposed new value for - // memory.high that the cgroup's memory usage must be for us to downscale - // - // In other words, we can downscale only when: - // - // memory.current + memory_high_buffer_bytes < (proposed) memory.high - // - // TODO: there's some minor issues with this approach -- in particular, that we might have - // memory in use by the kernel's page cache that we're actually ok with getting rid of. - pub(crate) memory_high_buffer_bytes: u64, - - // The maximum duration, in milliseconds, that we're allowed to pause - // the cgroup for while waiting for the autoscaler-agent to upscale us - max_upscale_wait: Duration, - - // The required minimum time, in milliseconds, that we must wait before re-freezing - // the cgroup while waiting for the autoscaler-agent to upscale us. - do_not_freeze_more_often_than: Duration, - - // The amount of memory, in bytes, that we should periodically increase memory.high - // by while waiting for the autoscaler-agent to upscale us. - // - // This exists to avoid the excessive throttling that happens when a cgroup is above its - // memory.high for too long. See more here: - // https://github.com/neondatabase/autoscaling/issues/44#issuecomment-1522487217 - memory_high_increase_by_bytes: u64, - - // The period, in milliseconds, at which we should repeatedly increase the value - // of the cgroup's memory.high while we're waiting on upscaling and memory.high - // is still being hit. - // - // Technically speaking, this actually serves as a rate limit to moderate responding to - // memory.high events, but these are roughly equivalent if the process is still allocating - // memory. - memory_high_increase_every: Duration, -} - -impl Config { - /// Calculate the new value for the cgroups memory.high based on system memory - pub fn calculate_memory_high_value(&self, total_system_mem: u64) -> u64 { - total_system_mem.saturating_sub(self.oom_buffer_bytes) - } + /// The number of samples used in constructing aggregated memory statistics + memory_history_len: usize, + /// The number of most recent samples that will be periodically logged. + /// + /// Each sample is logged exactly once. Increasing this value means that recent samples will be + /// logged less frequently, and vice versa. + /// + /// For simplicity, this value must be greater than or equal to `memory_history_len`. + memory_history_log_interval: usize, } impl Default for Config { fn default() -> Self { Self { - oom_buffer_bytes: 100 * MiB, - memory_high_buffer_bytes: 100 * MiB, - // while waiting for upscale, don't freeze for more than 20ms every 1s - max_upscale_wait: Duration::from_millis(20), - do_not_freeze_more_often_than: Duration::from_millis(1000), - // while waiting for upscale, increase memory.high by 10MiB every 25ms - memory_high_increase_by_bytes: 10 * MiB, - memory_high_increase_every: Duration::from_millis(25), - } - } -} - -/// Used to represent data that is associated with a certain point in time, such -/// as an upscale request or memory.high event. -/// -/// Internally, creating a `Sequenced` uses a static atomic counter to obtain -/// a unique sequence number. Sequence numbers are monotonically increasing, -/// allowing us to answer questions like "did this upscale happen after this -/// memory.high event?" by comparing the sequence numbers of the two events. -#[derive(Debug, Clone)] -pub struct Sequenced { - seqnum: u64, - data: T, -} - -impl Sequenced { - pub fn new(data: T) -> Self { - Self { - seqnum: EVENT_SEQUENCE_NUMBER.fetch_add(1, Ordering::AcqRel), - data, + memory_poll_interval: Duration::from_millis(100), + memory_history_len: 5, // use 500ms of history for decision-making + memory_history_log_interval: 20, // but only log every ~2s (otherwise it's spammy) } } } @@ -170,74 +47,14 @@ impl Sequenced { pub struct CgroupWatcher { pub config: Config, - /// The sequence number of the last upscale. - /// - /// If we receive a memory.high event that has a _lower_ sequence number than - /// `last_upscale_seqnum`, then we know it occured before the upscale, and we - /// can safely ignore it. - /// - /// Note: Like the `events` field, this doesn't _need_ interior mutability but we - /// use it anyways so that methods take `&self`, not `&mut self`. - last_upscale_seqnum: AtomicU64, - - /// A channel on which we send messages to request upscale from the dispatcher. - upscale_requester: mpsc::Sender<()>, - /// The actual cgroup we are watching and managing. cgroup: cgroups_rs::Cgroup, } -/// Read memory.events for the desired event type. -/// -/// `path` specifies the path to the desired `memory.events` file. -/// For more info, see the `memory.events` section of the [kernel docs] -/// -fn get_event_count(path: &str, event: MemoryEvent) -> anyhow::Result { - let contents = fs::read_to_string(path) - .with_context(|| format!("failed to read memory.events from {path}"))?; - - // Then contents of the file look like: - // low 42 - // high 101 - // ... - contents - .lines() - .filter_map(|s| s.split_once(' ')) - .find(|(e, _)| *e == event.as_str()) - .ok_or_else(|| anyhow!("failed to find entry for memory.{event} events in {path}")) - .and_then(|(_, count)| { - count - .parse::() - .with_context(|| format!("failed to parse memory.{event} as u64")) - }) -} - -/// Create an event stream that produces events whenever the file at the provided -/// path is modified. -fn create_file_watcher(path: &str) -> anyhow::Result> { - info!("creating file watcher for {path}"); - let inotify = Inotify::init().context("failed to initialize file watcher")?; - inotify - .watches() - .add(path, WatchMask::MODIFY) - .with_context(|| format!("failed to start watching {path}"))?; - inotify - // The inotify docs use [0u8; 1024] so we'll just copy them. We only need - // to store one event at a time - if the event gets written over, that's - // ok. We still see that there is an event. For more information, see: - // https://man7.org/linux/man-pages/man7/inotify.7.html - .into_event_stream([0u8; 1024]) - .context("failed to start inotify event stream") -} - impl CgroupWatcher { /// Create a new `CgroupWatcher`. #[tracing::instrument(skip_all, fields(%name))] - pub fn new( - name: String, - // A channel on which to send upscale requests - upscale_requester: mpsc::Sender<()>, - ) -> anyhow::Result<(Self, impl Stream>)> { + pub fn new(name: String) -> anyhow::Result { // TODO: clarify exactly why we need v2 // Make sure cgroups v2 (aka unified) are supported if !is_cgroup2_unified_mode() { @@ -245,410 +62,203 @@ impl CgroupWatcher { } let cgroup = cgroups_rs::Cgroup::load(hierarchies::auto(), &name); - // Start monitoring the cgroup for memory events. In general, for - // cgroups v2 (aka unified), metrics are reported in files like - // > `/sys/fs/cgroup/{name}/{metric}` - // We are looking for `memory.high` events, which are stored in the - // file `memory.events`. For more info, see the `memory.events` section - // of https://docs.kernel.org/admin-guide/cgroup-v2.html#memory-interface-files - let path = format!("{}/{}/memory.events", UNIFIED_MOUNTPOINT, &name); - let memory_events = create_file_watcher(&path) - .with_context(|| format!("failed to create event watcher for {path}"))? - // This would be nice with with .inspect_err followed by .ok - .filter_map(move |_| match get_event_count(&path, MemoryEvent::High) { - Ok(high) => Some(high), - Err(error) => { - // TODO: Might want to just panic here - warn!(?error, "failed to read high events count from {}", &path); - None - } - }) - // Only report the event if the memory.high count increased - .filter_map(|high| { - if MEMORY_EVENT_COUNT.fetch_max(high, Ordering::AcqRel) < high { - Some(high) - } else { - None - } - }) - .map(Sequenced::new); - - let initial_count = get_event_count( - &format!("{}/{}/memory.events", UNIFIED_MOUNTPOINT, &name), - MemoryEvent::High, - )?; - - info!(initial_count, "initial memory.high event count"); - - // Hard update `MEMORY_EVENT_COUNT` since there could have been processes - // running in the cgroup before that caused it to be non-zero. - MEMORY_EVENT_COUNT.fetch_max(initial_count, Ordering::AcqRel); - - Ok(( - Self { - cgroup, - upscale_requester, - last_upscale_seqnum: AtomicU64::new(0), - config: Default::default(), - }, - memory_events, - )) + Ok(Self { + cgroup, + config: Default::default(), + }) } /// The entrypoint for the `CgroupWatcher`. #[tracing::instrument(skip_all)] - pub async fn watch( + pub async fn watch( &self, - // These are ~dependency injected~ (fancy, I know) because this function - // should never return. - // -> therefore: when we tokio::spawn it, we don't await the JoinHandle. - // -> therefore: if we want to stick it in an Arc so many threads can access - // it, methods can never take mutable access. - // - note: we use the Arc strategy so that a) we can call this function - // right here and b) the runner can call the set/get_memory methods - // -> since calling recv() on a tokio::sync::mpsc::Receiver takes &mut self, - // we just pass them in here instead of holding them in fields, as that - // would require this method to take &mut self. - mut upscales: mpsc::Receiver>, - events: E, - ) -> anyhow::Result<()> - where - E: Stream>, - { - let mut wait_to_freeze = pin!(tokio::time::sleep(Duration::ZERO)); - let mut last_memory_high_increase_at: Option = None; - let mut events = pin!(events); - - // Are we waiting to be upscaled? Could be true if we request upscale due - // to a memory.high event and it does not arrive in time. - let mut waiting_on_upscale = false; - - loop { - tokio::select! { - upscale = upscales.recv() => { - let Sequenced { seqnum, data } = upscale - .context("failed to listen on upscale notification channel")?; - waiting_on_upscale = false; - last_memory_high_increase_at = None; - self.last_upscale_seqnum.store(seqnum, Ordering::Release); - info!(cpu = data.cpu, mem_bytes = data.mem, "received upscale"); - } - event = events.next() => { - let Some(Sequenced { seqnum, .. }) = event else { - bail!("failed to listen for memory.high events") - }; - // The memory.high came before our last upscale, so we consider - // it resolved - if self.last_upscale_seqnum.fetch_max(seqnum, Ordering::AcqRel) > seqnum { - info!( - "received memory.high event, but it came before our last upscale -> ignoring it" - ); - continue; - } - - // The memory.high came after our latest upscale. We don't - // want to do anything yet, so peek the next event in hopes - // that it's an upscale. - if let Some(upscale_num) = self - .upscaled(&mut upscales) - .context("failed to check if we were upscaled")? - { - if upscale_num > seqnum { - info!( - "received memory.high event, but it came before our last upscale -> ignoring it" - ); - continue; - } - } - - // If it's been long enough since we last froze, freeze the - // cgroup and request upscale - if wait_to_freeze.is_elapsed() { - info!("received memory.high event -> requesting upscale"); - waiting_on_upscale = self - .handle_memory_high_event(&mut upscales) - .await - .context("failed to handle upscale")?; - wait_to_freeze - .as_mut() - .reset(Instant::now() + self.config.do_not_freeze_more_often_than); - continue; - } - - // Ok, we can't freeze, just request upscale - if !waiting_on_upscale { - info!("received memory.high event, but too soon to refreeze -> requesting upscale"); - - // Make check to make sure we haven't been upscaled in the - // meantine (can happen if the agent independently decides - // to upscale us again) - if self - .upscaled(&mut upscales) - .context("failed to check if we were upscaled")? - .is_some() - { - info!("no need to request upscaling because we got upscaled"); - continue; - } - self.upscale_requester - .send(()) - .await - .context("failed to request upscale")?; - waiting_on_upscale = true; - continue; - } - - // Shoot, we can't freeze or and we're still waiting on upscale, - // increase memory.high to reduce throttling - let can_increase_memory_high = match last_memory_high_increase_at { - None => true, - Some(t) => t.elapsed() > self.config.memory_high_increase_every, - }; - if can_increase_memory_high { - info!( - "received memory.high event, \ - but too soon to refreeze and already requested upscale \ - -> increasing memory.high" - ); - - // Make check to make sure we haven't been upscaled in the - // meantine (can happen if the agent independently decides - // to upscale us again) - if self - .upscaled(&mut upscales) - .context("failed to check if we were upscaled")? - .is_some() - { - info!("no need to increase memory.high because got upscaled"); - continue; - } - - // Request upscale anyways (the agent will handle deduplicating - // requests) - self.upscale_requester - .send(()) - .await - .context("failed to request upscale")?; - - let memory_high = - self.get_memory_high_bytes().context("failed to get memory.high")?; - let new_high = memory_high + self.config.memory_high_increase_by_bytes; - info!( - current_high_bytes = memory_high, - new_high_bytes = new_high, - "updating memory.high" - ); - self.set_memory_high_bytes(new_high) - .context("failed to set memory.high")?; - last_memory_high_increase_at = Some(Instant::now()); - continue; - } - - info!("received memory.high event, but can't do anything"); - } - }; - } - } - - /// Handle a `memory.high`, returning whether we are still waiting on upscale - /// by the time the function returns. - /// - /// The general plan for handling a `memory.high` event is as follows: - /// 1. Freeze the cgroup - /// 2. Start a timer for `self.config.max_upscale_wait` - /// 3. Request upscale - /// 4. After the timer elapses or we receive upscale, thaw the cgroup. - /// 5. Return whether or not we are still waiting for upscale. If we are, - /// we'll increase the cgroups memory.high to avoid getting oom killed - #[tracing::instrument(skip_all)] - async fn handle_memory_high_event( - &self, - upscales: &mut mpsc::Receiver>, - ) -> anyhow::Result { - // Immediately freeze the cgroup before doing anything else. - info!("received memory.high event -> freezing cgroup"); - self.freeze().context("failed to freeze cgroup")?; - - // We'll use this for logging durations - let start_time = Instant::now(); - - // Await the upscale until we have to unfreeze - let timed = - tokio::time::timeout(self.config.max_upscale_wait, self.await_upscale(upscales)); - - // Request the upscale - info!( - wait = ?self.config.max_upscale_wait, - "sending request for immediate upscaling", - ); - self.upscale_requester - .send(()) - .await - .context("failed to request upscale")?; - - let waiting_on_upscale = match timed.await { - Ok(Ok(())) => { - info!(elapsed = ?start_time.elapsed(), "received upscale in time"); - false - } - // **important**: unfreeze the cgroup before ?-reporting the error - Ok(Err(e)) => { - info!("error waiting for upscale -> thawing cgroup"); - self.thaw() - .context("failed to thaw cgroup after errored waiting for upscale")?; - Err(e.context("failed to await upscale"))? - } - Err(_) => { - info!(elapsed = ?self.config.max_upscale_wait, "timed out waiting for upscale"); - true - } - }; - - info!("thawing cgroup"); - self.thaw().context("failed to thaw cgroup")?; - - Ok(waiting_on_upscale) - } - - /// Checks whether we were just upscaled, returning the upscale's sequence - /// number if so. - #[tracing::instrument(skip_all)] - fn upscaled( - &self, - upscales: &mut mpsc::Receiver>, - ) -> anyhow::Result> { - let Sequenced { seqnum, data } = match upscales.try_recv() { - Ok(upscale) => upscale, - Err(TryRecvError::Empty) => return Ok(None), - Err(TryRecvError::Disconnected) => { - bail!("upscale notification channel was disconnected") - } - }; - - // Make sure to update the last upscale sequence number - self.last_upscale_seqnum.store(seqnum, Ordering::Release); - info!(cpu = data.cpu, mem_bytes = data.mem, "received upscale"); - Ok(Some(seqnum)) - } - - /// Await an upscale event, discarding any `memory.high` events received in - /// the process. - /// - /// This is used in `handle_memory_high_event`, where we need to listen - /// for upscales in particular so we know if we can thaw the cgroup early. - #[tracing::instrument(skip_all)] - async fn await_upscale( - &self, - upscales: &mut mpsc::Receiver>, + updates: watch::Sender<(Instant, MemoryHistory)>, ) -> anyhow::Result<()> { - let Sequenced { seqnum, .. } = upscales - .recv() - .await - .context("error listening for upscales")?; + // this requirement makes the code a bit easier to work with; see the config for more. + assert!(self.config.memory_history_len <= self.config.memory_history_log_interval); - self.last_upscale_seqnum.store(seqnum, Ordering::Release); - Ok(()) - } + let mut ticker = tokio::time::interval(self.config.memory_poll_interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // ticker.reset_immediately(); // FIXME: enable this once updating to tokio >= 1.30.0 - /// Get the cgroup's name. - pub fn path(&self) -> &str { - self.cgroup.path() - } -} + let mem_controller = self.memory()?; -// Methods for manipulating the actual cgroup -impl CgroupWatcher { - /// Get a handle on the freezer subsystem. - fn freezer(&self) -> anyhow::Result<&FreezerController> { - if let Some(Freezer(freezer)) = self - .cgroup - .subsystems() - .iter() - .find(|sub| matches!(sub, Freezer(_))) - { - Ok(freezer) - } else { - anyhow::bail!("could not find freezer subsystem") + // buffer for samples that will be logged. once full, it remains so. + let history_log_len = self.config.memory_history_log_interval; + let mut history_log_buf = vec![MemoryStatus::zeroed(); history_log_len]; + + for t in 0_u64.. { + ticker.tick().await; + + let now = Instant::now(); + let mem = Self::memory_usage(mem_controller); + + let i = t as usize % history_log_len; + history_log_buf[i] = mem; + + // We're taking *at most* memory_history_len values; we may be bounded by the total + // number of samples that have come in so far. + let samples_count = (t + 1).min(self.config.memory_history_len as u64) as usize; + // NB: in `ring_buf_recent_values_iter`, `i` is *inclusive*, which matches the fact + // that we just inserted a value there, so the end of the iterator will *include* the + // value at i, rather than stopping just short of it. + let samples = ring_buf_recent_values_iter(&history_log_buf, i, samples_count); + + let summary = MemoryHistory { + avg_non_reclaimable: samples.map(|h| h.non_reclaimable).sum::() + / samples_count as u64, + samples_count, + samples_span: self.config.memory_poll_interval * (samples_count - 1) as u32, + }; + + // Log the current history if it's time to do so. Because `history_log_buf` has length + // equal to the logging interval, we can just log the entire buffer every time we set + // the last entry, which also means that for this log line, we can ignore that it's a + // ring buffer (because all the entries are in order of increasing time). + if i == history_log_len - 1 { + info!( + history = ?MemoryStatus::debug_slice(&history_log_buf), + summary = ?summary, + "Recent cgroup memory statistics history" + ); + } + + updates + .send((now, summary)) + .context("failed to send MemoryHistory")?; } - } - /// Attempt to freeze the cgroup. - pub fn freeze(&self) -> anyhow::Result<()> { - self.freezer() - .context("failed to get freezer subsystem")? - .freeze() - .context("failed to freeze") - } - - /// Attempt to thaw the cgroup. - pub fn thaw(&self) -> anyhow::Result<()> { - self.freezer() - .context("failed to get freezer subsystem")? - .thaw() - .context("failed to thaw") + unreachable!() } /// Get a handle on the memory subsystem. - /// - /// Note: this method does not require `self.memory_update_lock` because - /// getting a handle to the subsystem does not access any of the files we - /// care about, such as memory.high and memory.events fn memory(&self) -> anyhow::Result<&MemController> { - if let Some(Mem(memory)) = self - .cgroup + self.cgroup .subsystems() .iter() - .find(|sub| matches!(sub, Mem(_))) - { - Ok(memory) - } else { - anyhow::bail!("could not find memory subsystem") - } - } - - /// Get cgroup current memory usage. - pub fn current_memory_usage(&self) -> anyhow::Result { - Ok(self - .memory() - .context("failed to get memory subsystem")? - .memory_stat() - .usage_in_bytes) - } - - /// Set cgroup memory.high threshold. - pub fn set_memory_high_bytes(&self, bytes: u64) -> anyhow::Result<()> { - self.set_memory_high_internal(MaxValue::Value(u64::min(bytes, i64::MAX as u64) as i64)) - } - - /// Set the cgroup's memory.high to 'max', disabling it. - pub fn unset_memory_high(&self) -> anyhow::Result<()> { - self.set_memory_high_internal(MaxValue::Max) - } - - fn set_memory_high_internal(&self, value: MaxValue) -> anyhow::Result<()> { - self.memory() - .context("failed to get memory subsystem")? - .set_mem(cgroups_rs::memory::SetMemory { - low: None, - high: Some(value), - min: None, - max: None, + .find_map(|sub| match sub { + Subsystem::Mem(c) => Some(c), + _ => None, }) - .map_err(anyhow::Error::from) + .ok_or_else(|| anyhow!("could not find memory subsystem")) } - /// Get memory.high threshold. - pub fn get_memory_high_bytes(&self) -> anyhow::Result { - let high = self - .memory() - .context("failed to get memory subsystem while getting memory statistics")? - .get_mem() - .map(|mem| mem.high) - .context("failed to get memory statistics from subsystem")?; - match high { - Some(MaxValue::Max) => Ok(i64::MAX as u64), - Some(MaxValue::Value(high)) => Ok(high as u64), - None => anyhow::bail!("failed to read memory.high from memory subsystem"), + /// Given a handle on the memory subsystem, returns the current memory information + fn memory_usage(mem_controller: &MemController) -> MemoryStatus { + let stat = mem_controller.memory_stat().stat; + MemoryStatus { + non_reclaimable: stat.active_anon + stat.inactive_anon, } } } + +// Helper function for `CgroupWatcher::watch` +fn ring_buf_recent_values_iter( + buf: &[T], + last_value_idx: usize, + count: usize, +) -> impl '_ + Iterator { + // Assertion carried over from `CgroupWatcher::watch`, to make the logic in this function + // easier (we only have to add `buf.len()` once, rather than a dynamic number of times). + assert!(count <= buf.len()); + + buf.iter() + // 'cycle' because the values could wrap around + .cycle() + // with 'cycle', this skip is more like 'offset', and functionally this is + // offsettting by 'last_value_idx - count (mod buf.len())', but we have to be + // careful to avoid underflow, so we pre-add buf.len(). + // The '+ 1' is because `last_value_idx` is inclusive, rather than exclusive. + .skip((buf.len() + last_value_idx + 1 - count) % buf.len()) + .take(count) +} + +/// Summary of recent memory usage +#[derive(Debug, Copy, Clone)] +pub struct MemoryHistory { + /// Rolling average of non-reclaimable memory usage samples over the last `history_period` + pub avg_non_reclaimable: u64, + + /// The number of samples used to construct this summary + pub samples_count: usize, + /// Total timespan between the first and last sample used for this summary + pub samples_span: Duration, +} + +#[derive(Debug, Copy, Clone)] +pub struct MemoryStatus { + non_reclaimable: u64, +} + +impl MemoryStatus { + fn zeroed() -> Self { + MemoryStatus { non_reclaimable: 0 } + } + + fn debug_slice(slice: &[Self]) -> impl '_ + Debug { + struct DS<'a>(&'a [MemoryStatus]); + + impl<'a> Debug for DS<'a> { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.debug_struct("[MemoryStatus]") + .field( + "non_reclaimable[..]", + &Fields(self.0, |stat: &MemoryStatus| { + BytesToGB(stat.non_reclaimable) + }), + ) + .finish() + } + } + + struct Fields<'a, F>(&'a [MemoryStatus], F); + + impl<'a, F: Fn(&MemoryStatus) -> T, T: Debug> Debug for Fields<'a, F> { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.debug_list().entries(self.0.iter().map(&self.1)).finish() + } + } + + struct BytesToGB(u64); + + impl Debug for BytesToGB { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.write_fmt(format_args!( + "{:.3}Gi", + self.0 as f64 / (1_u64 << 30) as f64 + )) + } + } + + DS(slice) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn ring_buf_iter() { + let buf = vec![0_i32, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + + let values = |offset, count| { + super::ring_buf_recent_values_iter(&buf, offset, count) + .copied() + .collect::>() + }; + + // Boundary conditions: start, end, and entire thing: + assert_eq!(values(0, 1), [0]); + assert_eq!(values(3, 4), [0, 1, 2, 3]); + assert_eq!(values(9, 4), [6, 7, 8, 9]); + assert_eq!(values(9, 10), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + + // "normal" operation: no wraparound + assert_eq!(values(7, 4), [4, 5, 6, 7]); + + // wraparound: + assert_eq!(values(0, 4), [7, 8, 9, 0]); + assert_eq!(values(1, 4), [8, 9, 0, 1]); + assert_eq!(values(2, 4), [9, 0, 1, 2]); + assert_eq!(values(2, 10), [3, 4, 5, 6, 7, 8, 9, 0, 1, 2]); + } +} diff --git a/libs/vm_monitor/src/dispatcher.rs b/libs/vm_monitor/src/dispatcher.rs index 109a68fff1..c76baf04e7 100644 --- a/libs/vm_monitor/src/dispatcher.rs +++ b/libs/vm_monitor/src/dispatcher.rs @@ -12,12 +12,10 @@ use futures::{ stream::{SplitSink, SplitStream}, SinkExt, StreamExt, }; -use tokio::sync::mpsc; use tracing::info; -use crate::cgroup::Sequenced; use crate::protocol::{ - OutboundMsg, ProtocolRange, ProtocolResponse, ProtocolVersion, Resources, PROTOCOL_MAX_VERSION, + OutboundMsg, ProtocolRange, ProtocolResponse, ProtocolVersion, PROTOCOL_MAX_VERSION, PROTOCOL_MIN_VERSION, }; @@ -36,13 +34,6 @@ pub struct Dispatcher { /// We send messages to the agent through `sink` sink: SplitSink, - /// Used to notify the cgroup when we are upscaled. - pub(crate) notify_upscale_events: mpsc::Sender>, - - /// When the cgroup requests upscale it will send on this channel. In response - /// we send an `UpscaleRequst` to the agent. - pub(crate) request_upscale_events: mpsc::Receiver<()>, - /// The protocol version we have agreed to use with the agent. This is negotiated /// during the creation of the dispatcher, and should be the highest shared protocol /// version. @@ -61,11 +52,7 @@ impl Dispatcher { /// 1. Wait for the agent to sent the range of protocols it supports. /// 2. Send a protocol version that works for us as well, or an error if there /// is no compatible version. - pub async fn new( - stream: WebSocket, - notify_upscale_events: mpsc::Sender>, - request_upscale_events: mpsc::Receiver<()>, - ) -> anyhow::Result { + pub async fn new(stream: WebSocket) -> anyhow::Result { let (mut sink, mut source) = stream.split(); // Figure out the highest protocol version we both support @@ -119,22 +106,10 @@ impl Dispatcher { Ok(Self { sink, source, - notify_upscale_events, - request_upscale_events, proto_version: highest_shared_version, }) } - /// Notify the cgroup manager that we have received upscale and wait for - /// the acknowledgement. - #[tracing::instrument(skip_all, fields(?resources))] - pub async fn notify_upscale(&self, resources: Sequenced) -> anyhow::Result<()> { - self.notify_upscale_events - .send(resources) - .await - .context("failed to send resources and oneshot sender across channel") - } - /// Send a message to the agent. /// /// Although this function is small, it has one major benefit: it is the only diff --git a/libs/vm_monitor/src/runner.rs b/libs/vm_monitor/src/runner.rs index b0ee5f0310..a7a0995797 100644 --- a/libs/vm_monitor/src/runner.rs +++ b/libs/vm_monitor/src/runner.rs @@ -5,18 +5,16 @@ //! all functionality. use std::fmt::Debug; -use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{bail, Context}; use axum::extract::ws::{Message, WebSocket}; use futures::StreamExt; -use tokio::sync::broadcast; -use tokio::sync::mpsc; +use tokio::sync::{broadcast, watch}; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; -use crate::cgroup::{CgroupWatcher, Sequenced}; +use crate::cgroup::{self, CgroupWatcher}; use crate::dispatcher::Dispatcher; use crate::filecache::{FileCacheConfig, FileCacheState}; use crate::protocol::{InboundMsg, InboundMsgKind, OutboundMsg, OutboundMsgKind, Resources}; @@ -28,7 +26,7 @@ use crate::{bytes_to_mebibytes, get_total_system_memory, spawn_with_cancel, Args pub struct Runner { config: Config, filecache: Option, - cgroup: Option>, + cgroup: Option, dispatcher: Dispatcher, /// We "mint" new message ids by incrementing this counter and taking the value. @@ -45,6 +43,14 @@ pub struct Runner { kill: broadcast::Receiver<()>, } +#[derive(Debug)] +struct CgroupState { + watcher: watch::Receiver<(Instant, cgroup::MemoryHistory)>, + /// If [`cgroup::MemoryHistory::avg_non_reclaimable`] exceeds `threshold`, we send upscale + /// requests. + threshold: u64, +} + /// Configuration for a `Runner` #[derive(Debug)] pub struct Config { @@ -62,16 +68,56 @@ pub struct Config { /// upscale resource amounts (because we might not *actually* have been upscaled yet). This field /// should be removed once we have a better solution there. sys_buffer_bytes: u64, + + /// Minimum fraction of total system memory reserved *before* the the cgroup threshold; in + /// other words, providing a ceiling for the highest value of the threshold by enforcing that + /// there's at least `cgroup_min_overhead_fraction` of the total memory remaining beyond the + /// threshold. + /// + /// For example, a value of `0.1` means that 10% of total memory must remain after exceeding + /// the threshold, so the value of the cgroup threshold would always be capped at 90% of total + /// memory. + /// + /// The default value of `0.15` means that we *guarantee* sending upscale requests if the + /// cgroup is using more than 85% of total memory (even if we're *not* separately reserving + /// memory for the file cache). + cgroup_min_overhead_fraction: f64, + + cgroup_downscale_threshold_buffer_bytes: u64, } impl Default for Config { fn default() -> Self { Self { sys_buffer_bytes: 100 * MiB, + cgroup_min_overhead_fraction: 0.15, + cgroup_downscale_threshold_buffer_bytes: 100 * MiB, } } } +impl Config { + fn cgroup_threshold(&self, total_mem: u64, file_cache_disk_size: u64) -> u64 { + // If the file cache is in tmpfs, then it will count towards shmem usage of the cgroup, + // and thus be non-reclaimable, so we should allow for additional memory usage. + // + // If the file cache sits on disk, our desired stable system state is for it to be fully + // page cached (its contents should only be paged to/from disk in situations where we can't + // upscale fast enough). Page-cached memory is reclaimable, so we need to lower the + // threshold for non-reclaimable memory so we scale up *before* the kernel starts paging + // out the file cache. + let memory_remaining_for_cgroup = total_mem.saturating_sub(file_cache_disk_size); + + // Even if we're not separately making room for the file cache (if it's in tmpfs), we still + // want our threshold to be met gracefully instead of letting postgres get OOM-killed. + // So we guarantee that there's at least `cgroup_min_overhead_fraction` of total memory + // remaining above the threshold. + let max_threshold = (total_mem as f64 * (1.0 - self.cgroup_min_overhead_fraction)) as u64; + + memory_remaining_for_cgroup.min(max_threshold) + } +} + impl Runner { /// Create a new monitor. #[tracing::instrument(skip_all, fields(?config, ?args))] @@ -87,12 +133,7 @@ impl Runner { "invalid monitor Config: sys_buffer_bytes cannot be 0" ); - // *NOTE*: the dispatcher and cgroup manager talk through these channels - // so make sure they each get the correct half, nothing is droppped, etc. - let (notified_send, notified_recv) = mpsc::channel(1); - let (requesting_send, requesting_recv) = mpsc::channel(1); - - let dispatcher = Dispatcher::new(ws, notified_send, requesting_recv) + let dispatcher = Dispatcher::new(ws) .await .context("error creating new dispatcher")?; @@ -106,46 +147,10 @@ impl Runner { kill, }; - // If we have both the cgroup and file cache integrations enabled, it's possible for - // temporary failures to result in cgroup throttling (from memory.high), that in turn makes - // it near-impossible to connect to the file cache (because it times out). Unfortunately, - // we *do* still want to determine the file cache size before setting the cgroup's - // memory.high, so it's not as simple as just swapping the order. - // - // Instead, the resolution here is that on vm-monitor startup (note: happens on each - // connection from autoscaler-agent, possibly multiple times per compute_ctl lifecycle), we - // temporarily unset memory.high, to allow any existing throttling to dissipate. It's a bit - // of a hacky solution, but helps with reliability. - if let Some(name) = &args.cgroup { - // Best not to set up cgroup stuff more than once, so we'll initialize cgroup state - // now, and then set limits later. - info!("initializing cgroup"); - - let (cgroup, cgroup_event_stream) = CgroupWatcher::new(name.clone(), requesting_send) - .context("failed to create cgroup manager")?; - - info!("temporarily unsetting memory.high"); - - // Temporarily un-set cgroup memory.high; see above. - cgroup - .unset_memory_high() - .context("failed to unset memory.high")?; - - let cgroup = Arc::new(cgroup); - - let cgroup_clone = Arc::clone(&cgroup); - spawn_with_cancel( - token.clone(), - |_| error!("cgroup watcher terminated"), - async move { cgroup_clone.watch(notified_recv, cgroup_event_stream).await }, - ); - - state.cgroup = Some(cgroup); - } - - let mut file_cache_reserved_bytes = 0; let mem = get_total_system_memory(); + let mut file_cache_disk_size = 0; + // We need to process file cache initialization before cgroup initialization, so that the memory // allocated to the file cache is appropriately taken into account when we decide the cgroup's // memory limits. @@ -156,7 +161,7 @@ impl Runner { false => FileCacheConfig::default_in_memory(), }; - let mut file_cache = FileCacheState::new(connstr, config, token) + let mut file_cache = FileCacheState::new(connstr, config, token.clone()) .await .context("failed to create file cache")?; @@ -181,23 +186,40 @@ impl Runner { if actual_size != new_size { info!("file cache size actually got set to {actual_size}") } - // Mark the resources given to the file cache as reserved, but only if it's in memory. - if !args.file_cache_on_disk { - file_cache_reserved_bytes = actual_size; + + if args.file_cache_on_disk { + file_cache_disk_size = actual_size; } state.filecache = Some(file_cache); } - if let Some(cgroup) = &state.cgroup { - let available = mem - file_cache_reserved_bytes; - let value = cgroup.config.calculate_memory_high_value(available); + if let Some(name) = &args.cgroup { + // Best not to set up cgroup stuff more than once, so we'll initialize cgroup state + // now, and then set limits later. + info!("initializing cgroup"); - info!(value, "setting memory.high"); + let cgroup = + CgroupWatcher::new(name.clone()).context("failed to create cgroup manager")?; - cgroup - .set_memory_high_bytes(value) - .context("failed to set cgroup memory.high")?; + let init_value = cgroup::MemoryHistory { + avg_non_reclaimable: 0, + samples_count: 0, + samples_span: Duration::ZERO, + }; + let (hist_tx, hist_rx) = watch::channel((Instant::now(), init_value)); + + spawn_with_cancel(token, |_| error!("cgroup watcher terminated"), async move { + cgroup.watch(hist_tx).await + }); + + let threshold = state.config.cgroup_threshold(mem, file_cache_disk_size); + info!(threshold, "set initial cgroup threshold",); + + state.cgroup = Some(CgroupState { + watcher: hist_rx, + threshold, + }); } Ok(state) @@ -217,28 +239,40 @@ impl Runner { let requested_mem = target.mem; let usable_system_memory = requested_mem.saturating_sub(self.config.sys_buffer_bytes); - let expected_file_cache_mem_usage = self + let (expected_file_cache_size, expected_file_cache_disk_size) = self .filecache .as_ref() - .map(|file_cache| file_cache.config.calculate_cache_size(usable_system_memory)) - .unwrap_or(0); - let mut new_cgroup_mem_high = 0; + .map(|file_cache| { + let size = file_cache.config.calculate_cache_size(usable_system_memory); + match file_cache.config.in_memory { + true => (size, 0), + false => (size, size), + } + }) + .unwrap_or((0, 0)); if let Some(cgroup) = &self.cgroup { - new_cgroup_mem_high = cgroup + let (last_time, last_history) = *cgroup.watcher.borrow(); + + // TODO: make the duration here configurable. + if last_time.elapsed() > Duration::from_secs(5) { + bail!("haven't gotten cgroup memory stats recently enough to determine downscaling information"); + } else if last_history.samples_count <= 1 { + bail!("haven't received enough cgroup memory stats yet"); + } + + let new_threshold = self .config - .calculate_memory_high_value(usable_system_memory - expected_file_cache_mem_usage); + .cgroup_threshold(usable_system_memory, expected_file_cache_disk_size); - let current = cgroup - .current_memory_usage() - .context("failed to fetch cgroup memory")?; + let current = last_history.avg_non_reclaimable; - if new_cgroup_mem_high < current + cgroup.config.memory_high_buffer_bytes { + if new_threshold < current + self.config.cgroup_downscale_threshold_buffer_bytes { let status = format!( - "{}: {} MiB (new high) < {} (current usage) + {} (buffer)", - "calculated memory.high too low", - bytes_to_mebibytes(new_cgroup_mem_high), + "{}: {} MiB (new threshold) < {} (current usage) + {} (downscale buffer)", + "calculated memory threshold too low", + bytes_to_mebibytes(new_threshold), bytes_to_mebibytes(current), - bytes_to_mebibytes(cgroup.config.memory_high_buffer_bytes) + bytes_to_mebibytes(self.config.cgroup_downscale_threshold_buffer_bytes) ); info!(status, "discontinuing downscale"); @@ -249,14 +283,14 @@ impl Runner { // The downscaling has been approved. Downscale the file cache, then the cgroup. let mut status = vec![]; - let mut file_cache_mem_usage = 0; + let mut file_cache_disk_size = 0; if let Some(file_cache) = &mut self.filecache { let actual_usage = file_cache - .set_file_cache_size(expected_file_cache_mem_usage) + .set_file_cache_size(expected_file_cache_size) .await .context("failed to set file cache size")?; - if file_cache.config.in_memory { - file_cache_mem_usage = actual_usage; + if !file_cache.config.in_memory { + file_cache_disk_size = actual_usage; } let message = format!( "set file cache size to {} MiB (in memory = {})", @@ -267,24 +301,18 @@ impl Runner { status.push(message); } - if let Some(cgroup) = &self.cgroup { - let available_memory = usable_system_memory - file_cache_mem_usage; - - if file_cache_mem_usage != expected_file_cache_mem_usage { - new_cgroup_mem_high = cgroup.config.calculate_memory_high_value(available_memory); - } - - // new_cgroup_mem_high is initialized to 0 but it is guaranteed to not be here - // since it is properly initialized in the previous cgroup if let block - cgroup - .set_memory_high_bytes(new_cgroup_mem_high) - .context("failed to set cgroup memory.high")?; + if let Some(cgroup) = &mut self.cgroup { + let new_threshold = self + .config + .cgroup_threshold(usable_system_memory, file_cache_disk_size); let message = format!( - "set cgroup memory.high to {} MiB, of new max {} MiB", - bytes_to_mebibytes(new_cgroup_mem_high), - bytes_to_mebibytes(available_memory) + "set cgroup memory threshold from {} MiB to {} MiB, of new total {} MiB", + bytes_to_mebibytes(cgroup.threshold), + bytes_to_mebibytes(new_threshold), + bytes_to_mebibytes(usable_system_memory) ); + cgroup.threshold = new_threshold; info!("downscale: {message}"); status.push(message); } @@ -305,8 +333,7 @@ impl Runner { let new_mem = resources.mem; let usable_system_memory = new_mem.saturating_sub(self.config.sys_buffer_bytes); - // Get the file cache's expected contribution to the memory usage - let mut file_cache_mem_usage = 0; + let mut file_cache_disk_size = 0; if let Some(file_cache) = &mut self.filecache { let expected_usage = file_cache.config.calculate_cache_size(usable_system_memory); info!( @@ -319,8 +346,8 @@ impl Runner { .set_file_cache_size(expected_usage) .await .context("failed to set file cache size")?; - if file_cache.config.in_memory { - file_cache_mem_usage = actual_usage; + if !file_cache.config.in_memory { + file_cache_disk_size = actual_usage; } if actual_usage != expected_usage { @@ -332,18 +359,18 @@ impl Runner { } } - if let Some(cgroup) = &self.cgroup { - let available_memory = usable_system_memory - file_cache_mem_usage; - let new_cgroup_mem_high = cgroup.config.calculate_memory_high_value(available_memory); + if let Some(cgroup) = &mut self.cgroup { + let new_threshold = self + .config + .cgroup_threshold(usable_system_memory, file_cache_disk_size); + info!( - target = bytes_to_mebibytes(new_cgroup_mem_high), - total = bytes_to_mebibytes(new_mem), - name = cgroup.path(), - "updating cgroup memory.high", + "set cgroup memory threshold from {} MiB to {} MiB of new total {} MiB", + bytes_to_mebibytes(cgroup.threshold), + bytes_to_mebibytes(new_threshold), + bytes_to_mebibytes(usable_system_memory) ); - cgroup - .set_memory_high_bytes(new_cgroup_mem_high) - .context("failed to set cgroup memory.high")?; + cgroup.threshold = new_threshold; } Ok(()) @@ -361,10 +388,6 @@ impl Runner { self.handle_upscale(granted) .await .context("failed to handle upscale")?; - self.dispatcher - .notify_upscale(Sequenced::new(granted)) - .await - .context("failed to notify notify cgroup of upscale")?; Ok(Some(OutboundMsg::new( OutboundMsgKind::UpscaleConfirmation {}, id, @@ -408,33 +431,53 @@ impl Runner { Err(e) => bail!("failed to receive kill signal: {e}") } } - // we need to propagate an upscale request - request = self.dispatcher.request_upscale_events.recv(), if self.cgroup.is_some() => { - if request.is_none() { - bail!("failed to listen for upscale event from cgroup") + + // New memory stats from the cgroup, *may* need to request upscaling, if we've + // exceeded the threshold + result = self.cgroup.as_mut().unwrap().watcher.changed(), if self.cgroup.is_some() => { + result.context("failed to receive from cgroup memory stats watcher")?; + + let cgroup = self.cgroup.as_ref().unwrap(); + + let (_time, cgroup_mem_stat) = *cgroup.watcher.borrow(); + + // If we haven't exceeded the threshold, then we're all ok + if cgroup_mem_stat.avg_non_reclaimable < cgroup.threshold { + continue; } - // If it's been less than 1 second since the last time we requested upscaling, - // ignore the event, to avoid spamming the agent (otherwise, this can happen - // ~1k times per second). + // Otherwise, we generally want upscaling. But, if it's been less than 1 second + // since the last time we requested upscaling, ignore the event, to avoid + // spamming the agent. if let Some(t) = self.last_upscale_request_at { let elapsed = t.elapsed(); if elapsed < Duration::from_secs(1) { - info!(elapsed_millis = elapsed.as_millis(), "cgroup asked for upscale but too soon to forward the request, ignoring"); + info!( + elapsed_millis = elapsed.as_millis(), + avg_non_reclaimable = bytes_to_mebibytes(cgroup_mem_stat.avg_non_reclaimable), + threshold = bytes_to_mebibytes(cgroup.threshold), + "cgroup memory stats are high enough to upscale but too soon to forward the request, ignoring", + ); continue; } } self.last_upscale_request_at = Some(Instant::now()); - info!("cgroup asking for upscale; forwarding request"); + info!( + avg_non_reclaimable = bytes_to_mebibytes(cgroup_mem_stat.avg_non_reclaimable), + threshold = bytes_to_mebibytes(cgroup.threshold), + "cgroup memory stats are high enough to upscale, requesting upscale", + ); + self.counter += 2; // Increment, preserving parity (i.e. keep the // counter odd). See the field comment for more. self.dispatcher .send(OutboundMsg::new(OutboundMsgKind::UpscaleRequest {}, self.counter)) .await .context("failed to send message")?; - } + }, + // there is a message from the agent msg = self.dispatcher.source.next() => { if let Some(msg) = msg { From 16c87b5bdaa9e1de101fc10e4b875d7dbb0b585d Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Wed, 18 Oct 2023 02:10:01 -0700 Subject: [PATCH 13/49] Bump vm-builder v0.17.12 -> v0.18.1 (#5583) Only applicable change was neondatabase/autoscaling#566, updating pgbouncer to 1.21.0 and enabling support for prepared statements. --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a41258c401..1fed98f202 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -834,7 +834,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.17.12 + VM_BUILDER_VERSION: v0.18.1 steps: - name: Checkout From 9da67c4f19f08c745e47e4f8b2b6952c4f27d9c7 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 18 Oct 2023 12:23:06 +0200 Subject: [PATCH 14/49] walredo: make request_redo() an async fn (#5559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stacked atop https://github.com/neondatabase/neon/pull/5557 Prep work for https://github.com/neondatabase/neon/pull/5560 These changes have a 2% impact on `bench_walredo`. That's likely because of the `block_on() in the innermost piece of benchmark-only code. So, it doesn't affect production code. The use of closures in the benchmarking code prevents a straightforward conversion of the whole benchmarking code to async. before: ``` $ cargo bench --features testing --bench bench_walredo Compiling pageserver v0.1.0 (/home/cs/src/neon/pageserver) Finished bench [optimized + debuginfo] target(s) in 2m 11s Running benches/bench_walredo.rs (target/release/deps/bench_walredo-d99a324337dead70) Gnuplot not found, using plotters backend short/short/1 time: [26.363 µs 27.451 µs 28.573 µs] Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) high mild short/short/2 time: [64.340 µs 64.927 µs 65.485 µs] Found 2 outliers among 100 measurements (2.00%) 2 (2.00%) low mild short/short/4 time: [101.98 µs 104.06 µs 106.13 µs] short/short/8 time: [151.42 µs 152.74 µs 154.03 µs] short/short/16 time: [296.30 µs 297.53 µs 298.88 µs] Found 14 outliers among 100 measurements (14.00%) 10 (10.00%) high mild 4 (4.00%) high severe medium/medium/1 time: [225.12 µs 225.90 µs 226.66 µs] Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) low mild medium/medium/2 time: [490.80 µs 491.64 µs 492.49 µs] Found 1 outliers among 100 measurements (1.00%) 1 (1.00%) low mild medium/medium/4 time: [934.47 µs 936.49 µs 938.52 µs] Found 5 outliers among 100 measurements (5.00%) 3 (3.00%) low mild 1 (1.00%) high mild 1 (1.00%) high severe medium/medium/8 time: [1.8364 ms 1.8412 ms 1.8463 ms] Found 4 outliers among 100 measurements (4.00%) 4 (4.00%) high mild medium/medium/16 time: [3.6694 ms 3.6896 ms 3.7104 ms] ``` after: ``` $ cargo bench --features testing --bench bench_walredo Compiling pageserver v0.1.0 (/home/cs/src/neon/pageserver) Finished bench [optimized + debuginfo] target(s) in 2m 11s Running benches/bench_walredo.rs (target/release/deps/bench_walredo-d99a324337dead70) Gnuplot not found, using plotters backend short/short/1 time: [28.345 µs 28.529 µs 28.699 µs] change: [-0.2201% +3.9276% +8.2451%] (p = 0.07 > 0.05) No change in performance detected. Found 17 outliers among 100 measurements (17.00%) 4 (4.00%) low severe 5 (5.00%) high mild 8 (8.00%) high severe short/short/2 time: [66.145 µs 66.719 µs 67.274 µs] change: [+1.5467% +2.7605% +3.9927%] (p = 0.00 < 0.05) Performance has regressed. Found 5 outliers among 100 measurements (5.00%) 5 (5.00%) low mild short/short/4 time: [105.51 µs 107.52 µs 109.49 µs] change: [+0.5023% +3.3196% +6.1986%] (p = 0.02 < 0.05) Change within noise threshold. short/short/8 time: [151.90 µs 153.16 µs 154.41 µs] change: [-1.0001% +0.2779% +1.4221%] (p = 0.65 > 0.05) No change in performance detected. short/short/16 time: [297.38 µs 298.26 µs 299.20 µs] change: [-0.2953% +0.2462% +0.7763%] (p = 0.37 > 0.05) No change in performance detected. Found 2 outliers among 100 measurements (2.00%) 2 (2.00%) high mild medium/medium/1 time: [229.76 µs 230.72 µs 231.69 µs] change: [+1.5804% +2.1354% +2.6635%] (p = 0.00 < 0.05) Performance has regressed. medium/medium/2 time: [501.14 µs 502.31 µs 503.64 µs] change: [+1.8730% +2.1709% +2.5199%] (p = 0.00 < 0.05) Performance has regressed. Found 7 outliers among 100 measurements (7.00%) 1 (1.00%) low mild 1 (1.00%) high mild 5 (5.00%) high severe medium/medium/4 time: [954.15 µs 956.74 µs 959.33 µs] change: [+1.7962% +2.1627% +2.4905%] (p = 0.00 < 0.05) Performance has regressed. medium/medium/8 time: [1.8726 ms 1.8785 ms 1.8848 ms] change: [+1.5858% +2.0240% +2.4626%] (p = 0.00 < 0.05) Performance has regressed. Found 6 outliers among 100 measurements (6.00%) 1 (1.00%) low mild 3 (3.00%) high mild 2 (2.00%) high severe medium/medium/16 time: [3.7565 ms 3.7746 ms 3.7934 ms] change: [+1.5503% +2.3044% +3.0818%] (p = 0.00 < 0.05) Performance has regressed. Found 3 outliers among 100 measurements (3.00%) 3 (3.00%) high mild ``` --- pageserver/benches/bench_walredo.rs | 68 +++++++++++++++++++--------- pageserver/src/deletion_queue.rs | 4 -- pageserver/src/tenant.rs | 70 ++++++++++++++++++++++++----- pageserver/src/tenant/timeline.rs | 6 +-- pageserver/src/walredo.rs | 37 ++++----------- 5 files changed, 116 insertions(+), 69 deletions(-) diff --git a/pageserver/benches/bench_walredo.rs b/pageserver/benches/bench_walredo.rs index 9bcd3fa708..ba41866935 100644 --- a/pageserver/benches/bench_walredo.rs +++ b/pageserver/benches/bench_walredo.rs @@ -32,9 +32,15 @@ fn redo_scenarios(c: &mut Criterion) { let manager = Arc::new(manager); - tracing::info!("executing first"); - short().execute(&manager).unwrap(); - tracing::info!("first executed"); + { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + tracing::info!("executing first"); + short().execute(rt.handle(), &manager).unwrap(); + tracing::info!("first executed"); + } let thread_counts = [1, 2, 4, 8, 16]; @@ -77,9 +83,14 @@ fn add_multithreaded_walredo_requesters( assert_ne!(threads, 0); if threads == 1 { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let handle = rt.handle(); b.iter_batched_ref( || Some(input_factory()), - |input| execute_all(input.take(), manager), + |input| execute_all(input.take(), handle, manager), criterion::BatchSize::PerIteration, ); } else { @@ -95,19 +106,26 @@ fn add_multithreaded_walredo_requesters( let manager = manager.clone(); let barrier = barrier.clone(); let work_rx = work_rx.clone(); - move || loop { - // queue up and wait if we want to go another round - if work_rx.lock().unwrap().recv().is_err() { - break; + move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let handle = rt.handle(); + loop { + // queue up and wait if we want to go another round + if work_rx.lock().unwrap().recv().is_err() { + break; + } + + let input = Some(input_factory()); + + barrier.wait(); + + execute_all(input, handle, &manager).unwrap(); + + barrier.wait(); } - - let input = Some(input_factory()); - - barrier.wait(); - - execute_all(input, &manager).unwrap(); - - barrier.wait(); } }) }) @@ -149,13 +167,17 @@ impl Drop for JoinOnDrop { } } -fn execute_all(input: I, manager: &PostgresRedoManager) -> anyhow::Result<()> +fn execute_all( + input: I, + handle: &tokio::runtime::Handle, + manager: &PostgresRedoManager, +) -> anyhow::Result<()> where I: IntoIterator, { // just fire all requests as fast as possible input.into_iter().try_for_each(|req| { - let page = req.execute(manager)?; + let page = req.execute(handle, manager)?; assert_eq!(page.remaining(), 8192); anyhow::Ok(()) }) @@ -470,9 +492,11 @@ struct Request { } impl Request { - fn execute(self, manager: &PostgresRedoManager) -> anyhow::Result { - use pageserver::walredo::WalRedoManager; - + fn execute( + self, + rt: &tokio::runtime::Handle, + manager: &PostgresRedoManager, + ) -> anyhow::Result { let Request { key, lsn, @@ -481,6 +505,6 @@ impl Request { pg_version, } = self; - manager.request_redo(key, lsn, base_img, records, pg_version) + rt.block_on(manager.request_redo(key, lsn, base_img, records, pg_version)) } } diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 0bf851a8d7..22efa23f10 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -1298,10 +1298,6 @@ pub(crate) mod mock { } } - pub fn get_executed(&self) -> usize { - self.executed.load(Ordering::Relaxed) - } - #[allow(clippy::await_holding_lock)] pub async fn pump(&self) { if let Some(remote_storage) = &self.remote_storage { diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 0d987cfbfb..4dae183aea 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -83,7 +83,6 @@ use crate::tenant::timeline::delete::DeleteTimelineFlow; use crate::tenant::timeline::uninit::cleanup_timeline_directory; use crate::virtual_file::VirtualFile; use crate::walredo::PostgresRedoManager; -use crate::walredo::WalRedoManager; use crate::TEMP_FILE_SUFFIX; pub use pageserver_api::models::TenantState; @@ -230,7 +229,7 @@ pub struct Tenant { // with timelines, which in turn may cause dropping replication connection, expiration of wait_for_lsn // timeout... gc_cs: tokio::sync::Mutex<()>, - walredo_mgr: Arc, + walredo_mgr: Arc, // provides access to timeline data sitting in the remote storage pub(crate) remote_storage: Option, @@ -247,6 +246,48 @@ pub struct Tenant { pub(crate) delete_progress: Arc>, } +pub(crate) enum WalRedoManager { + Prod(PostgresRedoManager), + #[cfg(test)] + Test(harness::TestRedoManager), +} + +impl From for WalRedoManager { + fn from(mgr: PostgresRedoManager) -> Self { + Self::Prod(mgr) + } +} + +#[cfg(test)] +impl From for WalRedoManager { + fn from(mgr: harness::TestRedoManager) -> Self { + Self::Test(mgr) + } +} + +impl WalRedoManager { + pub async fn request_redo( + &self, + key: crate::repository::Key, + lsn: Lsn, + base_img: Option<(Lsn, bytes::Bytes)>, + records: Vec<(Lsn, crate::walrecord::NeonWalRecord)>, + pg_version: u32, + ) -> anyhow::Result { + match self { + Self::Prod(mgr) => { + mgr.request_redo(key, lsn, base_img, records, pg_version) + .await + } + #[cfg(test)] + Self::Test(mgr) => { + mgr.request_redo(key, lsn, base_img, records, pg_version) + .await + } + } + } +} + #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum GetTimelineError { #[error("Timeline {tenant_id}/{timeline_id} is not active, state: {state:?}")] @@ -488,7 +529,9 @@ impl Tenant { ctx: &RequestContext, ) -> anyhow::Result> { // TODO dedup with spawn_load - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( + conf, tenant_id, + ))); let TenantSharedResources { broker_client, @@ -815,7 +858,9 @@ impl Tenant { tenant_id: TenantId, reason: String, ) -> Arc { - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( + conf, tenant_id, + ))); Arc::new(Tenant::new( TenantState::Broken { reason, @@ -854,7 +899,9 @@ impl Tenant { let broker_client = resources.broker_client; let remote_storage = resources.remote_storage; - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( + conf, tenant_id, + ))); let tenant = Tenant::new( TenantState::Loading, conf, @@ -2419,7 +2466,7 @@ impl Tenant { state: TenantState, conf: &'static PageServerConf, attached_conf: AttachedTenantConf, - walredo_mgr: Arc, + walredo_mgr: Arc, tenant_id: TenantId, remote_storage: Option, deletion_queue_client: DeletionQueueClient, @@ -3550,7 +3597,7 @@ pub async fn dump_layerfile_from_path( } #[cfg(test)] -pub mod harness { +pub(crate) mod harness { use bytes::{Bytes, BytesMut}; use once_cell::sync::OnceCell; use std::fs; @@ -3561,7 +3608,6 @@ pub mod harness { use crate::deletion_queue::mock::MockDeletionQueue; use crate::{ config::PageServerConf, repository::Key, tenant::Tenant, walrecord::NeonWalRecord, - walredo::WalRedoManager, }; use super::*; @@ -3690,7 +3736,7 @@ pub mod harness { } pub async fn try_load(&self, ctx: &RequestContext) -> anyhow::Result> { - let walredo_mgr = Arc::new(TestRedoManager); + let walredo_mgr = Arc::new(WalRedoManager::from(TestRedoManager)); let tenant = Arc::new(Tenant::new( TenantState::Loading, @@ -3724,10 +3770,10 @@ pub mod harness { } // Mock WAL redo manager that doesn't do much - pub struct TestRedoManager; + pub(crate) struct TestRedoManager; - impl WalRedoManager for TestRedoManager { - fn request_redo( + impl TestRedoManager { + pub async fn request_redo( &self, key: Key, lsn: Lsn, diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 250047823e..2c76155e2a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -81,7 +81,6 @@ use crate::repository::GcResult; use crate::repository::{Key, Value}; use crate::task_mgr; use crate::task_mgr::TaskKind; -use crate::walredo::WalRedoManager; use crate::ZERO_PAGE; use self::delete::DeleteTimelineFlow; @@ -201,7 +200,7 @@ pub struct Timeline { last_freeze_ts: RwLock, // WAL redo manager - walredo_mgr: Arc, + walredo_mgr: Arc, /// Remote storage client. /// See [`remote_timeline_client`](super::remote_timeline_client) module comment for details. @@ -1471,7 +1470,7 @@ impl Timeline { timeline_id: TimelineId, tenant_id: TenantId, generation: Generation, - walredo_mgr: Arc, + walredo_mgr: Arc, resources: TimelineResources, pg_version: u32, initial_logical_size_can_start: Option, @@ -4324,6 +4323,7 @@ impl Timeline { let img = match self .walredo_mgr .request_redo(key, request_lsn, data.img, data.records, self.pg_version) + .await .context("Failed to reconstruct a page image:") { Ok(img) => img, diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 7056ef4f90..7e61a1dc37 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -70,27 +70,6 @@ pub(crate) struct BufferTag { pub blknum: u32, } -/// -/// WAL Redo Manager is responsible for replaying WAL records. -/// -/// Callers use the WAL redo manager through this abstract interface, -/// which makes it easy to mock it in tests. -pub trait WalRedoManager: Send + Sync { - /// Apply some WAL records. - /// - /// The caller passes an old page image, and WAL records that should be - /// applied over it. The return value is a new page image, after applying - /// the reords. - fn request_redo( - &self, - key: Key, - lsn: Lsn, - base_img: Option<(Lsn, Bytes)>, - records: Vec<(Lsn, NeonWalRecord)>, - pg_version: u32, - ) -> anyhow::Result; -} - struct ProcessInput { stdin: ChildStdin, stderr_fd: RawFd, @@ -135,14 +114,14 @@ fn can_apply_in_neon(rec: &NeonWalRecord) -> bool { /// /// Public interface of WAL redo manager /// -impl WalRedoManager for PostgresRedoManager { +impl PostgresRedoManager { /// /// Request the WAL redo manager to apply some WAL records /// /// The WAL redo is handled by a separate thread, so this just sends a request /// to the thread and waits for response. /// - fn request_redo( + pub async fn request_redo( &self, key: Key, lsn: Lsn, @@ -1156,15 +1135,15 @@ fn build_get_page_msg(tag: BufferTag, buf: &mut Vec) { #[cfg(test)] mod tests { - use super::{PostgresRedoManager, WalRedoManager}; + use super::PostgresRedoManager; use crate::repository::Key; use crate::{config::PageServerConf, walrecord::NeonWalRecord}; use bytes::Bytes; use std::str::FromStr; use utils::{id::TenantId, lsn::Lsn}; - #[test] - fn short_v14_redo() { + #[tokio::test] + async fn short_v14_redo() { let expected = std::fs::read("fixtures/short_v14_redo.page").unwrap(); let h = RedoHarness::new().unwrap(); @@ -1185,13 +1164,14 @@ mod tests { short_records(), 14, ) + .await .unwrap(); assert_eq!(&expected, &*page); } - #[test] - fn short_v14_fails_for_wrong_key_but_returns_zero_page() { + #[tokio::test] + async fn short_v14_fails_for_wrong_key_but_returns_zero_page() { let h = RedoHarness::new().unwrap(); let page = h @@ -1211,6 +1191,7 @@ mod tests { short_records(), 14, ) + .await .unwrap(); // TODO: there will be some stderr printout, which is forwarded to tracing that could From 1fa04789804aa74f9655c00e02b628d37e99fecb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:21:54 +0100 Subject: [PATCH 15/49] build(deps): bump urllib3 from 1.26.17 to 1.26.18 (#5582) --- poetry.lock | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index efc13f7c87..421e22b523 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2415,13 +2415,13 @@ files = [ [[package]] name = "urllib3" -version = "1.26.17" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.17-py2.py3-none-any.whl", hash = "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"}, - {file = "urllib3-1.26.17.tar.gz", hash = "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] @@ -2488,6 +2488,16 @@ files = [ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecee4132c6cd2ce5308e21672015ddfed1ff975ad0ac8d27168ea82e71413f55"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2020f391008ef874c6d9e208b24f28e31bcb85ccff4f335f15a3251d222b92d9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2feecf86e1f7a86517cab34ae6c2f081fd2d0dac860cb0c0ded96d799d20b335"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:240b1686f38ae665d1b15475966fe0472f78e71b1b4903c143a842659c8e4cb9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9008dad07d71f68487c91e96579c8567c98ca4c3881b9b113bc7b33e9fd78b8"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6447e9f3ba72f8e2b985a1da758767698efa72723d5b59accefd716e9e8272bf"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:acae32e13a4153809db37405f5eba5bac5fbe2e2ba61ab227926a22901051c0a"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49ef582b7a1152ae2766557f0550a9fcbf7bbd76f43fbdc94dd3bf07cc7168be"}, + {file = "wrapt-1.14.1-cp311-cp311-win32.whl", hash = "sha256:358fe87cc899c6bb0ddc185bf3dbfa4ba646f05b1b0b9b5a27c2cb92c2cea204"}, + {file = "wrapt-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:26046cd03936ae745a502abf44dac702a5e6880b2b01c29aea8ddf3353b68224"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, From 607d19f0e0babcae60551af9bfd2236f927fa93b Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 18 Oct 2023 13:28:38 +0100 Subject: [PATCH 16/49] pageserver: clean up page service Result handling for shutdown/disconnect (#5504) ## Problem - QueryError always logged at error severity, even though disconnections are not true errors. - QueryError type is not expressive enough to distinguish actual errors from shutdowns. - In some functions we're returning Ok(()) on shutdown, in others we're returning an error ## Summary of changes - Add QueryError::Shutdown and use it in places we check for cancellation - Adopt consistent Result behavior: disconnects and shutdowns are always QueryError, not ok - Transform shutdown+disconnect errors to Ok(()) at the very top of the task that runs query handler - Use the postgres protocol error code for "admin shutdown" in responses to clients when we are shutting down. Closes: #5517 --- libs/postgres_backend/src/lib.rs | 58 ++++++++++++++----- libs/pq_proto/src/lib.rs | 1 + pageserver/src/bin/pageserver.rs | 1 + pageserver/src/page_service.rs | 13 +++-- ...test_pageserver_restarts_under_workload.py | 2 - test_runner/regress/test_tenant_detach.py | 3 - 6 files changed, 52 insertions(+), 26 deletions(-) diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 08c4e03d13..8ce7f85481 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -19,8 +19,8 @@ use tracing::{debug, error, info, trace}; use pq_proto::framed::{ConnectionError, Framed, FramedReader, FramedWriter}; use pq_proto::{ - BeMessage, FeMessage, FeStartupPacket, ProtocolError, SQLSTATE_INTERNAL_ERROR, - SQLSTATE_SUCCESSFUL_COMPLETION, + BeMessage, FeMessage, FeStartupPacket, ProtocolError, SQLSTATE_ADMIN_SHUTDOWN, + SQLSTATE_INTERNAL_ERROR, SQLSTATE_SUCCESSFUL_COMPLETION, }; /// An error, occurred during query processing: @@ -30,6 +30,9 @@ pub enum QueryError { /// The connection was lost while processing the query. #[error(transparent)] Disconnected(#[from] ConnectionError), + /// We were instructed to shutdown while processing the query + #[error("Shutting down")] + Shutdown, /// Some other error #[error(transparent)] Other(#[from] anyhow::Error), @@ -44,7 +47,8 @@ impl From for QueryError { impl QueryError { pub fn pg_error_code(&self) -> &'static [u8; 5] { match self { - Self::Disconnected(_) => b"08006", // connection failure + Self::Disconnected(_) => b"08006", // connection failure + Self::Shutdown => SQLSTATE_ADMIN_SHUTDOWN, Self::Other(_) => SQLSTATE_INTERNAL_ERROR, // internal error } } @@ -396,7 +400,20 @@ impl PostgresBackend { // socket might be already closed, e.g. if previously received error, // so ignore result. self.framed.shutdown().await.ok(); - ret + match ret { + Ok(()) => Ok(()), + Err(QueryError::Shutdown) => { + info!("Stopped due to shutdown"); + Ok(()) + } + Err(QueryError::Disconnected(e)) => { + info!("Disconnected ({e:#})"); + // Disconnection is not an error: we just use it that way internally to drop + // out of loops. + Ok(()) + } + e => e, + } } async fn run_message_loop( @@ -416,15 +433,11 @@ impl PostgresBackend { _ = shutdown_watcher() => { // We were requested to shut down. tracing::info!("shutdown request received during handshake"); - return Ok(()) + return Err(QueryError::Shutdown) }, - result = self.handshake(handler) => { - // Handshake complete. - result?; - if self.state == ProtoState::Closed { - return Ok(()); // EOF during handshake - } + handshake_r = self.handshake(handler) => { + handshake_r?; } ); @@ -435,7 +448,7 @@ impl PostgresBackend { _ = shutdown_watcher() => { // We were requested to shut down. tracing::info!("shutdown request received in run_message_loop"); - Ok(None) + return Err(QueryError::Shutdown) }, msg = self.read_message() => { msg }, )? { @@ -447,7 +460,14 @@ impl PostgresBackend { _ = shutdown_watcher() => { // We were requested to shut down. tracing::info!("shutdown request received during response flush"); - return Ok(()) + + // If we exited process_message with a shutdown error, there may be + // some valid response content on in our transmit buffer: permit sending + // this within a short timeout. This is a best effort thing so we don't + // care about the result. + tokio::time::timeout(std::time::Duration::from_millis(500), self.flush()).await.ok(); + + return Err(QueryError::Shutdown) }, flush_r = self.flush() => { flush_r?; @@ -560,7 +580,9 @@ impl PostgresBackend { self.peer_addr ); self.state = ProtoState::Closed; - return Ok(()); + return Err(QueryError::Disconnected(ConnectionError::Protocol( + ProtocolError::Protocol("EOF during handshake".to_string()), + ))); } } } @@ -599,7 +621,9 @@ impl PostgresBackend { self.peer_addr ); self.state = ProtoState::Closed; - return Ok(()); + return Err(QueryError::Disconnected(ConnectionError::Protocol( + ProtocolError::Protocol("EOF during auth".to_string()), + ))); } } } @@ -923,6 +947,7 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> AsyncWrite for CopyDataWriter<'a, I pub fn short_error(e: &QueryError) -> String { match e { QueryError::Disconnected(connection_error) => connection_error.to_string(), + QueryError::Shutdown => "shutdown".to_string(), QueryError::Other(e) => format!("{e:#}"), } } @@ -939,6 +964,9 @@ fn log_query_error(query: &str, e: &QueryError) { QueryError::Disconnected(other_connection_error) => { error!("query handler for '{query}' failed with connection error: {other_connection_error:?}") } + QueryError::Shutdown => { + info!("query handler for '{query}' cancelled during tenant shutdown") + } QueryError::Other(e) => { error!("query handler for '{query}' failed: {e:?}"); } diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs index 47faad363f..94bc7f60f8 100644 --- a/libs/pq_proto/src/lib.rs +++ b/libs/pq_proto/src/lib.rs @@ -670,6 +670,7 @@ pub fn read_cstr(buf: &mut Bytes) -> Result { } pub const SQLSTATE_INTERNAL_ERROR: &[u8; 5] = b"XX000"; +pub const SQLSTATE_ADMIN_SHUTDOWN: &[u8; 5] = b"57P01"; pub const SQLSTATE_SUCCESSFUL_COMPLETION: &[u8; 5] = b"00000"; impl<'a> BeMessage<'a> { diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 827f74fcce..76cb0e8ec6 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -579,6 +579,7 @@ fn start_pageserver( pageserver_listener, conf.pg_auth_type, libpq_ctx, + task_mgr::shutdown_token(), ) .await }, diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 5ab4fbbd4c..536334d051 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -122,6 +122,7 @@ pub async fn libpq_listener_main( listener: TcpListener, auth_type: AuthType, listener_ctx: RequestContext, + cancel: CancellationToken, ) -> anyhow::Result<()> { listener.set_nonblocking(true)?; let tokio_listener = tokio::net::TcpListener::from_std(listener)?; @@ -130,7 +131,7 @@ pub async fn libpq_listener_main( while let Some(res) = tokio::select! { biased; - _ = task_mgr::shutdown_watcher() => { + _ = cancel.cancelled() => { // We were requested to shut down. None } @@ -299,7 +300,7 @@ impl PageServerHandler { Ok(flush_r?) }, _ = self.cancel.cancelled() => { - Err(QueryError::Other(anyhow::anyhow!("Shutting down"))) + Err(QueryError::Shutdown) } ) } @@ -316,11 +317,11 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = task_mgr::shutdown_watcher() => { + _ = self.cancel.cancelled() => { // We were requested to shut down. let msg = "pageserver is shutting down"; let _ = pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, None)); - Err(QueryError::Other(anyhow::anyhow!(msg))) + Err(QueryError::Shutdown) } msg = pgb.read_message() => { msg.map_err(QueryError::from)} @@ -414,10 +415,10 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = task_mgr::shutdown_watcher() => { + _ = self.cancel.cancelled() => { // We were requested to shut down. info!("shutdown request received in page handler"); - break; + return Err(QueryError::Shutdown) } msg = pgb.read_message() => { msg } diff --git a/test_runner/regress/test_pageserver_restarts_under_workload.py b/test_runner/regress/test_pageserver_restarts_under_workload.py index 71058268a6..65569f3bac 100644 --- a/test_runner/regress/test_pageserver_restarts_under_workload.py +++ b/test_runner/regress/test_pageserver_restarts_under_workload.py @@ -17,8 +17,6 @@ def test_pageserver_restarts_under_worload(neon_simple_env: NeonEnv, pg_bin: PgB n_restarts = 10 scale = 10 - env.pageserver.allowed_errors.append(".*query handler.*failed.*Shutting down") - def run_pgbench(connstr: str): log.info(f"Start a pgbench workload on pg {connstr}") pg_bin.run_capture(["pgbench", "-i", f"-s{scale}", connstr]) diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index ec6f1258e5..e92a906fab 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -744,9 +744,6 @@ def test_ignore_while_attaching( env.pageserver.allowed_errors.append( f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" ) - # An endpoint is starting up concurrently with our detach, it can - # experience RPC failure due to shutdown. - env.pageserver.allowed_errors.append(".*query handler.*failed.*Shutting down") data_id = 1 data_secret = "very secret secret" From 5c88213eaf1b1e29c610a078d0b380f69ed49a7e Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 18 Oct 2023 16:42:22 +0300 Subject: [PATCH 17/49] Logical replication (#5271) ## Problem See https://github.com/neondatabase/company_projects/issues/111 ## Summary of changes Save logical replication files in WAL at compute and include them in basebackup at pate server. ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik Co-authored-by: Arseny Sher --- compute_tools/src/compute.rs | 2 +- compute_tools/tests/pg_helpers_tests.rs | 2 +- control_plane/src/endpoint.rs | 2 +- .../var/db/postgres/specs/spec.json | 2 +- libs/compute_api/tests/cluster_spec.json | 2 +- libs/postgres_ffi/src/pg_constants.rs | 4 + pageserver/src/basebackup.rs | 30 ++++ pageserver/src/pgdatadir_mapping.rs | 83 +++++++++- pageserver/src/walingest.rs | 23 +-- pageserver/src/walrecord.rs | 20 +++ pgxn/neon/pagestore_smgr.c | 7 - pgxn/neon/walproposer.c | 22 +++ test_runner/fixtures/neon_fixtures.py | 16 ++ .../performance/test_logical_replication.py | 43 +++++ .../regress/test_logical_replication.py | 149 ++++++++++++++++++ vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/revisions.json | 6 +- 19 files changed, 391 insertions(+), 28 deletions(-) create mode 100644 test_runner/performance/test_logical_replication.py create mode 100644 test_runner/regress/test_logical_replication.py diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index e67430d061..9fd88e5818 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -252,7 +252,7 @@ fn create_neon_superuser(spec: &ComputeSpec, client: &mut Client) -> Result<()> IF NOT EXISTS ( SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser') THEN - CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN IN ROLE pg_read_all_data, pg_write_all_data; + CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN REPLICATION IN ROLE pg_read_all_data, pg_write_all_data; IF array_length(roles, 1) IS NOT NULL THEN EXECUTE format('GRANT neon_superuser TO %s', array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(roles) as x), ', ')); diff --git a/compute_tools/tests/pg_helpers_tests.rs b/compute_tools/tests/pg_helpers_tests.rs index 7d27d22a78..4961bc293d 100644 --- a/compute_tools/tests/pg_helpers_tests.rs +++ b/compute_tools/tests/pg_helpers_tests.rs @@ -28,7 +28,7 @@ mod pg_helpers_tests { assert_eq!( spec.cluster.settings.as_pg_settings(), r#"fsync = off -wal_level = replica +wal_level = logical hot_standby = on neon.safekeepers = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501' wal_log_hints = on diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index cba364c049..acd9061664 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -253,7 +253,7 @@ impl Endpoint { conf.append("shared_buffers", "1MB"); conf.append("fsync", "off"); conf.append("max_connections", "100"); - conf.append("wal_level", "replica"); + conf.append("wal_level", "logical"); // wal_sender_timeout is the maximum time to wait for WAL replication. // It also defines how often the walreciever will send a feedback message to the wal sender. conf.append("wal_sender_timeout", "5s"); diff --git a/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json b/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json index 565e5e368e..ccf0a91b90 100644 --- a/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json +++ b/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json @@ -25,7 +25,7 @@ }, { "name": "wal_level", - "value": "replica", + "value": "logical", "vartype": "enum" }, { diff --git a/libs/compute_api/tests/cluster_spec.json b/libs/compute_api/tests/cluster_spec.json index 96db13a5da..e2afa17ef0 100644 --- a/libs/compute_api/tests/cluster_spec.json +++ b/libs/compute_api/tests/cluster_spec.json @@ -76,7 +76,7 @@ }, { "name": "wal_level", - "value": "replica", + "value": "logical", "vartype": "enum" }, { diff --git a/libs/postgres_ffi/src/pg_constants.rs b/libs/postgres_ffi/src/pg_constants.rs index 9690dc0eb6..d59e0e4a15 100644 --- a/libs/postgres_ffi/src/pg_constants.rs +++ b/libs/postgres_ffi/src/pg_constants.rs @@ -220,6 +220,10 @@ pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10; pub const XLP_FIRST_IS_CONTRECORD: u16 = 0x0001; pub const XLP_LONG_HEADER: u16 = 0x0002; +/* From replication/slot.h */ +pub const REPL_SLOT_ON_DISK_OFFSETOF_RESTART_LSN: usize = 4*4 /* offset of `slotdata` in ReplicationSlotOnDisk */ + + 64 /* NameData */ + 4*4; + /* From fsm_internals.h */ const FSM_NODES_PER_PAGE: usize = BLCKSZ as usize - SIZEOF_PAGE_HEADER_DATA - 4; const FSM_NON_LEAF_NODES_PER_PAGE: usize = BLCKSZ as usize / 2 - 1; diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index a959f1cddc..ed452eae7d 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -13,6 +13,7 @@ use anyhow::{anyhow, bail, ensure, Context}; use bytes::{BufMut, BytesMut}; use fail::fail_point; +use postgres_ffi::pg_constants; use std::fmt::Write as FmtWrite; use std::time::SystemTime; use tokio::io; @@ -180,6 +181,7 @@ where } } + let mut min_restart_lsn: Lsn = Lsn::MAX; // Create tablespace directories for ((spcnode, dbnode), has_relmap_file) in self.timeline.list_dbdirs(self.lsn, self.ctx).await? @@ -213,6 +215,34 @@ where self.add_rel(rel, rel).await?; } } + + for (path, content) in self.timeline.list_aux_files(self.lsn, self.ctx).await? { + if path.starts_with("pg_replslot") { + let offs = pg_constants::REPL_SLOT_ON_DISK_OFFSETOF_RESTART_LSN; + let restart_lsn = Lsn(u64::from_le_bytes( + content[offs..offs + 8].try_into().unwrap(), + )); + info!("Replication slot {} restart LSN={}", path, restart_lsn); + min_restart_lsn = Lsn::min(min_restart_lsn, restart_lsn); + } + let header = new_tar_header(&path, content.len() as u64)?; + self.ar + .append(&header, &*content) + .await + .context("could not add aux file to basebackup tarball")?; + } + } + if min_restart_lsn != Lsn::MAX { + info!( + "Min restart LSN for logical replication is {}", + min_restart_lsn + ); + let data = min_restart_lsn.0.to_le_bytes(); + let header = new_tar_header("restart.lsn", data.len() as u64)?; + self.ar + .append(&header, &data[..]) + .await + .context("could not add restart.lsn file to basebackup tarball")?; } for xid in self .timeline diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 9a1281a522..ebba3d8579 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -499,6 +499,23 @@ impl Timeline { self.get(CHECKPOINT_KEY, lsn, ctx).await } + pub async fn list_aux_files( + &self, + lsn: Lsn, + ctx: &RequestContext, + ) -> Result, PageReconstructError> { + match self.get(AUX_FILES_KEY, lsn, ctx).await { + Ok(buf) => match AuxFilesDirectory::des(&buf).context("deserialization failure") { + Ok(dir) => Ok(dir.files), + Err(e) => Err(PageReconstructError::from(e)), + }, + Err(e) => { + warn!("Failed to get info about AUX files: {}", e); + Ok(HashMap::new()) + } + } + } + /// Does the same as get_current_logical_size but counted on demand. /// Used to initialize the logical size tracking on startup. /// @@ -616,6 +633,7 @@ impl Timeline { result.add_key(CONTROLFILE_KEY); result.add_key(CHECKPOINT_KEY); + result.add_key(AUX_FILES_KEY); Ok(result.to_keyspace()) } @@ -692,6 +710,12 @@ impl<'a> DatadirModification<'a> { })?; self.put(DBDIR_KEY, Value::Image(buf.into())); + // Create AuxFilesDirectory + let buf = AuxFilesDirectory::ser(&AuxFilesDirectory { + files: HashMap::new(), + })?; + self.put(AUX_FILES_KEY, Value::Image(Bytes::from(buf))); + let buf = TwoPhaseDirectory::ser(&TwoPhaseDirectory { xids: HashSet::new(), })?; @@ -796,6 +820,12 @@ impl<'a> DatadirModification<'a> { // 'true', now write the updated 'dbdirs' map back. let buf = DbDirectory::ser(&dbdir)?; self.put(DBDIR_KEY, Value::Image(buf.into())); + + // Create AuxFilesDirectory as well + let buf = AuxFilesDirectory::ser(&AuxFilesDirectory { + files: HashMap::new(), + })?; + self.put(AUX_FILES_KEY, Value::Image(Bytes::from(buf))); } if r.is_none() { // Create RelDirectory @@ -1120,6 +1150,36 @@ impl<'a> DatadirModification<'a> { Ok(()) } + pub async fn put_file( + &mut self, + path: &str, + content: &[u8], + ctx: &RequestContext, + ) -> anyhow::Result<()> { + let mut dir = match self.get(AUX_FILES_KEY, ctx).await { + Ok(buf) => AuxFilesDirectory::des(&buf)?, + Err(e) => { + warn!("Failed to get info about AUX files: {}", e); + AuxFilesDirectory { + files: HashMap::new(), + } + } + }; + let path = path.to_string(); + if content.is_empty() { + dir.files.remove(&path); + } else { + dir.files.insert(path, Bytes::copy_from_slice(content)); + } + self.put( + AUX_FILES_KEY, + Value::Image(Bytes::from( + AuxFilesDirectory::ser(&dir).context("serialize")?, + )), + ); + Ok(()) + } + /// /// Flush changes accumulated so far to the underlying repository. /// @@ -1255,6 +1315,11 @@ struct RelDirectory { rels: HashSet<(Oid, u8)>, } +#[derive(Debug, Serialize, Deserialize, Default)] +struct AuxFilesDirectory { + files: HashMap, +} + #[derive(Debug, Serialize, Deserialize)] struct RelSizeEntry { nblocks: u32, @@ -1303,10 +1368,12 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; BLCKSZ as usize]); // 02 pg_twophase // // 03 misc -// controlfile +// Controlfile // checkpoint // pg_version // +// 04 aux files +// // Below is a full list of the keyspace allocation: // // DbDir: @@ -1344,6 +1411,11 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; BLCKSZ as usize]); // // Checkpoint: // 03 00000000 00000000 00000000 00 00000001 +// +// AuxFiles: +// 03 00000000 00000000 00000000 00 00000002 +// + //-- Section 01: relation data and metadata const DBDIR_KEY: Key = Key { @@ -1567,6 +1639,15 @@ const CHECKPOINT_KEY: Key = Key { field6: 1, }; +const AUX_FILES_KEY: Key = Key { + field1: 0x03, + field2: 0, + field3: 0, + field4: 0, + field5: 0, + field6: 2, +}; + // Reverse mappings for a few Keys. // These are needed by WAL redo manager. diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index d290715938..fb1dbcd6ba 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -338,11 +338,20 @@ impl<'a> WalIngest<'a> { } else if decoded.xl_rmid == pg_constants::RM_LOGICALMSG_ID { let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; if info == pg_constants::XLOG_LOGICAL_MESSAGE { - // This is a convenient way to make the WAL ingestion pause at - // particular point in the WAL. For more fine-grained control, - // we could peek into the message and only pause if it contains - // a particular string, for example, but this is enough for now. - crate::failpoint_support::sleep_millis_async!("wal-ingest-logical-message-sleep"); + let xlrec = XlLogicalMessage::decode(&mut buf); + let prefix = std::str::from_utf8(&buf[0..xlrec.prefix_size - 1])?; + let message = &buf[xlrec.prefix_size..xlrec.prefix_size + xlrec.message_size]; + if prefix == "neon-test" { + // This is a convenient way to make the WAL ingestion pause at + // particular point in the WAL. For more fine-grained control, + // we could peek into the message and only pause if it contains + // a particular string, for example, but this is enough for now. + crate::failpoint_support::sleep_millis_async!( + "wal-ingest-logical-message-sleep" + ); + } else if let Some(path) = prefix.strip_prefix("neon-file:") { + modification.put_file(path, message, ctx).await?; + } } } @@ -459,7 +468,6 @@ impl<'a> WalIngest<'a> { } } else if info == pg_constants::XLOG_HEAP_DELETE { let xlrec = v14::XlHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } @@ -527,7 +535,6 @@ impl<'a> WalIngest<'a> { } } else if info == pg_constants::XLOG_HEAP_DELETE { let xlrec = v15::XlHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } @@ -595,7 +602,6 @@ impl<'a> WalIngest<'a> { } } else if info == pg_constants::XLOG_HEAP_DELETE { let xlrec = v16::XlHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } @@ -771,7 +777,6 @@ impl<'a> WalIngest<'a> { } pg_constants::XLOG_NEON_HEAP_DELETE => { let xlrec = v16::rm_neon::XlNeonHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } diff --git a/pageserver/src/walrecord.rs b/pageserver/src/walrecord.rs index 9c2e522f17..ff6bc9194b 100644 --- a/pageserver/src/walrecord.rs +++ b/pageserver/src/walrecord.rs @@ -748,6 +748,26 @@ impl XlMultiXactTruncate { } } +#[repr(C)] +#[derive(Debug)] +pub struct XlLogicalMessage { + pub db_id: Oid, + pub transactional: bool, + pub prefix_size: usize, + pub message_size: usize, +} + +impl XlLogicalMessage { + pub fn decode(buf: &mut Bytes) -> XlLogicalMessage { + XlLogicalMessage { + db_id: buf.get_u32_le(), + transactional: buf.get_u32_le() != 0, // 4-bytes alignment + prefix_size: buf.get_u64_le() as usize, + message_size: buf.get_u64_le() as usize, + } + } +} + /// Main routine to decode a WAL record and figure out which blocks are modified // // See xlogrecord.h for details diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index 5e172a0be4..3a841b04ec 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -63,7 +63,6 @@ #include "storage/md.h" #include "pgstat.h" - #if PG_VERSION_NUM >= 150000 #include "access/xlogutils.h" #include "access/xlogrecovery.h" @@ -1395,12 +1394,6 @@ neon_get_request_lsn(bool *latest, NRelFileInfo rinfo, ForkNumber forknum, Block elog(DEBUG1, "neon_get_request_lsn GetXLogReplayRecPtr %X/%X request lsn 0 ", (uint32) ((lsn) >> 32), (uint32) (lsn)); } - else if (am_walsender) - { - *latest = true; - lsn = InvalidXLogRecPtr; - elog(DEBUG1, "am walsender neon_get_request_lsn lsn 0 "); - } else { XLogRecPtr flushlsn; diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index c1fd5e3ef3..10612e7e35 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -861,8 +861,30 @@ RecvVoteResponse(Safekeeper *sk) static void HandleElectedProposer(WalProposer *wp) { + FILE* f; + XLogRecPtr lrRestartLsn; + DetermineEpochStartLsn(wp); + /* + * If there are active logical replication subscription we need + * to provide enough WAL for their WAL senders based on th position + * of their replication slots. + */ + f = fopen("restart.lsn", "rb"); + if (f != NULL && !wp->config->syncSafekeepers) + { + fread(&lrRestartLsn, sizeof(lrRestartLsn), 1, f); + fclose(f); + if (lrRestartLsn != InvalidXLogRecPtr) + { + elog(LOG, "Logical replication restart LSN %X/%X", LSN_FORMAT_ARGS(lrRestartLsn)); + /* start from the beginning of the segment to fetch page headers verifed by XLogReader */ + lrRestartLsn = lrRestartLsn - XLogSegmentOffset(lrRestartLsn, wal_segment_size); + wp->truncateLsn = Min(wp->truncateLsn, lrRestartLsn); + } + } + /* * Check if not all safekeepers are up-to-date, we need to download WAL * needed to synchronize them diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index d9a75637b8..68e29523b0 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3119,6 +3119,22 @@ def check_restored_datadir_content( assert (mismatch, error) == ([], []) +def logical_replication_sync(subscriber: VanillaPostgres, publisher: Endpoint) -> Lsn: + """Wait logical replication subscriber to sync with publisher.""" + publisher_lsn = Lsn(publisher.safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0]) + while True: + res = subscriber.safe_psql("select latest_end_lsn from pg_catalog.pg_stat_subscription")[0][ + 0 + ] + if res: + log.info(f"subscriber_lsn={res}") + subscriber_lsn = Lsn(res) + log.info(f"Subscriber LSN={subscriber_lsn}, publisher LSN={ publisher_lsn}") + if subscriber_lsn >= publisher_lsn: + return subscriber_lsn + time.sleep(0.5) + + def wait_for_last_flush_lsn( env: NeonEnv, endpoint: Endpoint, diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py new file mode 100644 index 0000000000..b799f7248f --- /dev/null +++ b/test_runner/performance/test_logical_replication.py @@ -0,0 +1,43 @@ +import time + +import pytest +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnv, PgBin, logical_replication_sync + + +@pytest.mark.timeout(1000) +def test_logical_replication(neon_simple_env: NeonEnv, pg_bin: PgBin, vanilla_pg): + env = neon_simple_env + + env.neon_cli.create_branch("test_logical_replication", "empty") + endpoint = env.endpoints.create_start("test_logical_replication") + + log.info("postgres is running on 'test_logical_replication' branch") + pg_bin.run_capture(["pgbench", "-i", "-s10", endpoint.connstr()]) + + endpoint.safe_psql("create publication pub1 for table pgbench_accounts, pgbench_history") + + # now start subscriber + vanilla_pg.start() + pg_bin.run_capture(["pgbench", "-i", "-s10", vanilla_pg.connstr()]) + + vanilla_pg.safe_psql("truncate table pgbench_accounts") + vanilla_pg.safe_psql("truncate table pgbench_history") + + connstr = endpoint.connstr().replace("'", "''") + print(f"connstr='{connstr}'") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + + # Wait logical replication channel to be established + logical_replication_sync(vanilla_pg, endpoint) + + pg_bin.run_capture(["pgbench", "-c10", "-T100", "-Mprepared", endpoint.connstr()]) + + # Wait logical replication to sync + start = time.time() + logical_replication_sync(vanilla_pg, endpoint) + log.info(f"Sync with master took {time.time() - start} seconds") + + sum_master = endpoint.safe_psql("select sum(abalance) from pgbench_accounts")[0][0] + sum_replica = vanilla_pg.safe_psql("select sum(abalance) from pgbench_accounts")[0][0] + assert sum_master == sum_replica diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py new file mode 100644 index 0000000000..726e5e5def --- /dev/null +++ b/test_runner/regress/test_logical_replication.py @@ -0,0 +1,149 @@ +import time + +from fixtures.log_helper import log +from fixtures.neon_fixtures import ( + NeonEnv, + logical_replication_sync, + wait_for_last_flush_lsn, +) + + +def test_logical_replication(neon_simple_env: NeonEnv, vanilla_pg): + env = neon_simple_env + + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_logical_replication", "empty") + endpoint = env.endpoints.create_start( + "test_logical_replication", config_lines=["log_statement=all"] + ) + + log.info("postgres is running on 'test_logical_replication' branch") + pg_conn = endpoint.connect() + cur = pg_conn.cursor() + + cur.execute("create table t(pk integer primary key, payload integer)") + cur.execute( + "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120));" + ) + cur.execute("create publication pub1 for table t, replication_example") + + # now start subscriber + vanilla_pg.start() + vanilla_pg.safe_psql("create table t(pk integer primary key, payload integer)") + vanilla_pg.safe_psql( + "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120), testcolumn1 int, testcolumn2 int, testcolumn3 int);" + ) + connstr = endpoint.connstr().replace("'", "''") + log.info(f"ep connstr is {endpoint.connstr()}, subscriber connstr {vanilla_pg.connstr()}") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + + # Wait logical replication channel to be established + logical_replication_sync(vanilla_pg, endpoint) + + # insert some data + cur.execute("insert into t values (generate_series(1,1000), 0)") + + # Wait logical replication to sync + logical_replication_sync(vanilla_pg, endpoint) + assert vanilla_pg.safe_psql("select count(*) from t")[0][0] == 1000 + + # now stop subscriber... + vanilla_pg.stop() + + # ... and insert some more data which should be delivered to subscriber after restart + cur.execute("insert into t values (generate_series(1001,2000), 0)") + + # Restart compute + endpoint.stop() + endpoint.start() + + # start subscriber + vanilla_pg.start() + + # Wait logical replication to sync + logical_replication_sync(vanilla_pg, endpoint) + + # Check that subscribers receives all data + assert vanilla_pg.safe_psql("select count(*) from t")[0][0] == 2000 + + # Test that save/restore of RewriteMappingFile works. Partial copy of + # rewrite.sql test. + log.info("checking rewriteheap") + vanilla_pg.stop() + cmds = """ +INSERT INTO replication_example(somedata) VALUES (1); + +BEGIN; +INSERT INTO replication_example(somedata) VALUES (2); +ALTER TABLE replication_example ADD COLUMN testcolumn1 int; +INSERT INTO replication_example(somedata, testcolumn1) VALUES (3, 1); +COMMIT; + +BEGIN; +INSERT INTO replication_example(somedata) VALUES (3); +ALTER TABLE replication_example ADD COLUMN testcolumn2 int; +INSERT INTO replication_example(somedata, testcolumn1, testcolumn2) VALUES (4, 2, 1); +COMMIT; + +VACUUM FULL pg_am; +VACUUM FULL pg_amop; +VACUUM FULL pg_proc; +VACUUM FULL pg_opclass; +VACUUM FULL pg_type; +VACUUM FULL pg_index; +VACUUM FULL pg_database; + + +-- repeated rewrites that fail +BEGIN; +CLUSTER pg_class USING pg_class_oid_index; +CLUSTER pg_class USING pg_class_oid_index; +ROLLBACK; + +-- repeated rewrites that succeed +BEGIN; +CLUSTER pg_class USING pg_class_oid_index; +CLUSTER pg_class USING pg_class_oid_index; +CLUSTER pg_class USING pg_class_oid_index; +COMMIT; + +-- repeated rewrites in different transactions +VACUUM FULL pg_class; +VACUUM FULL pg_class; + +-- reindexing of important relations / indexes +REINDEX TABLE pg_class; +REINDEX INDEX pg_class_oid_index; +REINDEX INDEX pg_class_tblspc_relfilenode_index; + +INSERT INTO replication_example(somedata, testcolumn1) VALUES (5, 3); + +BEGIN; +INSERT INTO replication_example(somedata, testcolumn1) VALUES (6, 4); +ALTER TABLE replication_example ADD COLUMN testcolumn3 int; +INSERT INTO replication_example(somedata, testcolumn1, testcolumn3) VALUES (7, 5, 1); +COMMIT; +""" + endpoint.safe_psql_many([q for q in cmds.splitlines() if q != "" and not q.startswith("-")]) + + # refetch rewrite files from pageserver + endpoint.stop() + endpoint.start() + + vanilla_pg.start() + logical_replication_sync(vanilla_pg, endpoint) + eq_q = "select testcolumn1, testcolumn2, testcolumn3 from replication_example order by 1, 2, 3" + assert vanilla_pg.safe_psql(eq_q) == endpoint.safe_psql(eq_q) + log.info("rewriteheap synced") + + # test that removal of repl slots works across restart + vanilla_pg.stop() + time.sleep(1) # wait for conn termination; active slots can't be dropped + endpoint.safe_psql("select pg_drop_replication_slot('sub1');") + endpoint.safe_psql("insert into t values (2001, 1);") # forces WAL flush + # wait for drop message to reach safekeepers (it is not transactional) + wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop() + endpoint.start() + # it must be gone (but walproposer slot still exists, hence 1) + assert endpoint.safe_psql("select count(*) from pg_replication_slots")[0][0] == 1 diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 5d5cfee127..ebcca9e9eb 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 5d5cfee12783f0989a9c9fe13bb40b5585812568 +Subproject commit ebcca9e9eb49621b5b17247833b59e836337e8aa diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 74cfe3e681..23f2d41102 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 74cfe3e681836747a31fdbd47bdd14b3d81b0772 +Subproject commit 23f2d411020a739375b32895ce1362ded2962084 diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 389ce36b4b..e5e255d2da 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 389ce36b4b3da7aa654a25e1b3f10b641319a87f +Subproject commit e5e255d2da05bc5f884b871c042014030a114a9b diff --git a/vendor/revisions.json b/vendor/revisions.json index d08cb25f43..14ccbef287 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "postgres-v16": "389ce36b4b3da7aa654a25e1b3f10b641319a87f", - "postgres-v15": "74cfe3e681836747a31fdbd47bdd14b3d81b0772", - "postgres-v14": "5d5cfee12783f0989a9c9fe13bb40b5585812568" + "postgres-v16": "e5e255d2da05bc5f884b871c042014030a114a9b", + "postgres-v15": "23f2d411020a739375b32895ce1362ded2962084", + "postgres-v14": "ebcca9e9eb49621b5b17247833b59e836337e8aa" } From 1f4805baf8cf93ed10e79e327f1d13e94aa064ec Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 18 Oct 2023 09:55:44 +0300 Subject: [PATCH 18/49] Remove remnants of num_computes field. Fixes https://github.com/neondatabase/neon/issues/5581 --- safekeeper/src/timeline.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 37821eae23..6e00e27b91 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -112,7 +112,6 @@ pub struct SharedState { /// TODO: it might be better to remove tli completely from GlobalTimelines /// when tli is inactive instead of having this flag. active: bool, - num_computes: u32, last_removed_segno: XLogSegNo, } @@ -151,7 +150,6 @@ impl SharedState { peers_info: PeersInfo(vec![]), wal_backup_active: false, active: false, - num_computes: 0, last_removed_segno: 0, }) } @@ -171,7 +169,6 @@ impl SharedState { peers_info: PeersInfo(vec![]), wal_backup_active: false, active: false, - num_computes: 0, last_removed_segno: 0, }) } @@ -219,7 +216,7 @@ impl SharedState { }; trace!( "timeline {} s3 offloading action {} pending: num_computes={}, commit_lsn={}, backup_lsn={}", - self.sk.state.timeline_id, action_pending, self.num_computes, self.sk.inmem.commit_lsn, self.sk.inmem.backup_lsn + self.sk.state.timeline_id, action_pending, num_computes, self.sk.inmem.commit_lsn, self.sk.inmem.backup_lsn ); } res @@ -531,7 +528,7 @@ impl Timeline { return true; } let shared_state = self.write_shared_state().await; - if shared_state.num_computes == 0 { + if self.walreceivers.get_num() == 0 { return shared_state.sk.inmem.commit_lsn == Lsn(0) || // no data at all yet reported_remote_consistent_lsn >= shared_state.sk.inmem.commit_lsn; } @@ -765,7 +762,7 @@ impl Timeline { ps_feedback, wal_backup_active: state.wal_backup_active, timeline_is_active: state.active, - num_computes: state.num_computes, + num_computes: self.walreceivers.get_num() as u32, last_removed_segno: state.last_removed_segno, epoch_start_lsn: state.sk.epoch_start_lsn, mem_state: state.sk.inmem.clone(), @@ -792,7 +789,7 @@ impl Timeline { walsenders: self.walsenders.get_all(), wal_backup_active: state.wal_backup_active, active: state.active, - num_computes: state.num_computes, + num_computes: self.walreceivers.get_num() as u32, last_removed_segno: state.last_removed_segno, epoch_start_lsn: state.sk.epoch_start_lsn, mem_state: state.sk.inmem.clone(), From 9a9d9eba423659356919171c3fb2f2a7b6158f0a Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Wed, 18 Oct 2023 12:40:07 +0000 Subject: [PATCH 19/49] Add test_idle_reconnections --- test_runner/regress/test_wal_acceptor.py | 72 +++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 631798d643..aaac4570b0 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -11,7 +11,7 @@ from contextlib import closing from dataclasses import dataclass, field from functools import partial from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional import psycopg2 import psycopg2.errors @@ -19,6 +19,7 @@ import psycopg2.extras import pytest from fixtures.broker import NeonBroker from fixtures.log_helper import log +from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( Endpoint, NeonEnv, @@ -1477,3 +1478,72 @@ def test_pull_timeline(neon_env_builder: NeonEnvBuilder): execute_payload(endpoint) show_statuses(env.safekeepers, tenant_id, timeline_id) + + +# In this test we check for excessive START_REPLICATION and START_WAL_PUSH queries +# when compute is active, but there are no writes to the timeline. In that case +# pageserver should maintain a single connection to safekeeper and don't attempt +# to reconnect extra times. +# +# The only way to verify this without manipulating time is to sleep for a while. +# In this test we sleep for 60 seconds, so this test takes at least 1 minute to run. +# This is longer than most other tests, we run it only for v16 to save CI resources. +def test_idle_reconnections(neon_env_builder: NeonEnvBuilder): + if os.environ.get("PYTEST_CURRENT_TEST", "").find("[debug-pg16]") == -1: + pytest.skip("run only on debug postgres v16 to save CI resources") + + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_sk_auth_restart_endpoint") + + def collect_stats() -> Dict[str, float]: + # we need to collect safekeeper_pg_queries_received_total metric from all safekeepers + sk_metrics = [ + parse_metrics(sk.http_client().get_metrics_str(), f"safekeeper_{sk.id}") + for sk in env.safekeepers + ] + + total: Dict[str, float] = {} + + for sk in sk_metrics: + queries_received = sk.query_all("safekeeper_pg_queries_received_total") + log.info(f"{sk.name} queries received: {queries_received}") + for sample in queries_received: + total[sample.labels["query"]] = total.get(sample.labels["query"], 0) + sample.value + + log.info(f"Total queries received: {total}") + + # in the perfect world, we should see only one START_REPLICATION query, + # here we check for 5 to prevent flakiness + assert total.get("START_REPLICATION", 0) <= 5 + + # in the perfect world, we should see ~6 START_WAL_PUSH queries, + # here we check for 15 to prevent flakiness + assert total.get("START_WAL_PUSH", 0) <= 15 + + return total + + collect_stats() + + endpoint = env.endpoints.create_start("test_sk_auth_restart_endpoint") + # just write something to the timeline + endpoint.safe_psql("create table t(i int)") + collect_stats() + + # sleep a bit + time.sleep(30) + + # force checkpoint in pageserver to advance remote_consistent_lsn + wait_lsn_force_checkpoint(tenant_id, timeline_id, endpoint, env.pageserver) + + collect_stats() + + time.sleep(30) + + final_stats = collect_stats() + # pageserver should connect to safekeepers at least once + assert final_stats.get("START_REPLICATION", 0) >= 1 + # walproposer should connect to each safekeeper at least once + assert final_stats.get("START_WAL_PUSH", 0) >= 3 From ecf759be6d40cc755892f569ea9d146d0a6db25b Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 18 Oct 2023 16:16:58 +0100 Subject: [PATCH 20/49] tests: allow-list S3 500 on DeleteObjects key (#5586) ## Problem S3 can give us a 500 whenever it likes: when this happens at request level we eat it in `backoff::retry`, but when it happens for a key inside a DeleteObjects request, we log it at warn level. ## Summary of changes Allow-list this class of log message in all tests. --- test_runner/fixtures/neon_fixtures.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 68e29523b0..a9fe5e376e 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1632,6 +1632,9 @@ class NeonPageserver(PgProtocol): # these can happen during shutdown, but it should not be a reason to fail a test ".*completed, took longer than expected.*", '.*registered custom resource manager "neon".*', + # AWS S3 may emit 500 errors for keys in a DeleteObjects response: we retry these + # and it is not a failure of our code when it happens. + ".*DeleteObjects.*We encountered an internal error. Please try again.*", ] def timeline_dir(self, tenant_id: TenantId, timeline_id: Optional[TimelineId] = None) -> Path: From c8637f37369098875162f194f92736355783b050 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Wed, 18 Oct 2023 11:24:04 -0500 Subject: [PATCH 21/49] Remove specific file references in NOTICE Seems like a burden to update this file with each major release. --- NOTICE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NOTICE b/NOTICE index 4fbec9763b..c13dc2f0b3 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Neon Copyright 2022 Neon Inc. -The PostgreSQL submodules in vendor/postgres-v14 and vendor/postgres-v15 are licensed under the -PostgreSQL license. See vendor/postgres-v14/COPYRIGHT and vendor/postgres-v15/COPYRIGHT. +The PostgreSQL submodules in vendor/ are licensed under the PostgreSQL license. +See vendor/postgres-vX/COPYRIGHT for details. From d444d4dcea6dafbc1eb82e75763c180aeeddbdaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 03:43:49 +0100 Subject: [PATCH 22/49] build(deps): bump rustix from 0.36.14 to 0.36.16 (#5591) Bumps [rustix](https://github.com/bytecodealliance/rustix) from 0.36.14 to 0.36.16. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c641bd36c..cb0fe3fe20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3734,7 +3734,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.14", + "rustix 0.36.16", ] [[package]] @@ -4317,9 +4317,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.14" +version = "0.36.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" +checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" dependencies = [ "bitflags", "errno", From f842b22b907d8682b6f900a8d7724a6aa4b52e77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 19 Oct 2023 04:50:49 +0200 Subject: [PATCH 23/49] Add endpoint for querying time info for lsn (#5497) ## Problem See #5468. ## Summary of changes Add a new `get_timestamp_of_lsn` endpoint, returning the timestamp associated with the given lsn. Fixes #5468. --------- Co-authored-by: Shany Pozin --- libs/postgres_ffi/src/lib.rs | 1 + libs/postgres_ffi/src/xlog_utils.rs | 61 +++++++++++--- pageserver/src/http/openapi_spec.yml | 61 ++++++++++++++ pageserver/src/http/routes.rs | 33 ++++++++ pageserver/src/pgdatadir_mapping.rs | 56 +++++++++++-- test_runner/fixtures/pageserver/http.py | 9 ++ test_runner/regress/test_lsn_mapping.py | 105 +++++++++++++++++++++++- 7 files changed, 307 insertions(+), 19 deletions(-) diff --git a/libs/postgres_ffi/src/lib.rs b/libs/postgres_ffi/src/lib.rs index c9e5df9f04..d0009d0dc9 100644 --- a/libs/postgres_ffi/src/lib.rs +++ b/libs/postgres_ffi/src/lib.rs @@ -131,6 +131,7 @@ pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16; // Export some version independent functions that are used outside of this mod pub use v14::xlog_utils::encode_logical_message; +pub use v14::xlog_utils::from_pg_timestamp; pub use v14::xlog_utils::get_current_timestamp; pub use v14::xlog_utils::to_pg_timestamp; pub use v14::xlog_utils::XLogFileName; diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index 61a9c38a84..0ca9bd8b45 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -136,21 +136,42 @@ pub fn get_current_timestamp() -> TimestampTz { to_pg_timestamp(SystemTime::now()) } -pub fn to_pg_timestamp(time: SystemTime) -> TimestampTz { - const UNIX_EPOCH_JDATE: u64 = 2440588; /* == date2j(1970, 1, 1) */ - const POSTGRES_EPOCH_JDATE: u64 = 2451545; /* == date2j(2000, 1, 1) */ +// Module to reduce the scope of the constants +mod timestamp_conversions { + use std::time::Duration; + + use super::*; + + const UNIX_EPOCH_JDATE: u64 = 2440588; // == date2j(1970, 1, 1) + const POSTGRES_EPOCH_JDATE: u64 = 2451545; // == date2j(2000, 1, 1) const SECS_PER_DAY: u64 = 86400; const USECS_PER_SEC: u64 = 1000000; - match time.duration_since(SystemTime::UNIX_EPOCH) { - Ok(n) => { - ((n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)) - * USECS_PER_SEC - + n.subsec_micros() as u64) as i64 + const SECS_DIFF_UNIX_TO_POSTGRES_EPOCH: u64 = + (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY; + + pub fn to_pg_timestamp(time: SystemTime) -> TimestampTz { + match time.duration_since(SystemTime::UNIX_EPOCH) { + Ok(n) => { + ((n.as_secs() - SECS_DIFF_UNIX_TO_POSTGRES_EPOCH) * USECS_PER_SEC + + n.subsec_micros() as u64) as i64 + } + Err(_) => panic!("SystemTime before UNIX EPOCH!"), } - Err(_) => panic!("SystemTime before UNIX EPOCH!"), + } + + pub fn from_pg_timestamp(time: TimestampTz) -> SystemTime { + let time: u64 = time + .try_into() + .expect("timestamp before millenium (postgres epoch)"); + let since_unix_epoch = time + SECS_DIFF_UNIX_TO_POSTGRES_EPOCH * USECS_PER_SEC; + SystemTime::UNIX_EPOCH + .checked_add(Duration::from_micros(since_unix_epoch)) + .expect("SystemTime overflow") } } +pub use timestamp_conversions::{from_pg_timestamp, to_pg_timestamp}; + // Returns (aligned) end_lsn of the last record in data_dir with WAL segments. // start_lsn must point to some previously known record boundary (beginning of // the next record). If no valid record after is found, start_lsn is returned @@ -481,4 +502,24 @@ pub fn encode_logical_message(prefix: &str, message: &str) -> Vec { wal } -// If you need to craft WAL and write tests for this module, put it at wal_craft crate. +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ts_conversion() { + let now = SystemTime::now(); + let round_trip = from_pg_timestamp(to_pg_timestamp(now)); + + let now_since = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + let round_trip_since = round_trip.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + assert_eq!(now_since.as_micros(), round_trip_since.as_micros()); + + let now_pg = get_current_timestamp(); + let round_trip_pg = to_pg_timestamp(from_pg_timestamp(now_pg)); + + assert_eq!(now_pg, round_trip_pg); + } + + // If you need to craft WAL and write tests for this module, put it at wal_craft crate. +} diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 477a2d378d..f0bf2666a7 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -306,6 +306,67 @@ paths: schema: $ref: "#/components/schemas/ServiceUnavailableError" + /v1/tenant/{tenant_id}/timeline/{timeline_id}/get_timestamp_of_lsn: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex + - name: timeline_id + in: path + required: true + schema: + type: string + format: hex + get: + description: Get timestamp for a given LSN + parameters: + - name: lsn + in: query + required: true + schema: + type: integer + description: A LSN to get the timestamp + responses: + "200": + description: OK + content: + application/json: + schema: + type: string + format: date-time + "400": + description: Error when no tenant id found in path, no timeline id or invalid timestamp + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "401": + description: Unauthorized Error + content: + application/json: + schema: + $ref: "#/components/schemas/UnauthorizedError" + "403": + description: Forbidden Error + content: + application/json: + schema: + $ref: "#/components/schemas/ForbiddenError" + "404": + description: Timeline not found, or there is no timestamp information for the given lsn + content: + application/json: + schema: + $ref: "#/components/schemas/NotFoundError" + "500": + description: Generic operation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" /v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp: parameters: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index c00dad50ac..142e85d203 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -2,10 +2,12 @@ //! Management HTTP API //! use std::collections::HashMap; +use std::str::FromStr; use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use futures::TryFutureExt; +use humantime::format_rfc3339; use hyper::header::CONTENT_TYPE; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; @@ -502,6 +504,33 @@ async fn get_lsn_by_timestamp_handler( json_response(StatusCode::OK, result) } +async fn get_timestamp_of_lsn_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + check_permission(&request, Some(tenant_id))?; + + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + + let lsn_str = must_get_query_param(&request, "lsn")?; + let lsn = Lsn::from_str(&lsn_str) + .with_context(|| format!("Invalid LSN: {lsn_str:?}")) + .map_err(ApiError::BadRequest)?; + + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); + let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; + let result = timeline.get_timestamp_for_lsn(lsn, &ctx).await?; + + match result { + Some(time) => { + let time = format_rfc3339(postgres_ffi::from_pg_timestamp(time)).to_string(); + json_response(StatusCode::OK, time) + } + None => json_response(StatusCode::NOT_FOUND, ()), + } +} + async fn tenant_attach_handler( mut request: Request, _cancel: CancellationToken, @@ -1680,6 +1709,10 @@ pub fn make_router( "/v1/tenant/:tenant_id/timeline/:timeline_id/get_lsn_by_timestamp", |r| api_handler(r, get_lsn_by_timestamp_handler), ) + .get( + "/v1/tenant/:tenant_id/timeline/:timeline_id/get_timestamp_of_lsn", + |r| api_handler(r, get_timestamp_of_lsn_handler), + ) .put("/v1/tenant/:tenant_id/timeline/:timeline_id/do_gc", |r| { api_handler(r, timeline_gc_handler) }) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index ebba3d8579..d27c8a3d5d 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -19,6 +19,7 @@ use postgres_ffi::BLCKSZ; use postgres_ffi::{Oid, TimestampTz, TransactionId}; use serde::{Deserialize, Serialize}; use std::collections::{hash_map, HashMap, HashSet}; +use std::ops::ControlFlow; use std::ops::Range; use tokio_util::sync::CancellationToken; use tracing::{debug, trace, warn}; @@ -370,7 +371,6 @@ impl Timeline { } } - /// /// Subroutine of find_lsn_for_timestamp(). Returns true, if there are any /// commits that committed after 'search_timestamp', at LSN 'probe_lsn'. /// @@ -385,6 +385,50 @@ impl Timeline { found_larger: &mut bool, ctx: &RequestContext, ) -> Result { + self.map_all_timestamps(probe_lsn, ctx, |timestamp| { + if timestamp >= search_timestamp { + *found_larger = true; + return ControlFlow::Break(true); + } else { + *found_smaller = true; + } + ControlFlow::Continue(()) + }) + .await + } + + /// Obtain the possible timestamp range for the given lsn. + /// + /// If the lsn has no timestamps, returns None. returns `(min, max, median)` if it has timestamps. + pub async fn get_timestamp_for_lsn( + &self, + probe_lsn: Lsn, + ctx: &RequestContext, + ) -> Result, PageReconstructError> { + let mut max: Option = None; + self.map_all_timestamps(probe_lsn, ctx, |timestamp| { + if let Some(max_prev) = max { + max = Some(max_prev.max(timestamp)); + } else { + max = Some(timestamp); + } + ControlFlow::Continue(()) + }) + .await?; + + Ok(max) + } + + /// Runs the given function on all the timestamps for a given lsn + /// + /// The return value is either given by the closure, or set to the `Default` + /// impl's output. + async fn map_all_timestamps( + &self, + probe_lsn: Lsn, + ctx: &RequestContext, + mut f: impl FnMut(TimestampTz) -> ControlFlow, + ) -> Result { for segno in self .list_slru_segments(SlruKind::Clog, probe_lsn, ctx) .await? @@ -402,16 +446,14 @@ impl Timeline { timestamp_bytes.copy_from_slice(&clog_page[BLCKSZ as usize..]); let timestamp = TimestampTz::from_be_bytes(timestamp_bytes); - if timestamp >= search_timestamp { - *found_larger = true; - return Ok(true); - } else { - *found_smaller = true; + match f(timestamp) { + ControlFlow::Break(b) => return Ok(b), + ControlFlow::Continue(()) => (), } } } } - Ok(false) + Ok(Default::default()) } /// Get a list of SLRU segments diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 460a30ad56..598b48b56f 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -453,6 +453,15 @@ class PageserverHttpClient(requests.Session): res_json = res.json() return res_json + def timeline_get_timestamp_of_lsn(self, tenant_id: TenantId, timeline_id: TimelineId, lsn: Lsn): + log.info(f"Requesting time range of lsn {lsn}, tenant {tenant_id}, timeline {timeline_id}") + res = self.get( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_timestamp_of_lsn?lsn={lsn}", + ) + self.verbose_error(res) + res_json = res.json() + return res_json + def timeline_checkpoint(self, tenant_id: TenantId, timeline_id: TimelineId): self.is_testing_enabled_or_skip() diff --git a/test_runner/regress/test_lsn_mapping.py b/test_runner/regress/test_lsn_mapping.py index 8ccfc21cf7..03606e3c1c 100644 --- a/test_runner/regress/test_lsn_mapping.py +++ b/test_runner/regress/test_lsn_mapping.py @@ -1,7 +1,10 @@ -from datetime import timedelta +import time +from datetime import datetime, timedelta, timezone from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder, wait_for_last_flush_lsn +from fixtures.pageserver.http import PageserverApiException +from fixtures.types import Lsn from fixtures.utils import query_scalar @@ -25,13 +28,14 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): cur.execute("CREATE TABLE foo (x integer)") tbl = [] for i in range(1000): - cur.execute(f"INSERT INTO foo VALUES({i})") + cur.execute("INSERT INTO foo VALUES(%s)", (i,)) # Get the timestamp at UTC after_timestamp = query_scalar(cur, "SELECT clock_timestamp()").replace(tzinfo=None) tbl.append([i, after_timestamp]) # Execute one more transaction with synchronous_commit enabled, to flush # all the previous transactions + cur.execute("SET synchronous_commit=on") cur.execute("INSERT INTO foo VALUES (-1)") # Wait until WAL is received by pageserver @@ -67,3 +71,100 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): assert endpoint_here.safe_psql("SELECT max(x) FROM foo")[0][0] == i endpoint_here.stop_and_destroy() + + +# Test pageserver get_timestamp_of_lsn API +def test_ts_of_lsn_api(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + + new_timeline_id = env.neon_cli.create_branch("test_ts_of_lsn_api") + endpoint_main = env.endpoints.create_start("test_ts_of_lsn_api") + log.info("postgres is running on 'test_ts_of_lsn_api' branch") + + cur = endpoint_main.connect().cursor() + # Create table, and insert rows, each in a separate transaction + # Disable synchronous_commit to make this initialization go faster. + # + # Each row contains current insert LSN and the current timestamp, when + # the row was inserted. + cur.execute("SET synchronous_commit=off") + cur.execute("CREATE TABLE foo (x integer)") + tbl = [] + for i in range(1000): + cur.execute("INSERT INTO foo VALUES(%s)", (i,)) + # Get the timestamp at UTC + after_timestamp = query_scalar(cur, "SELECT clock_timestamp()").replace(tzinfo=timezone.utc) + after_lsn = query_scalar(cur, "SELECT pg_current_wal_lsn()") + tbl.append([i, after_timestamp, after_lsn]) + time.sleep(0.005) + + # Execute one more transaction with synchronous_commit enabled, to flush + # all the previous transactions + cur.execute("SET synchronous_commit=on") + cur.execute("INSERT INTO foo VALUES (-1)") + + # Wait until WAL is received by pageserver + last_flush_lsn = wait_for_last_flush_lsn( + env, endpoint_main, env.initial_tenant, new_timeline_id + ) + + with env.pageserver.http_client() as client: + # Check edge cases: lsn larger than the last flush lsn + probe_lsn = Lsn(int(last_flush_lsn) * 20 + 80_000) + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + probe_lsn, + ) + + # lsn of zero + try: + probe_lsn = Lsn(0) + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + probe_lsn, + ) + # There should always be an error here. + raise RuntimeError("there should have been an 'Invalid LSN' error") + except PageserverApiException as error: + assert error.status_code == 500 + assert str(error) == "Invalid LSN" + env.pageserver.allowed_errors.append(".*Invalid LSN.*") + + # small lsn before initdb_lsn + try: + probe_lsn = Lsn(64) + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + probe_lsn, + ) + # There should always be an error here. + raise RuntimeError("there should have been an 'could not find data for key' error") + except PageserverApiException as error: + assert error.status_code == 500 + assert str(error).startswith("could not find data for key") + env.pageserver.allowed_errors.append(".*could not find data for key.*") + + # Probe a bunch of timestamps in the valid range + step_size = 100 + for i in range(step_size, len(tbl), step_size): + after_timestamp = tbl[i][1] + after_lsn = tbl[i][2] + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + after_lsn, + ) + log.info("result: %s, after_ts: %s", result, after_timestamp) + + # TODO use fromisoformat once we have Python 3.11+ + # which has https://github.com/python/cpython/pull/92177 + timestamp = datetime.strptime(result, "%Y-%m-%dT%H:%M:%S.%f000Z").replace( + tzinfo=timezone.utc + ) + assert timestamp < after_timestamp, "after_timestamp after timestamp" + if i > 1: + before_timestamp = tbl[i - step_size][1] + assert timestamp >= before_timestamp, "before_timestamp before timestamp" From b1d6af5ebe8a396f921ba05a720d3f004edaa231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 19 Oct 2023 14:31:09 +0200 Subject: [PATCH 24/49] Azure blobs: Simplify error conversion by addition of to_download_error (#5575) There is a bunch of duplication and manual Result handling that can be simplified by moving the error conversion into a shared function, using `map_err`, and the question mark operator. --- libs/remote_storage/src/azure_blob.rs | 78 ++++++++------------------- 1 file changed, 23 insertions(+), 55 deletions(-) diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index b81cb47606..152929ecd3 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -121,22 +121,7 @@ impl AzureBlobStorage { // https://github.com/neondatabase/neon/issues/5563 let mut buf = Vec::new(); while let Some(part) = response.next().await { - let part = match part { - Ok(l) => l, - Err(e) => { - return Err(if let Some(http_err) = e.as_http_error() { - match http_err.status() { - StatusCode::NotFound => DownloadError::NotFound, - StatusCode::BadRequest => { - DownloadError::BadInput(anyhow::Error::new(e)) - } - _ => DownloadError::Other(anyhow::Error::new(e)), - } - } else { - DownloadError::Other(e.into()) - }); - } - }; + let part = part.map_err(to_download_error)?; let data = part .data .collect() @@ -157,30 +142,16 @@ impl AzureBlobStorage { ) -> Result { let builder = blob_client.get_metadata(); - match builder.into_future().await { - Ok(r) => { - let mut map = HashMap::new(); + let response = builder.into_future().await.map_err(to_download_error)?; + let mut map = HashMap::new(); - for md in r.metadata.iter() { - map.insert( - md.name().as_str().to_string(), - md.value().as_str().to_string(), - ); - } - Ok(StorageMetadata(map)) - } - Err(e) => { - return Err(if let Some(http_err) = e.as_http_error() { - match http_err.status() { - StatusCode::NotFound => DownloadError::NotFound, - StatusCode::BadRequest => DownloadError::BadInput(anyhow::Error::new(e)), - _ => DownloadError::Other(anyhow::Error::new(e)), - } - } else { - DownloadError::Other(e.into()) - }); - } + for md in response.metadata.iter() { + map.insert( + md.name().as_str().to_string(), + md.value().as_str().to_string(), + ); } + Ok(StorageMetadata(map)) } async fn permit(&self, kind: RequestKind) -> tokio::sync::SemaphorePermit<'_> { @@ -199,6 +170,18 @@ fn to_azure_metadata(metadata: StorageMetadata) -> Metadata { res } +fn to_download_error(error: azure_core::Error) -> DownloadError { + if let Some(http_err) = error.as_http_error() { + match http_err.status() { + StatusCode::NotFound => DownloadError::NotFound, + StatusCode::BadRequest => DownloadError::BadInput(anyhow::Error::new(error)), + _ => DownloadError::Other(anyhow::Error::new(error)), + } + } else { + DownloadError::Other(error.into()) + } +} + #[async_trait::async_trait] impl RemoteStorage for AzureBlobStorage { async fn list_prefixes( @@ -233,23 +216,8 @@ impl RemoteStorage for AzureBlobStorage { let mut response = builder.into_stream(); let mut res = Vec::new(); - while let Some(l) = response.next().await { - let entry = match l { - Ok(l) => l, - Err(e) => { - return Err(if let Some(http_err) = e.as_http_error() { - match http_err.status() { - StatusCode::NotFound => DownloadError::NotFound, - StatusCode::BadRequest => { - DownloadError::BadInput(anyhow::Error::new(e)) - } - _ => DownloadError::Other(anyhow::Error::new(e)), - } - } else { - DownloadError::Other(e.into()) - }); - } - }; + while let Some(entry) = response.next().await { + let entry = entry.map_err(to_download_error)?; let name_iter = entry .blobs .prefixes() From 572eda44ee99a32d1be1b0f673c56f7a0eb6efae Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 19 Oct 2023 13:32:19 +0100 Subject: [PATCH 25/49] update tokio-postgres (#5597) https://github.com/neondatabase/rust-postgres/pull/23 --- Cargo.lock | 10 +++++----- Cargo.toml | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb0fe3fe20..67efa44872 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3561,7 +3561,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "bytes", "fallible-iterator", @@ -3574,7 +3574,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "native-tls", "tokio", @@ -3585,7 +3585,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "base64 0.20.0", "byteorder", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "bytes", "fallible-iterator", @@ -5407,7 +5407,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d#a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "async-trait", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index 727fcbd5a0..26f523ce19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,11 +160,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -200,7 +200,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="a2d0652ec3f8f710ff8cfc2e7c68f096fb852d9d" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } ################# Binary contents sections From 3a19da1066a3b84e13882620cb432da8336ce4c7 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 19 Oct 2023 13:49:06 +0100 Subject: [PATCH 26/49] build(deps): bump rustix from 0.37.19 to 0.37.25 (#5596) ## Problem @dependabot has bumped `rustix` 0.36 version to the latest in https://github.com/neondatabase/neon/pull/5591, but didn't bump 0.37. Also, update all Rust dependencies for `test_runner/pg_clients/rust/tokio-postgres`. Fixes - https://github.com/neondatabase/neon/security/dependabot/39 - https://github.com/neondatabase/neon/security/dependabot/40 ## Summary of changes - `cargo update -p rustix@0.37.19` - Update all dependencies for `test_runner/pg_clients/rust/tokio-postgres` --- Cargo.lock | 10 +- .../pg_clients/rust/tokio-postgres/Cargo.lock | 592 +++++++++--------- .../pg_clients/rust/tokio-postgres/Cargo.toml | 4 +- .../pg_clients/rust/tokio-postgres/Dockerfile | 2 +- 4 files changed, 309 insertions(+), 299 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67efa44872..3a916b464d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ dependencies = [ "log", "parking", "polling", - "rustix 0.37.19", + "rustix 0.37.25", "slab", "socket2 0.4.9", "waker-fn", @@ -2582,7 +2582,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi", "io-lifetimes", - "rustix 0.37.19", + "rustix 0.37.25", "windows-sys 0.48.0", ] @@ -4331,9 +4331,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.19" +version = "0.37.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" dependencies = [ "bitflags", "errno", @@ -5174,7 +5174,7 @@ dependencies = [ "cfg-if", "fastrand 1.9.0", "redox_syscall 0.3.5", - "rustix 0.37.19", + "rustix 0.37.25", "windows-sys 0.45.0", ] diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock index bdbbe0ad69..4ea21eb378 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock @@ -3,10 +3,25 @@ version = 3 [[package]] -name = "async-trait" -version = "0.1.68" +name = "addr2line" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -20,10 +35,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "base64" -version = "0.21.0" +name = "backtrace" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bitflags" @@ -31,6 +61,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.10.4" @@ -41,22 +77,31 @@ dependencies = [ ] [[package]] -name = "byteorder" -version = "1.4.3" +name = "bumpalo" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -82,9 +127,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -101,9 +146,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -112,23 +157,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys", ] [[package]] @@ -139,12 +173,15 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" [[package]] name = "foreign-types" @@ -262,9 +299,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -272,10 +309,10 @@ dependencies = [ ] [[package]] -name = "hermit-abi" -version = "0.3.1" +name = "gimli" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "hmac" @@ -287,23 +324,12 @@ dependencies = [ ] [[package]] -name = "instant" -version = "0.1.12" +name = "js-sys" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", + "wasm-bindgen", ] [[package]] @@ -314,21 +340,21 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "linux-raw-sys" -version = "0.3.4" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -336,38 +362,44 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -389,18 +421,27 @@ dependencies = [ ] [[package]] -name = "once_cell" -version = "1.17.1" +name = "object" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -428,9 +469,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -450,46 +491,46 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.4.1", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_shared", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -499,9 +540,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "postgres-native-tls" @@ -518,9 +559,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" +checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ "base64", "byteorder", @@ -536,9 +577,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6" +checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" dependencies = [ "bytes", "fallible-iterator", @@ -553,18 +594,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.26" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -599,22 +640,22 @@ dependencies = [ "getrandom", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", ] [[package]] @@ -628,41 +669,46 @@ dependencies = [ ] [[package]] -name = "rustix" -version = "0.37.15" +name = "rustc-demangle" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0661814f891c57c930a610266415528da53c4933e6dea5fb350cbfe048a9ece" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags", + "bitflags 2.4.1", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -671,9 +717,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -681,9 +727,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -692,66 +738,57 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" -version = "0.4.9" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d283f86695ae989d1e18440a943880967156325ba025f05049946bff47bcc2b" -dependencies = [ - "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "stringprep" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" dependencies = [ + "finl_unicode", "unicode-bidi", "unicode-normalization", ] [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.15" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -760,15 +797,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -788,18 +825,18 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", "pin-project-lite", - "socket2 0.4.9", + "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -825,9 +862,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" +checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" dependencies = [ "async-trait", "byteorder", @@ -842,16 +879,18 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2 0.5.2", + "rand", + "socket2", "tokio", "tokio-util", + "whoami", ] [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -863,9 +902,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.38" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9cf6a813d3f40c88b0b6b6f29a5c95c6cdbf97c1f9cc53fb820200f5ad814d" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", "tracing-core", @@ -873,18 +912,18 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" @@ -894,9 +933,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -926,49 +965,77 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "winapi" -version = "0.3.9" +name = "wasm-bindgen" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "cfg-if", + "wasm-bindgen-macro", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" +name = "wasm-bindgen-backend" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", ] [[package]] -name = "windows-sys" -version = "0.45.0" +name = "wasm-bindgen-macro" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "windows-targets 0.42.2", + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +dependencies = [ + "wasm-bindgen", + "web-sys", ] [[package]] @@ -977,119 +1044,62 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml b/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml index 4675ac8a3f..6f100aafd5 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml @@ -9,8 +9,8 @@ publish = false [dependencies] native-tls = "0.2.11" postgres-native-tls = "0.5.0" -tokio = { version = "1.28", features=["rt", "macros"] } -tokio-postgres = "0.7.8" +tokio = { version = "1.33", features=["rt", "macros"] } +tokio-postgres = "0.7.10" # This is not part of the main 'neon' workspace diff --git a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile index 35ae25a470..1d3709803e 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile +++ b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.70 +FROM rust:1.73 WORKDIR /source COPY . . From 66f8f5f1c88073ddd00a5172a07f26a649fed122 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 19 Oct 2023 14:17:15 +0100 Subject: [PATCH 27/49] Call walproposer from Rust (#5403) Create Rust bindings for C functions from walproposer. This allows to write better tests with real walproposer code without spawning multiple processes and starting up the whole environment. `make walproposer-lib` stage was added to build static libraries `libwalproposer.a`, `libpgport.a`, `libpgcommon.a`. These libraries can be statically linked to any executable to call walproposer functions. `libs/walproposer/src/walproposer.rs` contains `test_simple_sync_safekeepers` to test that walproposer can be called from Rust to emulate sync_safekeepers logic. It can also be used as a usage example. --- .github/workflows/build_and_test.yml | 3 + .github/workflows/neon_extra_builds.yml | 3 + Cargo.lock | 11 + Cargo.toml | 2 + Makefile | 36 +- libs/walproposer/Cargo.toml | 16 + libs/walproposer/bindgen_deps.h | 1 + libs/walproposer/build.rs | 113 ++++++ libs/walproposer/src/api_bindings.rs | 455 ++++++++++++++++++++++ libs/walproposer/src/lib.rs | 14 + libs/walproposer/src/walproposer.rs | 485 ++++++++++++++++++++++++ pgxn/neon/Makefile | 17 + pgxn/neon/walproposer.c | 283 +++++++------- pgxn/neon/walproposer.h | 138 ++++--- pgxn/neon/walproposer_compat.c | 192 ++++++++++ pgxn/neon/walproposer_pg.c | 204 ++++++---- 16 files changed, 1711 insertions(+), 262 deletions(-) create mode 100644 libs/walproposer/Cargo.toml create mode 100644 libs/walproposer/bindgen_deps.h create mode 100644 libs/walproposer/build.rs create mode 100644 libs/walproposer/src/api_bindings.rs create mode 100644 libs/walproposer/src/lib.rs create mode 100644 libs/walproposer/src/walproposer.rs create mode 100644 pgxn/neon/walproposer_compat.c diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1fed98f202..3f1c728c70 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -320,6 +320,9 @@ jobs: - name: Build neon extensions run: mold -run make neon-pg-ext -j$(nproc) + - name: Build walproposer-lib + run: mold -run make walproposer-lib -j$(nproc) + - name: Run cargo build run: | ${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 8a1e4571fd..891cc8472a 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -103,6 +103,9 @@ jobs: - name: Build neon extensions run: make neon-pg-ext -j$(nproc) + - name: Build walproposer-lib + run: make walproposer-lib -j$(nproc) + - name: Run cargo build run: cargo build --all --release diff --git a/Cargo.lock b/Cargo.lock index 3a916b464d..f7598a79cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6092,6 +6092,17 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "walproposer" +version = "0.1.0" +dependencies = [ + "anyhow", + "bindgen", + "postgres_ffi", + "utils", + "workspace_hack", +] + [[package]] name = "want" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 26f523ce19..a0be7bb9ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "libs/tracing-utils", "libs/postgres_ffi/wal_craft", "libs/vm_monitor", + "libs/walproposer", ] [workspace.package] @@ -185,6 +186,7 @@ tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" } tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" } utils = { version = "0.1", path = "./libs/utils/" } vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" } +walproposer = { version = "0.1", path = "./libs/walproposer/" } ## Common library dependency workspace_hack = { version = "0.1", path = "./workspace_hack/" } diff --git a/Makefile b/Makefile index 33b5dcad99..3b3f0e3dac 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ all: neon postgres neon-pg-ext # # The 'postgres_ffi' depends on the Postgres headers. .PHONY: neon -neon: postgres-headers +neon: postgres-headers walproposer-lib +@echo "Compiling Neon" $(CARGO_CMD_PREFIX) cargo build $(CARGO_BUILD_FLAGS) @@ -168,6 +168,40 @@ neon-pg-ext-clean-%: -C $(POSTGRES_INSTALL_DIR)/build/neon-utils-$* \ -f $(ROOT_PROJECT_DIR)/pgxn/neon_utils/Makefile clean +# Build walproposer as a static library. walproposer source code is located +# in the pgxn/neon directory. +# +# We also need to include libpgport.a and libpgcommon.a, because walproposer +# uses some functions from those libraries. +# +# Some object files are removed from libpgport.a and libpgcommon.a because +# they depend on openssl and other libraries that are not included in our +# Rust build. +.PHONY: walproposer-lib +walproposer-lib: neon-pg-ext-v16 + +@echo "Compiling walproposer-lib" + mkdir -p $(POSTGRES_INSTALL_DIR)/build/walproposer-lib + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v16/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \ + -C $(POSTGRES_INSTALL_DIR)/build/walproposer-lib \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile walproposer-lib + cp $(POSTGRES_INSTALL_DIR)/v16/lib/libpgport.a $(POSTGRES_INSTALL_DIR)/build/walproposer-lib + cp $(POSTGRES_INSTALL_DIR)/v16/lib/libpgcommon.a $(POSTGRES_INSTALL_DIR)/build/walproposer-lib + $(AR) d $(POSTGRES_INSTALL_DIR)/build/walproposer-lib/libpgport.a \ + pg_strong_random.o + $(AR) d $(POSTGRES_INSTALL_DIR)/build/walproposer-lib/libpgcommon.a \ + pg_crc32c.o \ + hmac_openssl.o \ + cryptohash_openssl.o \ + scram-common.o \ + md5_common.o \ + checksum_helper.o + +.PHONY: walproposer-lib-clean +walproposer-lib-clean: + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v16/bin/pg_config \ + -C $(POSTGRES_INSTALL_DIR)/build/walproposer-lib \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile clean + .PHONY: neon-pg-ext neon-pg-ext: \ neon-pg-ext-v14 \ diff --git a/libs/walproposer/Cargo.toml b/libs/walproposer/Cargo.toml new file mode 100644 index 0000000000..73aa073c44 --- /dev/null +++ b/libs/walproposer/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "walproposer" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +utils.workspace = true +postgres_ffi.workspace = true + +workspace_hack.workspace = true + +[build-dependencies] +anyhow.workspace = true +bindgen.workspace = true diff --git a/libs/walproposer/bindgen_deps.h b/libs/walproposer/bindgen_deps.h new file mode 100644 index 0000000000..b95788347c --- /dev/null +++ b/libs/walproposer/bindgen_deps.h @@ -0,0 +1 @@ +#include "walproposer.h" diff --git a/libs/walproposer/build.rs b/libs/walproposer/build.rs new file mode 100644 index 0000000000..d32c8ab299 --- /dev/null +++ b/libs/walproposer/build.rs @@ -0,0 +1,113 @@ +use std::{env, path::PathBuf, process::Command}; + +use anyhow::{anyhow, Context}; +use bindgen::CargoCallbacks; + +fn main() -> anyhow::Result<()> { + // Tell cargo to invalidate the built crate whenever the wrapper changes + println!("cargo:rerun-if-changed=bindgen_deps.h"); + + // Finding the location of built libraries and Postgres C headers: + // - if POSTGRES_INSTALL_DIR is set look into it, otherwise look into `/pg_install` + // - if there's a `bin/pg_config` file use it for getting include server, otherwise use `/pg_install/{PG_MAJORVERSION}/include/postgresql/server` + let pg_install_dir = if let Some(postgres_install_dir) = env::var_os("POSTGRES_INSTALL_DIR") { + postgres_install_dir.into() + } else { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../pg_install") + }; + + let pg_install_abs = std::fs::canonicalize(pg_install_dir)?; + let walproposer_lib_dir = pg_install_abs.join("build/walproposer-lib"); + let walproposer_lib_search_str = walproposer_lib_dir + .to_str() + .ok_or(anyhow!("Bad non-UTF path"))?; + + let pgxn_neon = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../pgxn/neon"); + let pgxn_neon = std::fs::canonicalize(pgxn_neon)?; + let pgxn_neon = pgxn_neon.to_str().ok_or(anyhow!("Bad non-UTF path"))?; + + println!("cargo:rustc-link-lib=static=pgport"); + println!("cargo:rustc-link-lib=static=pgcommon"); + println!("cargo:rustc-link-lib=static=walproposer"); + println!("cargo:rustc-link-search={walproposer_lib_search_str}"); + + let pg_config_bin = pg_install_abs.join("v16").join("bin").join("pg_config"); + let inc_server_path: String = if pg_config_bin.exists() { + let output = Command::new(pg_config_bin) + .arg("--includedir-server") + .output() + .context("failed to execute `pg_config --includedir-server`")?; + + if !output.status.success() { + panic!("`pg_config --includedir-server` failed") + } + + String::from_utf8(output.stdout) + .context("pg_config output is not UTF-8")? + .trim_end() + .into() + } else { + let server_path = pg_install_abs + .join("v16") + .join("include") + .join("postgresql") + .join("server") + .into_os_string(); + server_path + .into_string() + .map_err(|s| anyhow!("Bad postgres server path {s:?}"))? + }; + + // The bindgen::Builder is the main entry point + // to bindgen, and lets you build up options for + // the resulting bindings. + let bindings = bindgen::Builder::default() + // The input header we would like to generate + // bindings for. + .header("bindgen_deps.h") + // Tell cargo to invalidate the built crate whenever any of the + // included header files changed. + .parse_callbacks(Box::new(CargoCallbacks)) + .allowlist_type("WalProposer") + .allowlist_type("WalProposerConfig") + .allowlist_type("walproposer_api") + .allowlist_function("WalProposerCreate") + .allowlist_function("WalProposerStart") + .allowlist_function("WalProposerBroadcast") + .allowlist_function("WalProposerPoll") + .allowlist_function("WalProposerFree") + .allowlist_var("DEBUG5") + .allowlist_var("DEBUG4") + .allowlist_var("DEBUG3") + .allowlist_var("DEBUG2") + .allowlist_var("DEBUG1") + .allowlist_var("LOG") + .allowlist_var("INFO") + .allowlist_var("NOTICE") + .allowlist_var("WARNING") + .allowlist_var("ERROR") + .allowlist_var("FATAL") + .allowlist_var("PANIC") + .allowlist_var("WPEVENT") + .allowlist_var("WL_LATCH_SET") + .allowlist_var("WL_SOCKET_READABLE") + .allowlist_var("WL_SOCKET_WRITEABLE") + .allowlist_var("WL_TIMEOUT") + .allowlist_var("WL_SOCKET_CLOSED") + .allowlist_var("WL_SOCKET_MASK") + .clang_arg("-DWALPROPOSER_LIB") + .clang_arg(format!("-I{pgxn_neon}")) + .clang_arg(format!("-I{inc_server_path}")) + // Finish the builder and generate the bindings. + .generate() + // Unwrap the Result and panic on failure. + .expect("Unable to generate bindings"); + + // Write the bindings to the $OUT_DIR/bindings.rs file. + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("bindings.rs"); + bindings + .write_to_file(out_path) + .expect("Couldn't write bindings!"); + + Ok(()) +} diff --git a/libs/walproposer/src/api_bindings.rs b/libs/walproposer/src/api_bindings.rs new file mode 100644 index 0000000000..b0ac2a879e --- /dev/null +++ b/libs/walproposer/src/api_bindings.rs @@ -0,0 +1,455 @@ +#![allow(dead_code)] + +use std::ffi::CStr; +use std::ffi::CString; + +use crate::bindings::uint32; +use crate::bindings::walproposer_api; +use crate::bindings::PGAsyncReadResult; +use crate::bindings::PGAsyncWriteResult; +use crate::bindings::Safekeeper; +use crate::bindings::Size; +use crate::bindings::StringInfoData; +use crate::bindings::TimeLineID; +use crate::bindings::TimestampTz; +use crate::bindings::WalProposer; +use crate::bindings::WalProposerConnStatusType; +use crate::bindings::WalProposerConnectPollStatusType; +use crate::bindings::WalProposerExecStatusType; +use crate::bindings::WalproposerShmemState; +use crate::bindings::XLogRecPtr; +use crate::walproposer::ApiImpl; +use crate::walproposer::WaitResult; + +extern "C" fn get_shmem_state(wp: *mut WalProposer) -> *mut WalproposerShmemState { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_shmem_state() + } +} + +extern "C" fn start_streaming(wp: *mut WalProposer, startpos: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).start_streaming(startpos) + } +} + +extern "C" fn get_flush_rec_ptr(wp: *mut WalProposer) -> XLogRecPtr { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_flush_rec_ptr() + } +} + +extern "C" fn get_current_timestamp(wp: *mut WalProposer) -> TimestampTz { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_current_timestamp() + } +} + +extern "C" fn conn_error_message(sk: *mut Safekeeper) -> *mut ::std::os::raw::c_char { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + let msg = (*api).conn_error_message(&mut (*sk)); + let msg = CString::new(msg).unwrap(); + // TODO: fix leaking error message + msg.into_raw() + } +} + +extern "C" fn conn_status(sk: *mut Safekeeper) -> WalProposerConnStatusType { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_status(&mut (*sk)) + } +} + +extern "C" fn conn_connect_start(sk: *mut Safekeeper) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_connect_start(&mut (*sk)) + } +} + +extern "C" fn conn_connect_poll(sk: *mut Safekeeper) -> WalProposerConnectPollStatusType { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_connect_poll(&mut (*sk)) + } +} + +extern "C" fn conn_send_query(sk: *mut Safekeeper, query: *mut ::std::os::raw::c_char) -> bool { + let query = unsafe { CStr::from_ptr(query) }; + let query = query.to_str().unwrap(); + + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_send_query(&mut (*sk), query) + } +} + +extern "C" fn conn_get_query_result(sk: *mut Safekeeper) -> WalProposerExecStatusType { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_get_query_result(&mut (*sk)) + } +} + +extern "C" fn conn_flush(sk: *mut Safekeeper) -> ::std::os::raw::c_int { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_flush(&mut (*sk)) + } +} + +extern "C" fn conn_finish(sk: *mut Safekeeper) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_finish(&mut (*sk)) + } +} + +extern "C" fn conn_async_read( + sk: *mut Safekeeper, + buf: *mut *mut ::std::os::raw::c_char, + amount: *mut ::std::os::raw::c_int, +) -> PGAsyncReadResult { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + let (res, result) = (*api).conn_async_read(&mut (*sk)); + + // This function has guarantee that returned buf will be valid until + // the next call. So we can store a Vec in each Safekeeper and reuse + // it on the next call. + let mut inbuf = take_vec_u8(&mut (*sk).inbuf).unwrap_or_default(); + + inbuf.clear(); + inbuf.extend_from_slice(res); + + // Put a Vec back to sk->inbuf and return data ptr. + *buf = store_vec_u8(&mut (*sk).inbuf, inbuf); + *amount = res.len() as i32; + + result + } +} + +extern "C" fn conn_async_write( + sk: *mut Safekeeper, + buf: *const ::std::os::raw::c_void, + size: usize, +) -> PGAsyncWriteResult { + unsafe { + let buf = std::slice::from_raw_parts(buf as *const u8, size); + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_async_write(&mut (*sk), buf) + } +} + +extern "C" fn conn_blocking_write( + sk: *mut Safekeeper, + buf: *const ::std::os::raw::c_void, + size: usize, +) -> bool { + unsafe { + let buf = std::slice::from_raw_parts(buf as *const u8, size); + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_blocking_write(&mut (*sk), buf) + } +} + +extern "C" fn recovery_download( + sk: *mut Safekeeper, + _timeline: TimeLineID, + startpos: XLogRecPtr, + endpos: XLogRecPtr, +) -> bool { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).recovery_download(&mut (*sk), startpos, endpos) + } +} + +extern "C" fn wal_read( + sk: *mut Safekeeper, + buf: *mut ::std::os::raw::c_char, + startptr: XLogRecPtr, + count: Size, +) { + unsafe { + let buf = std::slice::from_raw_parts_mut(buf as *mut u8, count); + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).wal_read(&mut (*sk), buf, startptr) + } +} + +extern "C" fn wal_reader_allocate(sk: *mut Safekeeper) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).wal_reader_allocate(&mut (*sk)); + } +} + +extern "C" fn free_event_set(wp: *mut WalProposer) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).free_event_set(&mut (*wp)); + } +} + +extern "C" fn init_event_set(wp: *mut WalProposer) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).init_event_set(&mut (*wp)); + } +} + +extern "C" fn update_event_set(sk: *mut Safekeeper, events: uint32) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).update_event_set(&mut (*sk), events); + } +} + +extern "C" fn add_safekeeper_event_set(sk: *mut Safekeeper, events: uint32) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).add_safekeeper_event_set(&mut (*sk), events); + } +} + +extern "C" fn wait_event_set( + wp: *mut WalProposer, + timeout: ::std::os::raw::c_long, + event_sk: *mut *mut Safekeeper, + events: *mut uint32, +) -> ::std::os::raw::c_int { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + let result = (*api).wait_event_set(&mut (*wp), timeout); + match result { + WaitResult::Latch => { + *event_sk = std::ptr::null_mut(); + *events = crate::bindings::WL_LATCH_SET; + 1 + } + WaitResult::Timeout => { + *event_sk = std::ptr::null_mut(); + *events = crate::bindings::WL_TIMEOUT; + 0 + } + WaitResult::Network(sk, event_mask) => { + *event_sk = sk; + *events = event_mask; + 1 + } + } + } +} + +extern "C" fn strong_random( + wp: *mut WalProposer, + buf: *mut ::std::os::raw::c_void, + len: usize, +) -> bool { + unsafe { + let buf = std::slice::from_raw_parts_mut(buf as *mut u8, len); + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).strong_random(buf) + } +} + +extern "C" fn get_redo_start_lsn(wp: *mut WalProposer) -> XLogRecPtr { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_redo_start_lsn() + } +} + +extern "C" fn finish_sync_safekeepers(wp: *mut WalProposer, lsn: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).finish_sync_safekeepers(lsn) + } +} + +extern "C" fn process_safekeeper_feedback(wp: *mut WalProposer, commit_lsn: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).process_safekeeper_feedback(&mut (*wp), commit_lsn) + } +} + +extern "C" fn confirm_wal_streamed(wp: *mut WalProposer, lsn: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).confirm_wal_streamed(&mut (*wp), lsn) + } +} + +extern "C" fn log_internal( + wp: *mut WalProposer, + level: ::std::os::raw::c_int, + line: *const ::std::os::raw::c_char, +) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + let line = CStr::from_ptr(line); + let line = line.to_str().unwrap(); + (*api).log_internal(&mut (*wp), Level::from(level as u32), line) + } +} + +extern "C" fn after_election(wp: *mut WalProposer) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).after_election(&mut (*wp)) + } +} + +#[derive(Debug)] +pub enum Level { + Debug5, + Debug4, + Debug3, + Debug2, + Debug1, + Log, + Info, + Notice, + Warning, + Error, + Fatal, + Panic, + WPEvent, +} + +impl Level { + pub fn from(elevel: u32) -> Level { + use crate::bindings::*; + + match elevel { + DEBUG5 => Level::Debug5, + DEBUG4 => Level::Debug4, + DEBUG3 => Level::Debug3, + DEBUG2 => Level::Debug2, + DEBUG1 => Level::Debug1, + LOG => Level::Log, + INFO => Level::Info, + NOTICE => Level::Notice, + WARNING => Level::Warning, + ERROR => Level::Error, + FATAL => Level::Fatal, + PANIC => Level::Panic, + WPEVENT => Level::WPEvent, + _ => panic!("unknown log level {}", elevel), + } + } +} + +pub(crate) fn create_api() -> walproposer_api { + walproposer_api { + get_shmem_state: Some(get_shmem_state), + start_streaming: Some(start_streaming), + get_flush_rec_ptr: Some(get_flush_rec_ptr), + get_current_timestamp: Some(get_current_timestamp), + conn_error_message: Some(conn_error_message), + conn_status: Some(conn_status), + conn_connect_start: Some(conn_connect_start), + conn_connect_poll: Some(conn_connect_poll), + conn_send_query: Some(conn_send_query), + conn_get_query_result: Some(conn_get_query_result), + conn_flush: Some(conn_flush), + conn_finish: Some(conn_finish), + conn_async_read: Some(conn_async_read), + conn_async_write: Some(conn_async_write), + conn_blocking_write: Some(conn_blocking_write), + recovery_download: Some(recovery_download), + wal_read: Some(wal_read), + wal_reader_allocate: Some(wal_reader_allocate), + free_event_set: Some(free_event_set), + init_event_set: Some(init_event_set), + update_event_set: Some(update_event_set), + add_safekeeper_event_set: Some(add_safekeeper_event_set), + wait_event_set: Some(wait_event_set), + strong_random: Some(strong_random), + get_redo_start_lsn: Some(get_redo_start_lsn), + finish_sync_safekeepers: Some(finish_sync_safekeepers), + process_safekeeper_feedback: Some(process_safekeeper_feedback), + confirm_wal_streamed: Some(confirm_wal_streamed), + log_internal: Some(log_internal), + after_election: Some(after_election), + } +} + +impl std::fmt::Display for Level { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +/// Take ownership of `Vec` from StringInfoData. +pub(crate) fn take_vec_u8(pg: &mut StringInfoData) -> Option> { + if pg.data.is_null() { + return None; + } + + let ptr = pg.data as *mut u8; + let length = pg.len as usize; + let capacity = pg.maxlen as usize; + + pg.data = std::ptr::null_mut(); + pg.len = 0; + pg.maxlen = 0; + + unsafe { Some(Vec::from_raw_parts(ptr, length, capacity)) } +} + +/// Store `Vec` in StringInfoData. +fn store_vec_u8(pg: &mut StringInfoData, vec: Vec) -> *mut ::std::os::raw::c_char { + let ptr = vec.as_ptr() as *mut ::std::os::raw::c_char; + let length = vec.len(); + let capacity = vec.capacity(); + + assert!(pg.data.is_null()); + + pg.data = ptr; + pg.len = length as i32; + pg.maxlen = capacity as i32; + + std::mem::forget(vec); + + ptr +} diff --git a/libs/walproposer/src/lib.rs b/libs/walproposer/src/lib.rs new file mode 100644 index 0000000000..f26a13458b --- /dev/null +++ b/libs/walproposer/src/lib.rs @@ -0,0 +1,14 @@ +pub mod bindings { + #![allow(non_upper_case_globals)] + #![allow(non_camel_case_types)] + #![allow(non_snake_case)] + // bindgen creates some unsafe code with no doc comments. + #![allow(clippy::missing_safety_doc)] + // noted at 1.63 that in many cases there's a u32 -> u32 transmutes in bindgen code. + #![allow(clippy::useless_transmute)] + + include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +} + +pub mod api_bindings; +pub mod walproposer; diff --git a/libs/walproposer/src/walproposer.rs b/libs/walproposer/src/walproposer.rs new file mode 100644 index 0000000000..4be15344c5 --- /dev/null +++ b/libs/walproposer/src/walproposer.rs @@ -0,0 +1,485 @@ +use std::ffi::CString; + +use postgres_ffi::WAL_SEGMENT_SIZE; +use utils::id::TenantTimelineId; + +use crate::{ + api_bindings::{create_api, take_vec_u8, Level}, + bindings::{ + Safekeeper, WalProposer, WalProposerConfig, WalProposerCreate, WalProposerFree, + WalProposerStart, + }, +}; + +/// Rust high-level wrapper for C walproposer API. Many methods are not required +/// for simple cases, hence todo!() in default implementations. +/// +/// Refer to `pgxn/neon/walproposer.h` for documentation. +pub trait ApiImpl { + fn get_shmem_state(&self) -> &mut crate::bindings::WalproposerShmemState { + todo!() + } + + fn start_streaming(&self, _startpos: u64) { + todo!() + } + + fn get_flush_rec_ptr(&self) -> u64 { + todo!() + } + + fn get_current_timestamp(&self) -> i64 { + todo!() + } + + fn conn_error_message(&self, _sk: &mut Safekeeper) -> String { + todo!() + } + + fn conn_status(&self, _sk: &mut Safekeeper) -> crate::bindings::WalProposerConnStatusType { + todo!() + } + + fn conn_connect_start(&self, _sk: &mut Safekeeper) { + todo!() + } + + fn conn_connect_poll( + &self, + _sk: &mut Safekeeper, + ) -> crate::bindings::WalProposerConnectPollStatusType { + todo!() + } + + fn conn_send_query(&self, _sk: &mut Safekeeper, _query: &str) -> bool { + todo!() + } + + fn conn_get_query_result( + &self, + _sk: &mut Safekeeper, + ) -> crate::bindings::WalProposerExecStatusType { + todo!() + } + + fn conn_flush(&self, _sk: &mut Safekeeper) -> i32 { + todo!() + } + + fn conn_finish(&self, _sk: &mut Safekeeper) { + todo!() + } + + fn conn_async_read(&self, _sk: &mut Safekeeper) -> (&[u8], crate::bindings::PGAsyncReadResult) { + todo!() + } + + fn conn_async_write( + &self, + _sk: &mut Safekeeper, + _buf: &[u8], + ) -> crate::bindings::PGAsyncWriteResult { + todo!() + } + + fn conn_blocking_write(&self, _sk: &mut Safekeeper, _buf: &[u8]) -> bool { + todo!() + } + + fn recovery_download(&self, _sk: &mut Safekeeper, _startpos: u64, _endpos: u64) -> bool { + todo!() + } + + fn wal_read(&self, _sk: &mut Safekeeper, _buf: &mut [u8], _startpos: u64) { + todo!() + } + + fn wal_reader_allocate(&self, _sk: &mut Safekeeper) { + todo!() + } + + fn free_event_set(&self, _wp: &mut WalProposer) { + todo!() + } + + fn init_event_set(&self, _wp: &mut WalProposer) { + todo!() + } + + fn update_event_set(&self, _sk: &mut Safekeeper, _events_mask: u32) { + todo!() + } + + fn add_safekeeper_event_set(&self, _sk: &mut Safekeeper, _events_mask: u32) { + todo!() + } + + fn wait_event_set(&self, _wp: &mut WalProposer, _timeout_millis: i64) -> WaitResult { + todo!() + } + + fn strong_random(&self, _buf: &mut [u8]) -> bool { + todo!() + } + + fn get_redo_start_lsn(&self) -> u64 { + todo!() + } + + fn finish_sync_safekeepers(&self, _lsn: u64) { + todo!() + } + + fn process_safekeeper_feedback(&self, _wp: &mut WalProposer, _commit_lsn: u64) { + todo!() + } + + fn confirm_wal_streamed(&self, _wp: &mut WalProposer, _lsn: u64) { + todo!() + } + + fn log_internal(&self, _wp: &mut WalProposer, _level: Level, _msg: &str) { + todo!() + } + + fn after_election(&self, _wp: &mut WalProposer) { + todo!() + } +} + +pub enum WaitResult { + Latch, + Timeout, + Network(*mut Safekeeper, u32), +} + +pub struct Config { + /// Tenant and timeline id + pub ttid: TenantTimelineId, + /// List of safekeepers in format `host:port` + pub safekeepers_list: Vec, + /// Safekeeper reconnect timeout in milliseconds + pub safekeeper_reconnect_timeout: i32, + /// Safekeeper connection timeout in milliseconds + pub safekeeper_connection_timeout: i32, + /// walproposer mode, finish when all safekeepers are synced or subscribe + /// to WAL streaming + pub sync_safekeepers: bool, +} + +/// WalProposer main struct. C methods are reexported as Rust functions. +pub struct Wrapper { + wp: *mut WalProposer, + _safekeepers_list_vec: Vec, +} + +impl Wrapper { + pub fn new(api: Box, config: Config) -> Wrapper { + let neon_tenant = CString::new(config.ttid.tenant_id.to_string()) + .unwrap() + .into_raw(); + let neon_timeline = CString::new(config.ttid.timeline_id.to_string()) + .unwrap() + .into_raw(); + + let mut safekeepers_list_vec = CString::new(config.safekeepers_list.join(",")) + .unwrap() + .into_bytes_with_nul(); + assert!(safekeepers_list_vec.len() == safekeepers_list_vec.capacity()); + let safekeepers_list = safekeepers_list_vec.as_mut_ptr() as *mut i8; + + let callback_data = Box::into_raw(Box::new(api)) as *mut ::std::os::raw::c_void; + + let c_config = WalProposerConfig { + neon_tenant, + neon_timeline, + safekeepers_list, + safekeeper_reconnect_timeout: config.safekeeper_reconnect_timeout, + safekeeper_connection_timeout: config.safekeeper_connection_timeout, + wal_segment_size: WAL_SEGMENT_SIZE as i32, // default 16MB + syncSafekeepers: config.sync_safekeepers, + systemId: 0, + pgTimeline: 1, + callback_data, + }; + let c_config = Box::into_raw(Box::new(c_config)); + + let api = create_api(); + let wp = unsafe { WalProposerCreate(c_config, api) }; + Wrapper { + wp, + _safekeepers_list_vec: safekeepers_list_vec, + } + } + + pub fn start(&self) { + unsafe { WalProposerStart(self.wp) } + } +} + +impl Drop for Wrapper { + fn drop(&mut self) { + unsafe { + let config = (*self.wp).config; + drop(Box::from_raw( + (*config).callback_data as *mut Box, + )); + drop(CString::from_raw((*config).neon_tenant)); + drop(CString::from_raw((*config).neon_timeline)); + drop(Box::from_raw(config)); + + for i in 0..(*self.wp).n_safekeepers { + let sk = &mut (*self.wp).safekeeper[i as usize]; + take_vec_u8(&mut sk.inbuf); + } + + WalProposerFree(self.wp); + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + cell::Cell, + sync::{atomic::AtomicUsize, mpsc::sync_channel}, + }; + + use utils::id::TenantTimelineId; + + use crate::{api_bindings::Level, walproposer::Wrapper}; + + use super::ApiImpl; + + #[derive(Clone, Copy, Debug)] + struct WaitEventsData { + sk: *mut crate::bindings::Safekeeper, + event_mask: u32, + } + + struct MockImpl { + // data to return from wait_event_set + wait_events: Cell, + // walproposer->safekeeper messages + expected_messages: Vec>, + expected_ptr: AtomicUsize, + // safekeeper->walproposer messages + safekeeper_replies: Vec>, + replies_ptr: AtomicUsize, + // channel to send LSN to the main thread + sync_channel: std::sync::mpsc::SyncSender, + } + + impl MockImpl { + fn check_walproposer_msg(&self, msg: &[u8]) { + let ptr = self + .expected_ptr + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + + if ptr >= self.expected_messages.len() { + panic!("unexpected message from walproposer"); + } + + let expected_msg = &self.expected_messages[ptr]; + assert_eq!(msg, expected_msg.as_slice()); + } + + fn next_safekeeper_reply(&self) -> &[u8] { + let ptr = self + .replies_ptr + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + + if ptr >= self.safekeeper_replies.len() { + panic!("no more safekeeper replies"); + } + + &self.safekeeper_replies[ptr] + } + } + + impl ApiImpl for MockImpl { + fn get_current_timestamp(&self) -> i64 { + println!("get_current_timestamp"); + 0 + } + + fn conn_status( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> crate::bindings::WalProposerConnStatusType { + println!("conn_status"); + crate::bindings::WalProposerConnStatusType_WP_CONNECTION_OK + } + + fn conn_connect_start(&self, _: &mut crate::bindings::Safekeeper) { + println!("conn_connect_start"); + } + + fn conn_connect_poll( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> crate::bindings::WalProposerConnectPollStatusType { + println!("conn_connect_poll"); + crate::bindings::WalProposerConnectPollStatusType_WP_CONN_POLLING_OK + } + + fn conn_send_query(&self, _: &mut crate::bindings::Safekeeper, query: &str) -> bool { + println!("conn_send_query: {}", query); + true + } + + fn conn_get_query_result( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> crate::bindings::WalProposerExecStatusType { + println!("conn_get_query_result"); + crate::bindings::WalProposerExecStatusType_WP_EXEC_SUCCESS_COPYBOTH + } + + fn conn_async_read( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> (&[u8], crate::bindings::PGAsyncReadResult) { + println!("conn_async_read"); + let reply = self.next_safekeeper_reply(); + println!("conn_async_read result: {:?}", reply); + ( + reply, + crate::bindings::PGAsyncReadResult_PG_ASYNC_READ_SUCCESS, + ) + } + + fn conn_blocking_write(&self, _: &mut crate::bindings::Safekeeper, buf: &[u8]) -> bool { + println!("conn_blocking_write: {:?}", buf); + self.check_walproposer_msg(buf); + true + } + + fn wal_reader_allocate(&self, _: &mut crate::bindings::Safekeeper) { + println!("wal_reader_allocate") + } + + fn free_event_set(&self, _: &mut crate::bindings::WalProposer) { + println!("free_event_set") + } + + fn init_event_set(&self, _: &mut crate::bindings::WalProposer) { + println!("init_event_set") + } + + fn update_event_set(&self, sk: &mut crate::bindings::Safekeeper, event_mask: u32) { + println!( + "update_event_set, sk={:?}, events_mask={:#b}", + sk as *mut crate::bindings::Safekeeper, event_mask + ); + self.wait_events.set(WaitEventsData { sk, event_mask }); + } + + fn add_safekeeper_event_set(&self, sk: &mut crate::bindings::Safekeeper, event_mask: u32) { + println!( + "add_safekeeper_event_set, sk={:?}, events_mask={:#b}", + sk as *mut crate::bindings::Safekeeper, event_mask + ); + self.wait_events.set(WaitEventsData { sk, event_mask }); + } + + fn wait_event_set( + &self, + _: &mut crate::bindings::WalProposer, + timeout_millis: i64, + ) -> super::WaitResult { + let data = self.wait_events.get(); + println!( + "wait_event_set, timeout_millis={}, res={:?}", + timeout_millis, data + ); + super::WaitResult::Network(data.sk, data.event_mask) + } + + fn strong_random(&self, buf: &mut [u8]) -> bool { + println!("strong_random"); + buf.fill(0); + true + } + + fn finish_sync_safekeepers(&self, lsn: u64) { + self.sync_channel.send(lsn).unwrap(); + panic!("sync safekeepers finished at lsn={}", lsn); + } + + fn log_internal(&self, _wp: &mut crate::bindings::WalProposer, level: Level, msg: &str) { + println!("walprop_log[{}] {}", level, msg); + } + + fn after_election(&self, _wp: &mut crate::bindings::WalProposer) { + println!("after_election"); + } + } + + /// Test that walproposer can successfully connect to safekeeper and finish + /// sync_safekeepers. API is mocked in MockImpl. + /// + /// Run this test with valgrind to detect leaks: + /// `valgrind --leak-check=full target/debug/deps/walproposer-` + #[test] + fn test_simple_sync_safekeepers() -> anyhow::Result<()> { + let ttid = TenantTimelineId::new( + "9e4c8f36063c6c6e93bc20d65a820f3d".parse()?, + "9e4c8f36063c6c6e93bc20d65a820f3d".parse()?, + ); + + let (sender, receiver) = sync_channel(1); + + let my_impl: Box = Box::new(MockImpl { + wait_events: Cell::new(WaitEventsData { + sk: std::ptr::null_mut(), + event_mask: 0, + }), + expected_messages: vec![ + // Greeting(ProposerGreeting { protocol_version: 2, pg_version: 160000, proposer_id: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], system_id: 0, timeline_id: 9e4c8f36063c6c6e93bc20d65a820f3d, tenant_id: 9e4c8f36063c6c6e93bc20d65a820f3d, tli: 1, wal_seg_size: 16777216 }) + vec![ + 103, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 113, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 76, 143, 54, 6, 60, 108, 110, + 147, 188, 32, 214, 90, 130, 15, 61, 158, 76, 143, 54, 6, 60, 108, 110, 147, + 188, 32, 214, 90, 130, 15, 61, 1, 0, 0, 0, 0, 0, 0, 1, + ], + // VoteRequest(VoteRequest { term: 3 }) + vec![ + 118, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + ], + expected_ptr: AtomicUsize::new(0), + safekeeper_replies: vec![ + // Greeting(AcceptorGreeting { term: 2, node_id: NodeId(1) }) + vec![ + 103, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + ], + // VoteResponse(VoteResponse { term: 3, vote_given: 1, flush_lsn: 0/539, truncate_lsn: 0/539, term_history: [(2, 0/539)], timeline_start_lsn: 0/539 }) + vec![ + 118, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 57, + 5, 0, 0, 0, 0, 0, 0, 57, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, + 0, 57, 5, 0, 0, 0, 0, 0, 0, 57, 5, 0, 0, 0, 0, 0, 0, + ], + ], + replies_ptr: AtomicUsize::new(0), + sync_channel: sender, + }); + let config = crate::walproposer::Config { + ttid, + safekeepers_list: vec!["localhost:5000".to_string()], + safekeeper_reconnect_timeout: 1000, + safekeeper_connection_timeout: 10000, + sync_safekeepers: true, + }; + + let wp = Wrapper::new(my_impl, config); + + // walproposer will panic when it finishes sync_safekeepers + std::panic::catch_unwind(|| wp.start()).unwrap_err(); + // validate the resulting LSN + assert_eq!(receiver.recv()?, 1337); + Ok(()) + // drop() will free up resources here + } +} diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index e88901ed78..84835843bc 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -23,6 +23,23 @@ EXTENSION = neon DATA = neon--1.0.sql PGFILEDESC = "neon - cloud storage for PostgreSQL" +EXTRA_CLEAN = \ + libwalproposer.a + +WALPROP_OBJS = \ + $(WIN32RES) \ + walproposer.o \ + neon_utils.o \ + walproposer_compat.o + +.PHONY: walproposer-lib +walproposer-lib: CPPFLAGS += -DWALPROPOSER_LIB +walproposer-lib: libwalproposer.a; + +.PHONY: libwalproposer.a +libwalproposer.a: $(WALPROP_OBJS) + rm -f $@ + $(AR) $(AROPT) $@ $^ PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index 10612e7e35..10544ba7a8 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -79,7 +79,7 @@ static int CompareLsn(const void *a, const void *b); static char *FormatSafekeeperState(SafekeeperState state); static void AssertEventsOkForState(uint32 events, Safekeeper *sk); static uint32 SafekeeperStateDesiredEvents(SafekeeperState state); -static char *FormatEvents(uint32 events); +static char *FormatEvents(WalProposer *wp, uint32 events); WalProposer * WalProposerCreate(WalProposerConfig *config, walproposer_api api) @@ -98,7 +98,7 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) port = strchr(host, ':'); if (port == NULL) { - elog(FATAL, "port is not specified"); + walprop_log(FATAL, "port is not specified"); } *port++ = '\0'; sep = strchr(port, ','); @@ -106,12 +106,11 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) *sep++ = '\0'; if (wp->n_safekeepers + 1 >= MAX_SAFEKEEPERS) { - elog(FATAL, "Too many safekeepers"); + walprop_log(FATAL, "Too many safekeepers"); } wp->safekeeper[wp->n_safekeepers].host = host; wp->safekeeper[wp->n_safekeepers].port = port; wp->safekeeper[wp->n_safekeepers].state = SS_OFFLINE; - wp->safekeeper[wp->n_safekeepers].conn = NULL; wp->safekeeper[wp->n_safekeepers].wp = wp; { @@ -122,13 +121,11 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) "host=%s port=%s dbname=replication options='-c timeline_id=%s tenant_id=%s'", sk->host, sk->port, wp->config->neon_timeline, wp->config->neon_tenant); if (written > MAXCONNINFO || written < 0) - elog(FATAL, "could not create connection string for safekeeper %s:%s", sk->host, sk->port); + walprop_log(FATAL, "could not create connection string for safekeeper %s:%s", sk->host, sk->port); } initStringInfo(&wp->safekeeper[wp->n_safekeepers].outbuf); - wp->safekeeper[wp->n_safekeepers].xlogreader = wp->api.wal_reader_allocate(); - if (wp->safekeeper[wp->n_safekeepers].xlogreader == NULL) - elog(FATAL, "Failed to allocate xlog reader"); + wp->api.wal_reader_allocate(&wp->safekeeper[wp->n_safekeepers]); wp->safekeeper[wp->n_safekeepers].flushWrite = false; wp->safekeeper[wp->n_safekeepers].startStreamingAt = InvalidXLogRecPtr; wp->safekeeper[wp->n_safekeepers].streamingAt = InvalidXLogRecPtr; @@ -136,7 +133,7 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) } if (wp->n_safekeepers < 1) { - elog(FATAL, "Safekeepers addresses are not specified"); + walprop_log(FATAL, "Safekeepers addresses are not specified"); } wp->quorum = wp->n_safekeepers / 2 + 1; @@ -144,27 +141,47 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) wp->greetRequest.tag = 'g'; wp->greetRequest.protocolVersion = SK_PROTOCOL_VERSION; wp->greetRequest.pgVersion = PG_VERSION_NUM; - wp->api.strong_random(&wp->greetRequest.proposerId, sizeof(wp->greetRequest.proposerId)); + wp->api.strong_random(wp, &wp->greetRequest.proposerId, sizeof(wp->greetRequest.proposerId)); wp->greetRequest.systemId = wp->config->systemId; if (!wp->config->neon_timeline) - elog(FATAL, "neon.timeline_id is not provided"); + walprop_log(FATAL, "neon.timeline_id is not provided"); if (*wp->config->neon_timeline != '\0' && !HexDecodeString(wp->greetRequest.timeline_id, wp->config->neon_timeline, 16)) - elog(FATAL, "Could not parse neon.timeline_id, %s", wp->config->neon_timeline); + walprop_log(FATAL, "Could not parse neon.timeline_id, %s", wp->config->neon_timeline); if (!wp->config->neon_tenant) - elog(FATAL, "neon.tenant_id is not provided"); + walprop_log(FATAL, "neon.tenant_id is not provided"); if (*wp->config->neon_tenant != '\0' && !HexDecodeString(wp->greetRequest.tenant_id, wp->config->neon_tenant, 16)) - elog(FATAL, "Could not parse neon.tenant_id, %s", wp->config->neon_tenant); + walprop_log(FATAL, "Could not parse neon.tenant_id, %s", wp->config->neon_tenant); - wp->greetRequest.timeline = wp->api.get_timeline_id(); + wp->greetRequest.timeline = wp->config->pgTimeline; wp->greetRequest.walSegSize = wp->config->wal_segment_size; - wp->api.init_event_set(wp->n_safekeepers); + wp->api.init_event_set(wp); return wp; } +void +WalProposerFree(WalProposer *wp) +{ + for (int i = 0; i < wp->n_safekeepers; i++) + { + Safekeeper *sk = &wp->safekeeper[i]; + + Assert(sk->outbuf.data != NULL); + pfree(sk->outbuf.data); + if (sk->voteResponse.termHistory.entries) + pfree(sk->voteResponse.termHistory.entries); + sk->voteResponse.termHistory.entries = NULL; + } + if (wp->propTermHistory.entries != NULL) + pfree(wp->propTermHistory.entries); + wp->propTermHistory.entries = NULL; + + pfree(wp); +} + /* * Create new AppendRequest message and start sending it. This function is * called from walsender every time the new WAL is available. @@ -190,10 +207,10 @@ WalProposerPoll(WalProposer *wp) Safekeeper *sk = NULL; int rc = 0; uint32 events = 0; - TimestampTz now = wp->api.get_current_timestamp(); + TimestampTz now = wp->api.get_current_timestamp(wp); long timeout = TimeToReconnect(wp, now); - rc = wp->api.wait_event_set(timeout, &sk, &events); + rc = wp->api.wait_event_set(wp, timeout, &sk, &events); /* Exit loop if latch is set (we got new WAL) */ if ((rc == 1 && events & WL_LATCH_SET)) @@ -224,14 +241,14 @@ WalProposerPoll(WalProposer *wp) */ if (!wp->config->syncSafekeepers) { - XLogRecPtr flushed = wp->api.get_flush_rec_ptr(); + XLogRecPtr flushed = wp->api.get_flush_rec_ptr(wp); if (flushed > wp->availableLsn) break; } } - now = wp->api.get_current_timestamp(); + now = wp->api.get_current_timestamp(wp); /* timeout expired: poll state */ if (rc == 0 || TimeToReconnect(wp, now) <= 0) { @@ -249,7 +266,7 @@ WalProposerPoll(WalProposer *wp) /* * Abandon connection attempts which take too long. */ - now = wp->api.get_current_timestamp(); + now = wp->api.get_current_timestamp(wp); for (int i = 0; i < wp->n_safekeepers; i++) { Safekeeper *sk = &wp->safekeeper[i]; @@ -257,7 +274,7 @@ WalProposerPoll(WalProposer *wp) if (TimestampDifferenceExceeds(sk->latestMsgReceivedAt, now, wp->config->safekeeper_connection_timeout)) { - elog(WARNING, "terminating connection to safekeeper '%s:%s' in '%s' state: no messages received during the last %dms or connection attempt took longer than that", + walprop_log(WARNING, "terminating connection to safekeeper '%s:%s' in '%s' state: no messages received during the last %dms or connection attempt took longer than that", sk->host, sk->port, FormatSafekeeperState(sk->state), wp->config->safekeeper_connection_timeout); ShutdownConnection(sk); } @@ -296,10 +313,10 @@ HackyRemoveWalProposerEvent(Safekeeper *to_remove) { WalProposer *wp = to_remove->wp; - /* Remove the existing event set */ - wp->api.free_event_set(); + /* Remove the existing event set, assign sk->eventPos = -1 */ + wp->api.free_event_set(wp); /* Re-initialize it without adding any safekeeper events */ - wp->api.init_event_set(wp->n_safekeepers); + wp->api.init_event_set(wp); /* * loop through the existing safekeepers. If they aren't the one we're @@ -311,13 +328,11 @@ HackyRemoveWalProposerEvent(Safekeeper *to_remove) uint32 desired_events = WL_NO_EVENTS; Safekeeper *sk = &wp->safekeeper[i]; - sk->eventPos = -1; - if (sk == to_remove) continue; /* If this safekeeper isn't offline, add an event for it! */ - if (sk->conn != NULL) + if (sk->state != SS_OFFLINE) { desired_events = SafekeeperStateDesiredEvents(sk->state); /* will set sk->eventPos */ @@ -330,9 +345,7 @@ HackyRemoveWalProposerEvent(Safekeeper *to_remove) static void ShutdownConnection(Safekeeper *sk) { - if (sk->conn) - sk->wp->api.conn_finish(sk->conn); - sk->conn = NULL; + sk->wp->api.conn_finish(sk); sk->state = SS_OFFLINE; sk->flushWrite = false; sk->streamingAt = InvalidXLogRecPtr; @@ -361,23 +374,16 @@ ResetConnection(Safekeeper *sk) } /* - * Try to establish new connection + * Try to establish new connection, it will update sk->conn. */ - sk->conn = wp->api.conn_connect_start((char *) &sk->conninfo); - - /* - * "If the result is null, then libpq has been unable to allocate a new - * PGconn structure" - */ - if (!sk->conn) - elog(FATAL, "failed to allocate new PGconn object"); + wp->api.conn_connect_start(sk); /* * PQconnectStart won't actually start connecting until we run * PQconnectPoll. Before we do that though, we need to check that it * didn't immediately fail. */ - if (wp->api.conn_status(sk->conn) == WP_CONNECTION_BAD) + if (wp->api.conn_status(sk) == WP_CONNECTION_BAD) { /*--- * According to libpq docs: @@ -388,15 +394,14 @@ ResetConnection(Safekeeper *sk) * * https://www.postgresql.org/docs/devel/libpq-connect.html#LIBPQ-PQCONNECTSTARTPARAMS */ - elog(WARNING, "Immediate failure to connect with node '%s:%s':\n\terror: %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "Immediate failure to connect with node '%s:%s':\n\terror: %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); /* * Even though the connection failed, we still need to clean up the * object */ - wp->api.conn_finish(sk->conn); - sk->conn = NULL; + wp->api.conn_finish(sk); return; } @@ -413,10 +418,10 @@ ResetConnection(Safekeeper *sk) * (see libpqrcv_connect, defined in * src/backend/replication/libpqwalreceiver/libpqwalreceiver.c) */ - elog(LOG, "connecting with node %s:%s", sk->host, sk->port); + walprop_log(LOG, "connecting with node %s:%s", sk->host, sk->port); sk->state = SS_CONNECTING_WRITE; - sk->latestMsgReceivedAt = wp->api.get_current_timestamp(); + sk->latestMsgReceivedAt = wp->api.get_current_timestamp(wp); wp->api.add_safekeeper_event_set(sk, WL_SOCKET_WRITEABLE); return; @@ -447,7 +452,7 @@ TimeToReconnect(WalProposer *wp, TimestampTz now) static void ReconnectSafekeepers(WalProposer *wp) { - TimestampTz now = wp->api.get_current_timestamp(); + TimestampTz now = wp->api.get_current_timestamp(wp); if (TimeToReconnect(wp, now) == 0) { @@ -467,6 +472,8 @@ ReconnectSafekeepers(WalProposer *wp) static void AdvancePollState(Safekeeper *sk, uint32 events) { + WalProposer *wp = sk->wp; + /* * Sanity check. We assume further down that the operations don't block * because the socket is ready. @@ -481,7 +488,7 @@ AdvancePollState(Safekeeper *sk, uint32 events) * ResetConnection */ case SS_OFFLINE: - elog(FATAL, "Unexpected safekeeper %s:%s state advancement: is offline", + walprop_log(FATAL, "Unexpected safekeeper %s:%s state advancement: is offline", sk->host, sk->port); break; /* actually unreachable, but prevents * -Wimplicit-fallthrough */ @@ -517,7 +524,7 @@ AdvancePollState(Safekeeper *sk, uint32 events) * requests. */ case SS_VOTING: - elog(WARNING, "EOF from node %s:%s in %s state", sk->host, + walprop_log(WARNING, "EOF from node %s:%s in %s state", sk->host, sk->port, FormatSafekeeperState(sk->state)); ResetConnection(sk); return; @@ -546,7 +553,7 @@ AdvancePollState(Safekeeper *sk, uint32 events) * Idle state for waiting votes from quorum. */ case SS_IDLE: - elog(WARNING, "EOF from node %s:%s in %s state", sk->host, + walprop_log(WARNING, "EOF from node %s:%s in %s state", sk->host, sk->port, FormatSafekeeperState(sk->state)); ResetConnection(sk); return; @@ -564,7 +571,7 @@ static void HandleConnectionEvent(Safekeeper *sk) { WalProposer *wp = sk->wp; - WalProposerConnectPollStatusType result = wp->api.conn_connect_poll(sk->conn); + WalProposerConnectPollStatusType result = wp->api.conn_connect_poll(sk); /* The new set of events we'll wait on, after updating */ uint32 new_events = WL_NO_EVENTS; @@ -572,9 +579,9 @@ HandleConnectionEvent(Safekeeper *sk) switch (result) { case WP_CONN_POLLING_OK: - elog(LOG, "connected with node %s:%s", sk->host, + walprop_log(LOG, "connected with node %s:%s", sk->host, sk->port); - sk->latestMsgReceivedAt = wp->api.get_current_timestamp(); + sk->latestMsgReceivedAt = wp->api.get_current_timestamp(wp); /* * We have to pick some event to update event set. We'll @@ -596,8 +603,8 @@ HandleConnectionEvent(Safekeeper *sk) break; case WP_CONN_POLLING_FAILED: - elog(WARNING, "failed to connect to node '%s:%s': %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "failed to connect to node '%s:%s': %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); /* * If connecting failed, we don't want to restart the connection @@ -631,10 +638,10 @@ SendStartWALPush(Safekeeper *sk) { WalProposer *wp = sk->wp; - if (!wp->api.conn_send_query(sk->conn, "START_WAL_PUSH")) + if (!wp->api.conn_send_query(sk, "START_WAL_PUSH")) { - elog(WARNING, "Failed to send 'START_WAL_PUSH' query to safekeeper %s:%s: %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "Failed to send 'START_WAL_PUSH' query to safekeeper %s:%s: %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); ShutdownConnection(sk); return; } @@ -647,7 +654,7 @@ RecvStartWALPushResult(Safekeeper *sk) { WalProposer *wp = sk->wp; - switch (wp->api.conn_get_query_result(sk->conn)) + switch (wp->api.conn_get_query_result(sk)) { /* * Successful result, move on to starting the handshake @@ -670,8 +677,8 @@ RecvStartWALPushResult(Safekeeper *sk) break; case WP_EXEC_FAILED: - elog(WARNING, "Failed to send query to safekeeper %s:%s: %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "Failed to send query to safekeeper %s:%s: %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); ShutdownConnection(sk); return; @@ -681,7 +688,7 @@ RecvStartWALPushResult(Safekeeper *sk) * wrong" */ case WP_EXEC_UNEXPECTED_SUCCESS: - elog(WARNING, "Received bad response from safekeeper %s:%s query execution", + walprop_log(WARNING, "Received bad response from safekeeper %s:%s query execution", sk->host, sk->port); ShutdownConnection(sk); return; @@ -717,7 +724,7 @@ RecvAcceptorGreeting(Safekeeper *sk) if (!AsyncReadMessage(sk, (AcceptorProposerMessage *) &sk->greetResponse)) return; - elog(LOG, "received AcceptorGreeting from safekeeper %s:%s", sk->host, sk->port); + walprop_log(LOG, "received AcceptorGreeting from safekeeper %s:%s", sk->host, sk->port); /* Protocol is all good, move to voting. */ sk->state = SS_VOTING; @@ -737,7 +744,7 @@ RecvAcceptorGreeting(Safekeeper *sk) if (wp->n_connected == wp->quorum) { wp->propTerm++; - elog(LOG, "proposer connected to quorum (%d) safekeepers, propTerm=" INT64_FORMAT, wp->quorum, wp->propTerm); + walprop_log(LOG, "proposer connected to quorum (%d) safekeepers, propTerm=" INT64_FORMAT, wp->quorum, wp->propTerm); wp->voteRequest = (VoteRequest) { @@ -750,7 +757,7 @@ RecvAcceptorGreeting(Safekeeper *sk) else if (sk->greetResponse.term > wp->propTerm) { /* Another compute with higher term is running. */ - elog(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", + walprop_log(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", sk->host, sk->port, sk->greetResponse.term, wp->propTerm); } @@ -792,7 +799,7 @@ SendVoteRequest(Safekeeper *sk) WalProposer *wp = sk->wp; /* We have quorum for voting, send our vote request */ - elog(LOG, "requesting vote from %s:%s for term " UINT64_FORMAT, sk->host, sk->port, wp->voteRequest.term); + walprop_log(LOG, "requesting vote from %s:%s for term " UINT64_FORMAT, sk->host, sk->port, wp->voteRequest.term); /* On failure, logging & resetting is handled */ if (!BlockingWrite(sk, &wp->voteRequest, sizeof(wp->voteRequest), SS_WAIT_VERDICT)) return; @@ -809,7 +816,7 @@ RecvVoteResponse(Safekeeper *sk) if (!AsyncReadMessage(sk, (AcceptorProposerMessage *) &sk->voteResponse)) return; - elog(LOG, + walprop_log(LOG, "got VoteResponse from acceptor %s:%s, voteGiven=" UINT64_FORMAT ", epoch=" UINT64_FORMAT ", flushLsn=%X/%X, truncateLsn=%X/%X, timelineStartLsn=%X/%X", sk->host, sk->port, sk->voteResponse.voteGiven, GetHighestTerm(&sk->voteResponse.termHistory), LSN_FORMAT_ARGS(sk->voteResponse.flushLsn), @@ -824,7 +831,7 @@ RecvVoteResponse(Safekeeper *sk) if ((!sk->voteResponse.voteGiven) && (sk->voteResponse.term > wp->propTerm || wp->n_votes < wp->quorum)) { - elog(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", + walprop_log(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", sk->host, sk->port, sk->voteResponse.term, wp->propTerm); } @@ -861,49 +868,27 @@ RecvVoteResponse(Safekeeper *sk) static void HandleElectedProposer(WalProposer *wp) { - FILE* f; - XLogRecPtr lrRestartLsn; - DetermineEpochStartLsn(wp); - /* - * If there are active logical replication subscription we need - * to provide enough WAL for their WAL senders based on th position - * of their replication slots. - */ - f = fopen("restart.lsn", "rb"); - if (f != NULL && !wp->config->syncSafekeepers) - { - fread(&lrRestartLsn, sizeof(lrRestartLsn), 1, f); - fclose(f); - if (lrRestartLsn != InvalidXLogRecPtr) - { - elog(LOG, "Logical replication restart LSN %X/%X", LSN_FORMAT_ARGS(lrRestartLsn)); - /* start from the beginning of the segment to fetch page headers verifed by XLogReader */ - lrRestartLsn = lrRestartLsn - XLogSegmentOffset(lrRestartLsn, wal_segment_size); - wp->truncateLsn = Min(wp->truncateLsn, lrRestartLsn); - } - } - /* * Check if not all safekeepers are up-to-date, we need to download WAL * needed to synchronize them */ if (wp->truncateLsn < wp->propEpochStartLsn) { - elog(LOG, + walprop_log(LOG, "start recovery because truncateLsn=%X/%X is not " "equal to epochStartLsn=%X/%X", LSN_FORMAT_ARGS(wp->truncateLsn), LSN_FORMAT_ARGS(wp->propEpochStartLsn)); /* Perform recovery */ if (!wp->api.recovery_download(&wp->safekeeper[wp->donor], wp->greetRequest.timeline, wp->truncateLsn, wp->propEpochStartLsn)) - elog(FATAL, "Failed to recover state"); + walprop_log(FATAL, "Failed to recover state"); } else if (wp->config->syncSafekeepers) { /* Sync is not needed: just exit */ - wp->api.finish_sync_safekeepers(wp->propEpochStartLsn); + wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); /* unreachable */ } @@ -1004,7 +989,7 @@ DetermineEpochStartLsn(WalProposer *wp) if (wp->timelineStartLsn != InvalidXLogRecPtr && wp->timelineStartLsn != wp->safekeeper[i].voteResponse.timelineStartLsn) { - elog(WARNING, + walprop_log(WARNING, "inconsistent timelineStartLsn: current %X/%X, received %X/%X", LSN_FORMAT_ARGS(wp->timelineStartLsn), LSN_FORMAT_ARGS(wp->safekeeper[i].voteResponse.timelineStartLsn)); @@ -1020,12 +1005,12 @@ DetermineEpochStartLsn(WalProposer *wp) */ if (wp->propEpochStartLsn == InvalidXLogRecPtr && !wp->config->syncSafekeepers) { - wp->propEpochStartLsn = wp->truncateLsn = wp->api.get_redo_start_lsn(); + wp->propEpochStartLsn = wp->truncateLsn = wp->api.get_redo_start_lsn(wp); if (wp->timelineStartLsn == InvalidXLogRecPtr) { - wp->timelineStartLsn = wp->api.get_redo_start_lsn(); + wp->timelineStartLsn = wp->api.get_redo_start_lsn(wp); } - elog(LOG, "bumped epochStartLsn to the first record %X/%X", LSN_FORMAT_ARGS(wp->propEpochStartLsn)); + walprop_log(LOG, "bumped epochStartLsn to the first record %X/%X", LSN_FORMAT_ARGS(wp->propEpochStartLsn)); } /* @@ -1052,7 +1037,7 @@ DetermineEpochStartLsn(WalProposer *wp) wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].term = wp->propTerm; wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].lsn = wp->propEpochStartLsn; - elog(LOG, "got votes from majority (%d) of nodes, term " UINT64_FORMAT ", epochStartLsn %X/%X, donor %s:%s, truncate_lsn %X/%X", + walprop_log(LOG, "got votes from majority (%d) of nodes, term " UINT64_FORMAT ", epochStartLsn %X/%X, donor %s:%s, truncate_lsn %X/%X", wp->quorum, wp->propTerm, LSN_FORMAT_ARGS(wp->propEpochStartLsn), @@ -1066,7 +1051,7 @@ DetermineEpochStartLsn(WalProposer *wp) */ if (!wp->config->syncSafekeepers) { - WalproposerShmemState *walprop_shared = wp->api.get_shmem_state(); + WalproposerShmemState *walprop_shared = wp->api.get_shmem_state(wp); /* * Basebackup LSN always points to the beginning of the record (not @@ -1074,7 +1059,7 @@ DetermineEpochStartLsn(WalProposer *wp) * Safekeepers don't skip header as they need continious stream of * data, so correct LSN for comparison. */ - if (SkipXLogPageHeader(wp, wp->propEpochStartLsn) != wp->api.get_redo_start_lsn()) + if (SkipXLogPageHeader(wp, wp->propEpochStartLsn) != wp->api.get_redo_start_lsn(wp)) { /* * However, allow to proceed if previously elected leader was me; @@ -1084,14 +1069,21 @@ DetermineEpochStartLsn(WalProposer *wp) if (!((dth->n_entries >= 1) && (dth->entries[dth->n_entries - 1].term == walprop_shared->mineLastElectedTerm))) { - elog(PANIC, + walprop_log(PANIC, "collected propEpochStartLsn %X/%X, but basebackup LSN %X/%X", LSN_FORMAT_ARGS(wp->propEpochStartLsn), - LSN_FORMAT_ARGS(wp->api.get_redo_start_lsn())); + LSN_FORMAT_ARGS(wp->api.get_redo_start_lsn(wp))); } } walprop_shared->mineLastElectedTerm = wp->propTerm; } + + /* + * WalProposer has just elected itself and initialized history, so + * we can call election callback. Usually it updates truncateLsn to + * fetch WAL for logical replication. + */ + wp->api.after_election(wp); } /* @@ -1162,7 +1154,7 @@ SendProposerElected(Safekeeper *sk) */ sk->startStreamingAt = wp->truncateLsn; - elog(WARNING, "empty safekeeper joined cluster as %s:%s, historyStart=%X/%X, sk->startStreamingAt=%X/%X", + walprop_log(WARNING, "empty safekeeper joined cluster as %s:%s, historyStart=%X/%X, sk->startStreamingAt=%X/%X", sk->host, sk->port, LSN_FORMAT_ARGS(wp->propTermHistory.entries[0].lsn), LSN_FORMAT_ARGS(sk->startStreamingAt)); } @@ -1197,7 +1189,7 @@ SendProposerElected(Safekeeper *sk) msg.timelineStartLsn = wp->timelineStartLsn; lastCommonTerm = i >= 0 ? wp->propTermHistory.entries[i].term : 0; - elog(LOG, + walprop_log(LOG, "sending elected msg to node " UINT64_FORMAT " term=" UINT64_FORMAT ", startStreamingAt=%X/%X (lastCommonTerm=" UINT64_FORMAT "), termHistory.n_entries=%u to %s:%s, timelineStartLsn=%X/%X", sk->greetResponse.nodeId, msg.term, LSN_FORMAT_ARGS(msg.startStreamingAt), lastCommonTerm, msg.termHistory->n_entries, sk->host, sk->port, LSN_FORMAT_ARGS(msg.timelineStartLsn)); @@ -1362,13 +1354,12 @@ SendAppendRequests(Safekeeper *sk) req = &sk->appendRequest; PrepareAppendRequest(sk->wp, &sk->appendRequest, sk->streamingAt, endLsn); - ereport(DEBUG2, - (errmsg("sending message len %ld beginLsn=%X/%X endLsn=%X/%X commitLsn=%X/%X truncateLsn=%X/%X to %s:%s", + walprop_log(DEBUG2, "sending message len %ld beginLsn=%X/%X endLsn=%X/%X commitLsn=%X/%X truncateLsn=%X/%X to %s:%s", req->endLsn - req->beginLsn, LSN_FORMAT_ARGS(req->beginLsn), LSN_FORMAT_ARGS(req->endLsn), LSN_FORMAT_ARGS(req->commitLsn), - LSN_FORMAT_ARGS(wp->truncateLsn), sk->host, sk->port))); + LSN_FORMAT_ARGS(wp->truncateLsn), sk->host, sk->port); resetStringInfo(&sk->outbuf); @@ -1378,13 +1369,13 @@ SendAppendRequests(Safekeeper *sk) /* write the WAL itself */ enlargeStringInfo(&sk->outbuf, req->endLsn - req->beginLsn); /* wal_read will raise error on failure */ - wp->api.wal_read(sk->xlogreader, + wp->api.wal_read(sk, &sk->outbuf.data[sk->outbuf.len], req->beginLsn, req->endLsn - req->beginLsn); sk->outbuf.len += req->endLsn - req->beginLsn; - writeResult = wp->api.conn_async_write(sk->conn, sk->outbuf.data, sk->outbuf.len); + writeResult = wp->api.conn_async_write(sk, sk->outbuf.data, sk->outbuf.len); /* Mark current message as sent, whatever the result is */ sk->streamingAt = endLsn; @@ -1406,9 +1397,9 @@ SendAppendRequests(Safekeeper *sk) return true; case PG_ASYNC_WRITE_FAIL: - elog(WARNING, "Failed to send to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to send to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; default: @@ -1446,17 +1437,16 @@ RecvAppendResponses(Safekeeper *sk) if (!AsyncReadMessage(sk, (AcceptorProposerMessage *) &sk->appendResponse)) break; - ereport(DEBUG2, - (errmsg("received message term=" INT64_FORMAT " flushLsn=%X/%X commitLsn=%X/%X from %s:%s", + walprop_log(DEBUG2, "received message term=" INT64_FORMAT " flushLsn=%X/%X commitLsn=%X/%X from %s:%s", sk->appendResponse.term, LSN_FORMAT_ARGS(sk->appendResponse.flushLsn), LSN_FORMAT_ARGS(sk->appendResponse.commitLsn), - sk->host, sk->port))); + sk->host, sk->port); if (sk->appendResponse.term > wp->propTerm) { /* Another compute with higher term is running. */ - elog(PANIC, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT "", + walprop_log(PANIC, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT "", sk->host, sk->port, sk->appendResponse.term, wp->propTerm); } @@ -1484,7 +1474,7 @@ RecvAppendResponses(Safekeeper *sk) /* Parse a PageserverFeedback message, or the PageserverFeedback part of an AppendResponse */ void -ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) +ParsePageserverFeedbackMessage(WalProposer *wp, StringInfo reply_message, PageserverFeedback *rf) { uint8 nkeys; int i; @@ -1502,7 +1492,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->currentClusterSize = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: current_timeline_size %lu", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: current_timeline_size %lu", rf->currentClusterSize); } else if ((strcmp(key, "ps_writelsn") == 0) || (strcmp(key, "last_received_lsn") == 0)) @@ -1510,7 +1500,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->last_received_lsn = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: last_received_lsn %X/%X", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: last_received_lsn %X/%X", LSN_FORMAT_ARGS(rf->last_received_lsn)); } else if ((strcmp(key, "ps_flushlsn") == 0) || (strcmp(key, "disk_consistent_lsn") == 0)) @@ -1518,7 +1508,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->disk_consistent_lsn = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: disk_consistent_lsn %X/%X", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: disk_consistent_lsn %X/%X", LSN_FORMAT_ARGS(rf->disk_consistent_lsn)); } else if ((strcmp(key, "ps_applylsn") == 0) || (strcmp(key, "remote_consistent_lsn") == 0)) @@ -1526,7 +1516,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->remote_consistent_lsn = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: remote_consistent_lsn %X/%X", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: remote_consistent_lsn %X/%X", LSN_FORMAT_ARGS(rf->remote_consistent_lsn)); } else if ((strcmp(key, "ps_replytime") == 0) || (strcmp(key, "replytime") == 0)) @@ -1539,7 +1529,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) /* Copy because timestamptz_to_str returns a static buffer */ replyTimeStr = pstrdup(timestamptz_to_str(rf->replytime)); - elog(DEBUG2, "ParsePageserverFeedbackMessage: replytime %lu reply_time: %s", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: replytime %lu reply_time: %s", rf->replytime, replyTimeStr); pfree(replyTimeStr); @@ -1554,7 +1544,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) * Skip unknown keys to support backward compatibile protocol * changes */ - elog(LOG, "ParsePageserverFeedbackMessage: unknown key: %s len %d", key, len); + walprop_log(LOG, "ParsePageserverFeedbackMessage: unknown key: %s len %d", key, len); pq_getmsgbytes(reply_message, len); }; } @@ -1637,7 +1627,7 @@ HandleSafekeeperResponse(WalProposer *wp) * Advance the replication slot to free up old WAL files. Note that * slot doesn't exist if we are in syncSafekeepers mode. */ - wp->api.confirm_wal_streamed(wp->truncateLsn); + wp->api.confirm_wal_streamed(wp, wp->truncateLsn); } /* @@ -1684,7 +1674,7 @@ HandleSafekeeperResponse(WalProposer *wp) */ BroadcastAppendRequest(wp); - wp->api.finish_sync_safekeepers(wp->propEpochStartLsn); + wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); /* unreachable */ } } @@ -1699,7 +1689,7 @@ AsyncRead(Safekeeper *sk, char **buf, int *buf_size) { WalProposer *wp = sk->wp; - switch (wp->api.conn_async_read(sk->conn, buf, buf_size)) + switch (wp->api.conn_async_read(sk, buf, buf_size)) { case PG_ASYNC_READ_SUCCESS: return true; @@ -1709,9 +1699,9 @@ AsyncRead(Safekeeper *sk, char **buf, int *buf_size) return false; case PG_ASYNC_READ_FAIL: - elog(WARNING, "Failed to read from node %s:%s in %s state: %s", sk->host, + walprop_log(WARNING, "Failed to read from node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; } @@ -1749,12 +1739,12 @@ AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg) tag = pq_getmsgint64_le(&s); if (tag != anymsg->tag) { - elog(WARNING, "unexpected message tag %c from node %s:%s in state %s", (char) tag, sk->host, + walprop_log(WARNING, "unexpected message tag %c from node %s:%s in state %s", (char) tag, sk->host, sk->port, FormatSafekeeperState(sk->state)); ResetConnection(sk); return false; } - sk->latestMsgReceivedAt = wp->api.get_current_timestamp(); + sk->latestMsgReceivedAt = wp->api.get_current_timestamp(wp); switch (tag) { case 'g': @@ -1798,7 +1788,7 @@ AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg) msg->hs.xmin.value = pq_getmsgint64_le(&s); msg->hs.catalog_xmin.value = pq_getmsgint64_le(&s); if (buf_size > APPENDRESPONSE_FIXEDPART_SIZE) - ParsePageserverFeedbackMessage(&s, &msg->rf); + ParsePageserverFeedbackMessage(wp, &s, &msg->rf); pq_getmsgend(&s); return true; } @@ -1823,11 +1813,11 @@ BlockingWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState succes WalProposer *wp = sk->wp; uint32 events; - if (!wp->api.conn_blocking_write(sk->conn, msg, msg_size)) + if (!wp->api.conn_blocking_write(sk, msg, msg_size)) { - elog(WARNING, "Failed to send to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to send to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; } @@ -1857,7 +1847,7 @@ AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_sta { WalProposer *wp = sk->wp; - switch (wp->api.conn_async_write(sk->conn, msg, msg_size)) + switch (wp->api.conn_async_write(sk, msg, msg_size)) { case PG_ASYNC_WRITE_SUCCESS: return true; @@ -1872,9 +1862,9 @@ AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_sta wp->api.update_event_set(sk, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); return false; case PG_ASYNC_WRITE_FAIL: - elog(WARNING, "Failed to send to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to send to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; default: @@ -1902,7 +1892,7 @@ AsyncFlush(Safekeeper *sk) * 1 if unable to send everything yet [call PQflush again] * -1 if it failed [emit an error] */ - switch (wp->api.conn_flush(sk->conn)) + switch (wp->api.conn_flush(sk)) { case 0: /* flush is done */ @@ -1911,9 +1901,9 @@ AsyncFlush(Safekeeper *sk) /* Nothing to do; try again when the socket's ready */ return false; case -1: - elog(WARNING, "Failed to flush write to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to flush write to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ResetConnection(sk); return false; default: @@ -1942,11 +1932,11 @@ CompareLsn(const void *a, const void *b) * * The strings are intended to be used as a prefix to "state", e.g.: * - * elog(LOG, "currently in %s state", FormatSafekeeperState(sk->state)); + * walprop_log(LOG, "currently in %s state", FormatSafekeeperState(sk->state)); * * If this sort of phrasing doesn't fit the message, instead use something like: * - * elog(LOG, "currently in state [%s]", FormatSafekeeperState(sk->state)); + * walprop_log(LOG, "currently in state [%s]", FormatSafekeeperState(sk->state)); */ static char * FormatSafekeeperState(SafekeeperState state) @@ -1994,6 +1984,7 @@ FormatSafekeeperState(SafekeeperState state) static void AssertEventsOkForState(uint32 events, Safekeeper *sk) { + WalProposer *wp = sk->wp; uint32 expected = SafekeeperStateDesiredEvents(sk->state); /* @@ -2016,8 +2007,8 @@ AssertEventsOkForState(uint32 events, Safekeeper *sk) * To give a descriptive message in the case of failure, we use elog * and then an assertion that's guaranteed to fail. */ - elog(WARNING, "events %s mismatched for safekeeper %s:%s in state [%s]", - FormatEvents(events), sk->host, sk->port, FormatSafekeeperState(sk->state)); + walprop_log(WARNING, "events %s mismatched for safekeeper %s:%s in state [%s]", + FormatEvents(wp, events), sk->host, sk->port, FormatSafekeeperState(sk->state)); Assert(events_ok_for_state); } } @@ -2090,7 +2081,7 @@ SafekeeperStateDesiredEvents(SafekeeperState state) * The string should not be freed. It should also not be expected to remain the same between * function calls. */ static char * -FormatEvents(uint32 events) +FormatEvents(WalProposer *wp, uint32 events) { static char return_str[8]; @@ -2119,7 +2110,7 @@ FormatEvents(uint32 events) if (events & (~all_flags)) { - elog(WARNING, "Event formatting found unexpected component %d", + walprop_log(WARNING, "Event formatting found unexpected component %d", events & (~all_flags)); return_str[6] = '*'; return_str[7] = '\0'; diff --git a/pgxn/neon/walproposer.h b/pgxn/neon/walproposer.h index a1a9ccdfdd..664aeedfa7 100644 --- a/pgxn/neon/walproposer.h +++ b/pgxn/neon/walproposer.h @@ -333,24 +333,11 @@ typedef struct Safekeeper */ char conninfo[MAXCONNINFO]; - /* - * postgres protocol connection to the WAL acceptor - * - * Equals NULL only when state = SS_OFFLINE. Nonblocking is set once we - * reach SS_ACTIVE; not before. - */ - WalProposerConn *conn; - /* * Temporary buffer for the message being sent to the safekeeper. */ StringInfoData outbuf; - /* - * WAL reader, allocated for each safekeeper. - */ - XLogReaderState *xlogreader; - /* * Streaming will start here; must be record boundary. */ @@ -361,13 +348,43 @@ typedef struct Safekeeper XLogRecPtr streamingAt; /* current streaming position */ AppendRequestHeader appendRequest; /* request for sending to safekeeper */ - int eventPos; /* position in wait event set. Equal to -1 if* - * no event */ SafekeeperState state; /* safekeeper state machine state */ TimestampTz latestMsgReceivedAt; /* when latest msg is received */ AcceptorGreeting greetResponse; /* acceptor greeting */ VoteResponse voteResponse; /* the vote */ AppendResponse appendResponse; /* feedback for master */ + + + /* postgres-specific fields */ + #ifndef WALPROPOSER_LIB + /* + * postgres protocol connection to the WAL acceptor + * + * Equals NULL only when state = SS_OFFLINE. Nonblocking is set once we + * reach SS_ACTIVE; not before. + */ + WalProposerConn *conn; + + /* + * WAL reader, allocated for each safekeeper. + */ + XLogReaderState *xlogreader; + + /* + * Position in wait event set. Equal to -1 if no event + */ + int eventPos; + #endif + + + /* WalProposer library specifics */ + #ifdef WALPROPOSER_LIB + /* + * Buffer for incoming messages. Usually Rust vector is stored here. + * Caller is responsible for freeing the buffer. + */ + StringInfoData inbuf; + #endif } Safekeeper; /* Re-exported PostgresPollingStatusType */ @@ -433,7 +450,7 @@ typedef struct walproposer_api * Get WalproposerShmemState. This is used to store information about last * elected term. */ - WalproposerShmemState *(*get_shmem_state) (void); + WalproposerShmemState *(*get_shmem_state) (WalProposer *wp); /* * Start receiving notifications about new WAL. This is an infinite loop @@ -443,61 +460,63 @@ typedef struct walproposer_api void (*start_streaming) (WalProposer *wp, XLogRecPtr startpos); /* Get pointer to the latest available WAL. */ - XLogRecPtr (*get_flush_rec_ptr) (void); + XLogRecPtr (*get_flush_rec_ptr) (WalProposer *wp); /* Get current time. */ - TimestampTz (*get_current_timestamp) (void); - - /* Get postgres timeline. */ - TimeLineID (*get_timeline_id) (void); + TimestampTz (*get_current_timestamp) (WalProposer *wp); /* Current error message, aka PQerrorMessage. */ - char *(*conn_error_message) (WalProposerConn *conn); + char *(*conn_error_message) (Safekeeper *sk); /* Connection status, aka PQstatus. */ - WalProposerConnStatusType (*conn_status) (WalProposerConn *conn); + WalProposerConnStatusType (*conn_status) (Safekeeper *sk); /* Start the connection, aka PQconnectStart. */ - WalProposerConn *(*conn_connect_start) (char *conninfo); + void (*conn_connect_start) (Safekeeper *sk); /* Poll an asynchronous connection, aka PQconnectPoll. */ - WalProposerConnectPollStatusType (*conn_connect_poll) (WalProposerConn *conn); + WalProposerConnectPollStatusType (*conn_connect_poll) (Safekeeper *sk); /* Send a blocking SQL query, aka PQsendQuery. */ - bool (*conn_send_query) (WalProposerConn *conn, char *query); + bool (*conn_send_query) (Safekeeper *sk, char *query); /* Read the query result, aka PQgetResult. */ - WalProposerExecStatusType (*conn_get_query_result) (WalProposerConn *conn); + WalProposerExecStatusType (*conn_get_query_result) (Safekeeper *sk); /* Flush buffer to the network, aka PQflush. */ - int (*conn_flush) (WalProposerConn *conn); + int (*conn_flush) (Safekeeper *sk); /* Close the connection, aka PQfinish. */ - void (*conn_finish) (WalProposerConn *conn); + void (*conn_finish) (Safekeeper *sk); - /* Try to read CopyData message, aka PQgetCopyData. */ - PGAsyncReadResult (*conn_async_read) (WalProposerConn *conn, char **buf, int *amount); + /* + * Try to read CopyData message from the safekeeper, aka PQgetCopyData. + * + * On success, the data is placed in *buf. It is valid until the next call + * to this function. + */ + PGAsyncReadResult (*conn_async_read) (Safekeeper *sk, char **buf, int *amount); /* Try to write CopyData message, aka PQputCopyData. */ - PGAsyncWriteResult (*conn_async_write) (WalProposerConn *conn, void const *buf, size_t size); + PGAsyncWriteResult (*conn_async_write) (Safekeeper *sk, void const *buf, size_t size); /* Blocking CopyData write, aka PQputCopyData + PQflush. */ - bool (*conn_blocking_write) (WalProposerConn *conn, void const *buf, size_t size); + bool (*conn_blocking_write) (Safekeeper *sk, void const *buf, size_t size); /* Download WAL from startpos to endpos and make it available locally. */ bool (*recovery_download) (Safekeeper *sk, TimeLineID timeline, XLogRecPtr startpos, XLogRecPtr endpos); /* Read WAL from disk to buf. */ - void (*wal_read) (XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count); + void (*wal_read) (Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count); /* Allocate WAL reader. */ - XLogReaderState *(*wal_reader_allocate) (void); + void (*wal_reader_allocate) (Safekeeper *sk); /* Deallocate event set. */ - void (*free_event_set) (void); + void (*free_event_set) (WalProposer *wp); /* Initialize event set. */ - void (*init_event_set) (int n_safekeepers); + void (*init_event_set) (WalProposer *wp); /* Update events for an existing safekeeper connection. */ void (*update_event_set) (Safekeeper *sk, uint32 events); @@ -513,22 +532,22 @@ typedef struct walproposer_api * events mask to indicate events and sets sk to the safekeeper which has * an event. */ - int (*wait_event_set) (long timeout, Safekeeper **sk, uint32 *events); + int (*wait_event_set) (WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events); /* Read random bytes. */ - bool (*strong_random) (void *buf, size_t len); + bool (*strong_random) (WalProposer *wp, void *buf, size_t len); /* * Get a basebackup LSN. Used to cross-validate with the latest available * LSN on the safekeepers. */ - XLogRecPtr (*get_redo_start_lsn) (void); + XLogRecPtr (*get_redo_start_lsn) (WalProposer *wp); /* * Finish sync safekeepers with the given LSN. This function should not * return and should exit the program. */ - void (*finish_sync_safekeepers) (XLogRecPtr lsn); + void (*finish_sync_safekeepers) (WalProposer *wp, XLogRecPtr lsn); /* * Called after every new message from the safekeeper. Used to propagate @@ -541,7 +560,22 @@ typedef struct walproposer_api * Called on peer_horizon_lsn updates. Used to advance replication slot * and to free up disk space by deleting unnecessary WAL. */ - void (*confirm_wal_streamed) (XLogRecPtr lsn); + void (*confirm_wal_streamed) (WalProposer *wp, XLogRecPtr lsn); + + /* + * Write a log message to the internal log processor. This is used only + * when walproposer is compiled as a library. Otherwise, all logging is + * handled by elog(). + */ + void (*log_internal) (WalProposer *wp, int level, const char *line); + + /* + * Called right after the proposer was elected, but before it started + * recovery and sent ProposerElected message to the safekeepers. + * + * Used by logical replication to update truncateLsn. + */ + void (*after_election) (WalProposer *wp); } walproposer_api; /* @@ -590,6 +624,13 @@ typedef struct WalProposerConfig /* Will be passed to safekeepers in greet request. */ uint64 systemId; + + /* Will be passed to safekeepers in greet request. */ + TimeLineID pgTimeline; + +#ifdef WALPROPOSER_LIB + void *callback_data; +#endif } WalProposerConfig; @@ -666,7 +707,16 @@ extern WalProposer *WalProposerCreate(WalProposerConfig *config, walproposer_api extern void WalProposerStart(WalProposer *wp); extern void WalProposerBroadcast(WalProposer *wp, XLogRecPtr startpos, XLogRecPtr endpos); extern void WalProposerPoll(WalProposer *wp); -extern void ParsePageserverFeedbackMessage(StringInfo reply_message, - PageserverFeedback *rf); +extern void WalProposerFree(WalProposer *wp); + + +#define WPEVENT 1337 /* special log level for walproposer internal events */ + +#ifdef WALPROPOSER_LIB +void WalProposerLibLog(WalProposer *wp, int elevel, char *fmt, ...); +#define walprop_log(elevel, ...) WalProposerLibLog(wp, elevel, __VA_ARGS__) +#else +#define walprop_log(elevel, ...) elog(elevel, __VA_ARGS__) +#endif #endif /* __NEON_WALPROPOSER_H__ */ diff --git a/pgxn/neon/walproposer_compat.c b/pgxn/neon/walproposer_compat.c new file mode 100644 index 0000000000..7617f21a26 --- /dev/null +++ b/pgxn/neon/walproposer_compat.c @@ -0,0 +1,192 @@ +/* + * Contains copied/adapted functions from libpq and some internal postgres functions. + * This is needed to avoid linking to full postgres server installation. This file + * is compiled as a part of libwalproposer static library. + */ + +#include +#include "walproposer.h" +#include "utils/datetime.h" +#include "miscadmin.h" + +void ExceptionalCondition(const char *conditionName, + const char *fileName, int lineNumber) +{ + fprintf(stderr, "ExceptionalCondition: %s:%d: %s\n", + fileName, lineNumber, conditionName); + fprintf(stderr, "aborting...\n"); + exit(1); +} + +void +pq_copymsgbytes(StringInfo msg, char *buf, int datalen) +{ + if (datalen < 0 || datalen > (msg->len - msg->cursor)) + ExceptionalCondition("insufficient data left in message", __FILE__, __LINE__); + memcpy(buf, &msg->data[msg->cursor], datalen); + msg->cursor += datalen; +} + +/* -------------------------------- + * pq_getmsgint - get a binary integer from a message buffer + * + * Values are treated as unsigned. + * -------------------------------- + */ +unsigned int +pq_getmsgint(StringInfo msg, int b) +{ + unsigned int result; + unsigned char n8; + uint16 n16; + uint32 n32; + + switch (b) + { + case 1: + pq_copymsgbytes(msg, (char *) &n8, 1); + result = n8; + break; + case 2: + pq_copymsgbytes(msg, (char *) &n16, 2); + result = pg_ntoh16(n16); + break; + case 4: + pq_copymsgbytes(msg, (char *) &n32, 4); + result = pg_ntoh32(n32); + break; + default: + fprintf(stderr, "unsupported integer size %d\n", b); + ExceptionalCondition("unsupported integer size", __FILE__, __LINE__); + result = 0; /* keep compiler quiet */ + break; + } + return result; +} + +/* -------------------------------- + * pq_getmsgint64 - get a binary 8-byte int from a message buffer + * + * It is tempting to merge this with pq_getmsgint, but we'd have to make the + * result int64 for all data widths --- that could be a big performance + * hit on machines where int64 isn't efficient. + * -------------------------------- + */ +int64 +pq_getmsgint64(StringInfo msg) +{ + uint64 n64; + + pq_copymsgbytes(msg, (char *) &n64, sizeof(n64)); + + return pg_ntoh64(n64); +} + +/* -------------------------------- + * pq_getmsgbyte - get a raw byte from a message buffer + * -------------------------------- + */ +int +pq_getmsgbyte(StringInfo msg) +{ + if (msg->cursor >= msg->len) + ExceptionalCondition("no data left in message", __FILE__, __LINE__); + return (unsigned char) msg->data[msg->cursor++]; +} + +/* -------------------------------- + * pq_getmsgbytes - get raw data from a message buffer + * + * Returns a pointer directly into the message buffer; note this + * may not have any particular alignment. + * -------------------------------- + */ +const char * +pq_getmsgbytes(StringInfo msg, int datalen) +{ + const char *result; + + if (datalen < 0 || datalen > (msg->len - msg->cursor)) + ExceptionalCondition("insufficient data left in message", __FILE__, __LINE__); + result = &msg->data[msg->cursor]; + msg->cursor += datalen; + return result; +} + +/* -------------------------------- + * pq_getmsgstring - get a null-terminated text string (with conversion) + * + * May return a pointer directly into the message buffer, or a pointer + * to a palloc'd conversion result. + * -------------------------------- + */ +const char * +pq_getmsgstring(StringInfo msg) +{ + char *str; + int slen; + + str = &msg->data[msg->cursor]; + + /* + * It's safe to use strlen() here because a StringInfo is guaranteed to + * have a trailing null byte. But check we found a null inside the + * message. + */ + slen = strlen(str); + if (msg->cursor + slen >= msg->len) + ExceptionalCondition("invalid string in message", __FILE__, __LINE__); + msg->cursor += slen + 1; + + return str; +} + +/* -------------------------------- + * pq_getmsgend - verify message fully consumed + * -------------------------------- + */ +void +pq_getmsgend(StringInfo msg) +{ + if (msg->cursor != msg->len) + ExceptionalCondition("invalid msg format", __FILE__, __LINE__); +} + + +/* + * Produce a C-string representation of a TimestampTz. + * + * This is mostly for use in emitting messages. + */ +const char * +timestamptz_to_str(TimestampTz t) +{ + static char buf[MAXDATELEN + 1]; + + snprintf(buf, sizeof(buf), "TimestampTz(%ld)", t); + return buf; +} + +bool +TimestampDifferenceExceeds(TimestampTz start_time, + TimestampTz stop_time, + int msec) +{ + TimestampTz diff = stop_time - start_time; + return (diff >= msec * INT64CONST(1000)); +} + +void +WalProposerLibLog(WalProposer *wp, int elevel, char *fmt, ...) +{ + char buf[1024]; + va_list args; + + fmt = _(fmt); + + va_start(args, fmt); + vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + + wp->api.log_internal(wp, elevel, buf); +} diff --git a/pgxn/neon/walproposer_pg.c b/pgxn/neon/walproposer_pg.c index 654b411e94..865f91165b 100644 --- a/pgxn/neon/walproposer_pg.c +++ b/pgxn/neon/walproposer_pg.c @@ -73,7 +73,8 @@ static void walprop_register_bgworker(void); static void walprop_pg_init_standalone_sync_safekeepers(void); static void walprop_pg_init_walsender(void); static void walprop_pg_init_bgworker(void); -static TimestampTz walprop_pg_get_current_timestamp(void); +static TimestampTz walprop_pg_get_current_timestamp(WalProposer *wp); +static TimeLineID walprop_pg_get_timeline_id(void); static void walprop_pg_load_libpqwalreceiver(void); static process_interrupts_callback_t PrevProcessInterruptsCallback; @@ -104,6 +105,7 @@ init_walprop_config(bool syncSafekeepers) walprop_config.systemId = GetSystemIdentifier(); else walprop_config.systemId = 0; + walprop_config.pgTimeline = walprop_pg_get_timeline_id(); } /* @@ -136,7 +138,7 @@ WalProposerMain(Datum main_arg) walprop_pg_load_libpqwalreceiver(); wp = WalProposerCreate(&walprop_config, walprop_pg); - wp->last_reconnect_attempt = walprop_pg_get_current_timestamp(); + wp->last_reconnect_attempt = walprop_pg_get_current_timestamp(wp); walprop_pg_init_walsender(); WalProposerStart(wp); @@ -379,7 +381,7 @@ nwp_shmem_startup_hook(void) } static WalproposerShmemState * -walprop_pg_get_shmem_state(void) +walprop_pg_get_shmem_state(WalProposer *wp) { Assert(walprop_shared != NULL); return walprop_shared; @@ -505,7 +507,7 @@ walprop_pg_init_bgworker(void) } static XLogRecPtr -walprop_pg_get_flush_rec_ptr(void) +walprop_pg_get_flush_rec_ptr(WalProposer *wp) { #if PG_MAJORVERSION_NUM < 15 return GetFlushRecPtr(); @@ -515,7 +517,7 @@ walprop_pg_get_flush_rec_ptr(void) } static TimestampTz -walprop_pg_get_current_timestamp(void) +walprop_pg_get_current_timestamp(WalProposer *wp) { return GetCurrentTimestamp(); } @@ -565,15 +567,15 @@ ensure_nonblocking_status(WalProposerConn *conn, bool is_nonblocking) /* Exported function definitions */ static char * -walprop_error_message(WalProposerConn *conn) +walprop_error_message(Safekeeper *sk) { - return PQerrorMessage(conn->pg_conn); + return PQerrorMessage(sk->conn->pg_conn); } static WalProposerConnStatusType -walprop_status(WalProposerConn *conn) +walprop_status(Safekeeper *sk) { - switch (PQstatus(conn->pg_conn)) + switch (PQstatus(sk->conn->pg_conn)) { case CONNECTION_OK: return WP_CONNECTION_OK; @@ -584,16 +586,17 @@ walprop_status(WalProposerConn *conn) } } -static WalProposerConn * -walprop_connect_start(char *conninfo) +static void +walprop_connect_start(Safekeeper *sk) { - WalProposerConn *conn; PGconn *pg_conn; const char *keywords[3]; const char *values[3]; int n; char *password = neon_auth_token; + Assert(sk->conn == NULL); + /* * Connect using the given connection string. If the NEON_AUTH_TOKEN * environment variable was set, use that as the password. @@ -611,7 +614,7 @@ walprop_connect_start(char *conninfo) n++; } keywords[n] = "dbname"; - values[n] = conninfo; + values[n] = sk->conninfo; n++; keywords[n] = NULL; values[n] = NULL; @@ -619,11 +622,11 @@ walprop_connect_start(char *conninfo) pg_conn = PQconnectStartParams(keywords, values, 1); /* - * Allocation of a PQconn can fail, and will return NULL. We want to fully - * replicate the behavior of PQconnectStart here. + * "If the result is null, then libpq has been unable to allocate a new + * PGconn structure" */ if (!pg_conn) - return NULL; + elog(FATAL, "failed to allocate new PGconn object"); /* * And in theory this allocation can fail as well, but it's incredibly @@ -632,20 +635,19 @@ walprop_connect_start(char *conninfo) * palloc will exit on failure though, so there's not much we could do if * it *did* fail. */ - conn = palloc(sizeof(WalProposerConn)); - conn->pg_conn = pg_conn; - conn->is_nonblocking = false; /* connections always start in blocking + sk->conn = palloc(sizeof(WalProposerConn)); + sk->conn->pg_conn = pg_conn; + sk->conn->is_nonblocking = false; /* connections always start in blocking * mode */ - conn->recvbuf = NULL; - return conn; + sk->conn->recvbuf = NULL; } static WalProposerConnectPollStatusType -walprop_connect_poll(WalProposerConn *conn) +walprop_connect_poll(Safekeeper *sk) { WalProposerConnectPollStatusType return_val; - switch (PQconnectPoll(conn->pg_conn)) + switch (PQconnectPoll(sk->conn->pg_conn)) { case PGRES_POLLING_FAILED: return_val = WP_CONN_POLLING_FAILED; @@ -682,24 +684,24 @@ walprop_connect_poll(WalProposerConn *conn) } static bool -walprop_send_query(WalProposerConn *conn, char *query) +walprop_send_query(Safekeeper *sk, char *query) { /* * We need to be in blocking mode for sending the query to run without * requiring a call to PQflush */ - if (!ensure_nonblocking_status(conn, false)) + if (!ensure_nonblocking_status(sk->conn, false)) return false; /* PQsendQuery returns 1 on success, 0 on failure */ - if (!PQsendQuery(conn->pg_conn, query)) + if (!PQsendQuery(sk->conn->pg_conn, query)) return false; return true; } static WalProposerExecStatusType -walprop_get_query_result(WalProposerConn *conn) +walprop_get_query_result(Safekeeper *sk) { PGresult *result; WalProposerExecStatusType return_val; @@ -708,14 +710,14 @@ walprop_get_query_result(WalProposerConn *conn) char *unexpected_success = NULL; /* Consume any input that we might be missing */ - if (!PQconsumeInput(conn->pg_conn)) + if (!PQconsumeInput(sk->conn->pg_conn)) return WP_EXEC_FAILED; - if (PQisBusy(conn->pg_conn)) + if (PQisBusy(sk->conn->pg_conn)) return WP_EXEC_NEEDS_INPUT; - result = PQgetResult(conn->pg_conn); + result = PQgetResult(sk->conn->pg_conn); /* * PQgetResult returns NULL only if getting the result was successful & @@ -777,24 +779,28 @@ walprop_get_query_result(WalProposerConn *conn) } static pgsocket -walprop_socket(WalProposerConn *conn) +walprop_socket(Safekeeper *sk) { - return PQsocket(conn->pg_conn); + return PQsocket(sk->conn->pg_conn); } static int -walprop_flush(WalProposerConn *conn) +walprop_flush(Safekeeper *sk) { - return (PQflush(conn->pg_conn)); + return (PQflush(sk->conn->pg_conn)); } static void -walprop_finish(WalProposerConn *conn) +walprop_finish(Safekeeper *sk) { - if (conn->recvbuf != NULL) - PQfreemem(conn->recvbuf); - PQfinish(conn->pg_conn); - pfree(conn); + if (!sk->conn) + return; + + if (sk->conn->recvbuf != NULL) + PQfreemem(sk->conn->recvbuf); + PQfinish(sk->conn->pg_conn); + pfree(sk->conn); + sk->conn = NULL; } /* @@ -804,18 +810,18 @@ walprop_finish(WalProposerConn *conn) * to this function. */ static PGAsyncReadResult -walprop_async_read(WalProposerConn *conn, char **buf, int *amount) +walprop_async_read(Safekeeper *sk, char **buf, int *amount) { int result; - if (conn->recvbuf != NULL) + if (sk->conn->recvbuf != NULL) { - PQfreemem(conn->recvbuf); - conn->recvbuf = NULL; + PQfreemem(sk->conn->recvbuf); + sk->conn->recvbuf = NULL; } /* Call PQconsumeInput so that we have the data we need */ - if (!PQconsumeInput(conn->pg_conn)) + if (!PQconsumeInput(sk->conn->pg_conn)) { *amount = 0; *buf = NULL; @@ -833,7 +839,7 @@ walprop_async_read(WalProposerConn *conn, char **buf, int *amount) * sometimes be triggered by the server returning an ErrorResponse (which * also happens to have the effect that the copy is done). */ - switch (result = PQgetCopyData(conn->pg_conn, &conn->recvbuf, true)) + switch (result = PQgetCopyData(sk->conn->pg_conn, &sk->conn->recvbuf, true)) { case 0: *amount = 0; @@ -848,7 +854,7 @@ walprop_async_read(WalProposerConn *conn, char **buf, int *amount) * We can check PQgetResult to make sure that the server * failed; it'll always result in PGRES_FATAL_ERROR */ - ExecStatusType status = PQresultStatus(PQgetResult(conn->pg_conn)); + ExecStatusType status = PQresultStatus(PQgetResult(sk->conn->pg_conn)); if (status != PGRES_FATAL_ERROR) elog(FATAL, "unexpected result status %d after failed PQgetCopyData", status); @@ -869,18 +875,18 @@ walprop_async_read(WalProposerConn *conn, char **buf, int *amount) default: /* Positive values indicate the size of the returned result */ *amount = result; - *buf = conn->recvbuf; + *buf = sk->conn->recvbuf; return PG_ASYNC_READ_SUCCESS; } } static PGAsyncWriteResult -walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) +walprop_async_write(Safekeeper *sk, void const *buf, size_t size) { int result; /* If we aren't in non-blocking mode, switch to it. */ - if (!ensure_nonblocking_status(conn, true)) + if (!ensure_nonblocking_status(sk->conn, true)) return PG_ASYNC_WRITE_FAIL; /* @@ -888,7 +894,7 @@ walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) * queued, 0 if it was not queued because of full buffers, or -1 if an * error occurred */ - result = PQputCopyData(conn->pg_conn, buf, size); + result = PQputCopyData(sk->conn->pg_conn, buf, size); /* * We won't get a result of zero because walproposer always empties the @@ -916,7 +922,7 @@ walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) * sucessful, 1 if it was unable to send all the data in the send queue * yet -1 if it failed for some reason */ - switch (result = PQflush(conn->pg_conn)) + switch (result = PQflush(sk->conn->pg_conn)) { case 0: return PG_ASYNC_WRITE_SUCCESS; @@ -934,22 +940,22 @@ walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) * information, refer to the comments there. */ static bool -walprop_blocking_write(WalProposerConn *conn, void const *buf, size_t size) +walprop_blocking_write(Safekeeper *sk, void const *buf, size_t size) { int result; /* If we are in non-blocking mode, switch out of it. */ - if (!ensure_nonblocking_status(conn, false)) + if (!ensure_nonblocking_status(sk->conn, false)) return false; - if ((result = PQputCopyData(conn->pg_conn, buf, size)) == -1) + if ((result = PQputCopyData(sk->conn->pg_conn, buf, size)) == -1) return false; Assert(result == 1); /* Because the connection is non-blocking, flushing returns 0 or -1 */ - if ((result = PQflush(conn->pg_conn)) == -1) + if ((result = PQflush(sk->conn->pg_conn)) == -1) return false; Assert(result == 0); @@ -1381,11 +1387,11 @@ XLogWalPropClose(XLogRecPtr recptr) } static void -walprop_pg_wal_read(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count) +walprop_pg_wal_read(Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count) { WALReadError errinfo; - if (!WALRead(state, + if (!WALRead(sk->xlogreader, buf, startptr, count, @@ -1396,31 +1402,38 @@ walprop_pg_wal_read(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size } } -static XLogReaderState * -walprop_pg_wal_reader_allocate(void) +static void +walprop_pg_wal_reader_allocate(Safekeeper *sk) { - return XLogReaderAllocate(wal_segment_size, NULL, XL_ROUTINE(.segment_open = wal_segment_open,.segment_close = wal_segment_close), NULL); + sk->xlogreader = XLogReaderAllocate(wal_segment_size, NULL, XL_ROUTINE(.segment_open = wal_segment_open,.segment_close = wal_segment_close), NULL); + if (sk->xlogreader == NULL) + elog(FATAL, "Failed to allocate xlog reader"); } static WaitEventSet *waitEvents; static void -walprop_pg_free_event_set(void) +walprop_pg_free_event_set(WalProposer *wp) { if (waitEvents) { FreeWaitEventSet(waitEvents); waitEvents = NULL; } + + for (int i = 0; i < wp->n_safekeepers; i++) + { + wp->safekeeper[i].eventPos = -1; + } } static void -walprop_pg_init_event_set(int n_safekeepers) +walprop_pg_init_event_set(WalProposer *wp) { if (waitEvents) elog(FATAL, "double-initialization of event set"); - waitEvents = CreateWaitEventSet(TopMemoryContext, 2 + n_safekeepers); + waitEvents = CreateWaitEventSet(TopMemoryContext, 2 + wp->n_safekeepers); AddWaitEventToSet(waitEvents, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); AddWaitEventToSet(waitEvents, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, @@ -1439,11 +1452,11 @@ walprop_pg_update_event_set(Safekeeper *sk, uint32 events) static void walprop_pg_add_safekeeper_event_set(Safekeeper *sk, uint32 events) { - sk->eventPos = AddWaitEventToSet(waitEvents, events, walprop_socket(sk->conn), NULL, sk); + sk->eventPos = AddWaitEventToSet(waitEvents, events, walprop_socket(sk), NULL, sk); } static int -walprop_pg_wait_event_set(long timeout, Safekeeper **sk, uint32 *events) +walprop_pg_wait_event_set(WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events) { WaitEvent event = {0}; int rc = 0; @@ -1499,7 +1512,7 @@ walprop_pg_wait_event_set(long timeout, Safekeeper **sk, uint32 *events) } static void -walprop_pg_finish_sync_safekeepers(XLogRecPtr lsn) +walprop_pg_finish_sync_safekeepers(WalProposer *wp, XLogRecPtr lsn) { fprintf(stdout, "%X/%X\n", LSN_FORMAT_ARGS(lsn)); exit(0); @@ -1611,7 +1624,7 @@ walprop_pg_process_safekeeper_feedback(WalProposer *wp, XLogRecPtr commitLsn) * pageserver. */ quorumFeedback.rf.disk_consistent_lsn, - walprop_pg_get_current_timestamp(), false); + walprop_pg_get_current_timestamp(wp), false); } CombineHotStanbyFeedbacks(&hsFeedback, wp); @@ -1628,18 +1641,65 @@ walprop_pg_process_safekeeper_feedback(WalProposer *wp, XLogRecPtr commitLsn) } static void -walprop_pg_confirm_wal_streamed(XLogRecPtr lsn) +walprop_pg_confirm_wal_streamed(WalProposer *wp, XLogRecPtr lsn) { if (MyReplicationSlot) PhysicalConfirmReceivedLocation(lsn); } +static XLogRecPtr +walprop_pg_get_redo_start_lsn(WalProposer *wp) +{ + return GetRedoStartLsn(); +} + +static bool +walprop_pg_strong_random(WalProposer *wp, void *buf, size_t len) +{ + return pg_strong_random(buf, len); +} + +static void +walprop_pg_log_internal(WalProposer *wp, int level, const char *line) +{ + elog(FATAL, "unexpected log_internal message at level %d: %s", level, line); +} + +static void +walprop_pg_after_election(WalProposer *wp) +{ + FILE* f; + XLogRecPtr lrRestartLsn; + + /* We don't need to do anything in syncSafekeepers mode.*/ + if (wp->config->syncSafekeepers) + return; + + /* + * If there are active logical replication subscription we need + * to provide enough WAL for their WAL senders based on th position + * of their replication slots. + */ + f = fopen("restart.lsn", "rb"); + if (f != NULL && !wp->config->syncSafekeepers) + { + fread(&lrRestartLsn, sizeof(lrRestartLsn), 1, f); + fclose(f); + if (lrRestartLsn != InvalidXLogRecPtr) + { + elog(LOG, "Logical replication restart LSN %X/%X", LSN_FORMAT_ARGS(lrRestartLsn)); + /* start from the beginning of the segment to fetch page headers verifed by XLogReader */ + lrRestartLsn = lrRestartLsn - XLogSegmentOffset(lrRestartLsn, wal_segment_size); + wp->truncateLsn = Min(wp->truncateLsn, lrRestartLsn); + } + } +} + static const walproposer_api walprop_pg = { .get_shmem_state = walprop_pg_get_shmem_state, .start_streaming = walprop_pg_start_streaming, .get_flush_rec_ptr = walprop_pg_get_flush_rec_ptr, .get_current_timestamp = walprop_pg_get_current_timestamp, - .get_timeline_id = walprop_pg_get_timeline_id, .conn_error_message = walprop_error_message, .conn_status = walprop_status, .conn_connect_start = walprop_connect_start, @@ -1659,9 +1719,11 @@ static const walproposer_api walprop_pg = { .update_event_set = walprop_pg_update_event_set, .add_safekeeper_event_set = walprop_pg_add_safekeeper_event_set, .wait_event_set = walprop_pg_wait_event_set, - .strong_random = pg_strong_random, - .get_redo_start_lsn = GetRedoStartLsn, + .strong_random = walprop_pg_strong_random, + .get_redo_start_lsn = walprop_pg_get_redo_start_lsn, .finish_sync_safekeepers = walprop_pg_finish_sync_safekeepers, .process_safekeeper_feedback = walprop_pg_process_safekeeper_feedback, .confirm_wal_streamed = walprop_pg_confirm_wal_streamed, + .log_internal = walprop_pg_log_internal, + .after_election = walprop_pg_after_election, }; From 893b7bac9abb279cc0097d2c7bf640d89304a92f Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Thu, 19 Oct 2023 17:24:23 +0300 Subject: [PATCH 28/49] Fix neon_extra_builds.yml : nproc is not supported in mac os (#5598) ## Problem nproc is not supported in mac os, use sysctl -n hw.ncpu instead --- .github/workflows/neon_extra_builds.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 891cc8472a..03506651c8 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 @@ -90,15 +90,15 @@ jobs: - name: Build postgres v14 if: steps.cache_pg_14.outputs.cache-hit != 'true' - run: make postgres-v14 -j$(nproc) + run: make postgres-v14 -j$(sysctl -n hw.ncpu) - name: Build postgres v15 if: steps.cache_pg_15.outputs.cache-hit != 'true' - run: make postgres-v15 -j$(nproc) + run: make postgres-v15 -j$(sysctl -n hw.ncpu) - name: Build postgres v16 if: steps.cache_pg_16.outputs.cache-hit != 'true' - run: make postgres-v16 -j$(nproc) + run: make postgres-v16 -j$(sysctl -n hw.ncpu) - name: Build neon extensions run: make neon-pg-ext -j$(nproc) @@ -129,14 +129,14 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 # Some of our rust modules use FFI and need those to be checked - name: Get postgres headers - run: make postgres-headers -j$(nproc) + run: make postgres-headers -j$(sysctl -n hw.ncpu) - name: Produce the build stats run: cargo build --all --release --timings From 2c8741a5ed37f4f7554eff891cfcfc608e380866 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 19 Oct 2023 09:10:33 -0700 Subject: [PATCH 29/49] vm-monitor: Log full error on message handling failure (#5604) There's currently an issue with the vm-monitor on staging that's not really feasible to debug because the current display impl gives no context to the errors (just says "failed to downscale"). Logging the full error should help. For communications with the autoscaler-agent, it's ok to only provide the outermost cause, because we can cross-reference with the VM logs. At some point in the future, we may want to change that. --- libs/vm_monitor/src/runner.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/vm_monitor/src/runner.rs b/libs/vm_monitor/src/runner.rs index a7a0995797..e818292196 100644 --- a/libs/vm_monitor/src/runner.rs +++ b/libs/vm_monitor/src/runner.rs @@ -505,11 +505,14 @@ impl Runner { Ok(Some(out)) => out, Ok(None) => continue, Err(e) => { - let error = e.to_string(); - warn!(?error, "error handling message"); + // use {:#} for our logging because the display impl only + // gives the outermost cause, and the debug impl + // pretty-prints the error, whereas {:#} contains all the + // causes, but is compact (no newlines). + warn!(error = format!("{e:#}"), "error handling message"); OutboundMsg::new( OutboundMsgKind::InternalError { - error + error: e.to_string(), }, message.id ) From 5a8bcdccb0f0386f7aa05410f845723b174e90ae Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 19 Oct 2023 20:24:35 +0300 Subject: [PATCH 30/49] Fix elog format error in wallog_mapping_file (#5602) ## Problem Fix elog format error in wallog_mapping_file ## Summary of changes Use proper case to avoid compilation warning=error in C at MacOS. ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist Co-authored-by: Konstantin Knizhnik --- vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/revisions.json | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index ebcca9e9eb..6669a672ee 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit ebcca9e9eb49621b5b17247833b59e836337e8aa +Subproject commit 6669a672ee14ab2c09d44c4552f9a13fad3afc10 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 23f2d41102..ab67ab9635 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 23f2d411020a739375b32895ce1362ded2962084 +Subproject commit ab67ab96355d61e9d0218630be4aa7db53bf83e7 diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index e5e255d2da..550ffa6495 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit e5e255d2da05bc5f884b871c042014030a114a9b +Subproject commit 550ffa6495a5dc62fccc3a8b449386633758680b diff --git a/vendor/revisions.json b/vendor/revisions.json index 14ccbef287..012fb14035 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "postgres-v16": "e5e255d2da05bc5f884b871c042014030a114a9b", - "postgres-v15": "23f2d411020a739375b32895ce1362ded2962084", - "postgres-v14": "ebcca9e9eb49621b5b17247833b59e836337e8aa" + "postgres-v16": "550ffa6495a5dc62fccc3a8b449386633758680b", + "postgres-v15": "ab67ab96355d61e9d0218630be4aa7db53bf83e7", + "postgres-v14": "6669a672ee14ab2c09d44c4552f9a13fad3afc10" } From 2cf6a47cca9adee72fd2cad4ac69414f4ba2d059 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 19 Oct 2023 11:09:37 -0700 Subject: [PATCH 31/49] vm-monitor: Deny not fail downscale if no memory stats yet (#5606) Fixes an issue we observed on staging that happens when the autoscaler-agent attempts to immediately downscale the VM after binding, which is typical for pooled computes. The issue was occurring because the autoscaler-agent was requesting downscaling before the vm-monitor had gathered sufficient cgroup memory stats to be confident in approving it. When the vm-monitor returned an internal error instead of denying downscaling, the autoscaler-agent retried the connection and immediately hit the same issue (in part because cgroup stats are collected per-connection, rather than globally). --- libs/vm_monitor/src/runner.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libs/vm_monitor/src/runner.rs b/libs/vm_monitor/src/runner.rs index e818292196..99bea6d7ca 100644 --- a/libs/vm_monitor/src/runner.rs +++ b/libs/vm_monitor/src/runner.rs @@ -253,11 +253,22 @@ impl Runner { if let Some(cgroup) = &self.cgroup { let (last_time, last_history) = *cgroup.watcher.borrow(); + // NB: The ordering of these conditions is intentional. During startup, we should deny + // downscaling until we have enough information to determine that it's safe to do so + // (i.e. enough samples have come in). But if it's been a while and we *still* haven't + // received any information, we should *fail* instead of just denying downscaling. + // + // `last_time` is set to `Instant::now()` on startup, so checking `last_time.elapsed()` + // serves double-duty: it trips if we haven't received *any* metrics for long enough, + // OR if we haven't received metrics *recently enough*. + // // TODO: make the duration here configurable. if last_time.elapsed() > Duration::from_secs(5) { bail!("haven't gotten cgroup memory stats recently enough to determine downscaling information"); } else if last_history.samples_count <= 1 { - bail!("haven't received enough cgroup memory stats yet"); + let status = "haven't received enough cgroup memory stats yet"; + info!(status, "discontinuing downscale"); + return Ok((false, status.to_owned())); } let new_threshold = self From ba856140e799bb186b4b5636b3052f5888b39776 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 19 Oct 2023 22:20:39 +0100 Subject: [PATCH 32/49] Fix neon_extra_build.yml (#5605) Build walproposer-lib in gather-rust-build-stats, fix nproc usage, fix walproposer-lib on macos. --- .github/workflows/neon_extra_builds.yml | 9 ++++++--- Makefile | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 03506651c8..d7f5295c5b 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -101,10 +101,10 @@ jobs: run: make postgres-v16 -j$(sysctl -n hw.ncpu) - name: Build neon extensions - run: make neon-pg-ext -j$(nproc) + run: make neon-pg-ext -j$(sysctl -n hw.ncpu) - name: Build walproposer-lib - run: make walproposer-lib -j$(nproc) + run: make walproposer-lib -j$(sysctl -n hw.ncpu) - name: Run cargo build run: cargo build --all --release @@ -136,7 +136,10 @@ jobs: # Some of our rust modules use FFI and need those to be checked - name: Get postgres headers - run: make postgres-headers -j$(sysctl -n hw.ncpu) + run: make postgres-headers -j$(nproc) + + - name: Build walproposer-lib + run: make walproposer-lib -j$(nproc) - name: Produce the build stats run: cargo build --all --release --timings diff --git a/Makefile b/Makefile index 3b3f0e3dac..64bbc1677c 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,7 @@ walproposer-lib: neon-pg-ext-v16 -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile walproposer-lib cp $(POSTGRES_INSTALL_DIR)/v16/lib/libpgport.a $(POSTGRES_INSTALL_DIR)/build/walproposer-lib cp $(POSTGRES_INSTALL_DIR)/v16/lib/libpgcommon.a $(POSTGRES_INSTALL_DIR)/build/walproposer-lib +ifeq ($(UNAME_S),Linux) $(AR) d $(POSTGRES_INSTALL_DIR)/build/walproposer-lib/libpgport.a \ pg_strong_random.o $(AR) d $(POSTGRES_INSTALL_DIR)/build/walproposer-lib/libpgcommon.a \ @@ -195,6 +196,7 @@ walproposer-lib: neon-pg-ext-v16 scram-common.o \ md5_common.o \ checksum_helper.o +endif .PHONY: walproposer-lib-clean walproposer-lib-clean: From 76c702219cca04e56b4afdda5475e4380540b02e Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 1 Sep 2023 13:40:34 +0300 Subject: [PATCH 33/49] Don't use AppenRequestHeader.epoch_start_lsn. It is simpler to get it once from ProposerEelected. --- safekeeper/src/safekeeper.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 85e556aff2..9c6efeb349 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -344,7 +344,8 @@ pub struct AppendRequest { pub struct AppendRequestHeader { // safekeeper's current term; if it is higher than proposer's, the compute is out of date. pub term: Term, - // LSN since the proposer appends WAL; determines epoch switch point. + // TODO: remove this field, it in unused -- LSN of term switch can be taken + // from ProposerElected (as well as from term history). pub epoch_start_lsn: Lsn, /// start position of message in WAL pub begin_lsn: Lsn, @@ -810,6 +811,14 @@ where info!("start receiving WAL since {:?}", msg.start_streaming_at); + // Cache LSN where term starts to immediately fsync control file with + // commit_lsn once we reach it -- sync-safekeepers finishes when + // persisted commit_lsn on majority of safekeepers aligns. + self.epoch_start_lsn = match msg.term_history.0.last() { + None => bail!("proposer elected with empty term history"), + Some(term_lsn_start) => term_lsn_start.lsn, + }; + Ok(None) } @@ -835,10 +844,7 @@ where // file: walproposer in sync mode is very interested when this // happens. Note: this is for sync-safekeepers mode only, as // otherwise commit_lsn might jump over epoch_start_lsn. - // Also note that commit_lsn can reach epoch_start_lsn earlier - // that we receive new epoch_start_lsn, and we still need to sync - // control file in this case. - if commit_lsn == self.epoch_start_lsn && self.state.commit_lsn != commit_lsn { + if commit_lsn >= self.epoch_start_lsn && self.state.commit_lsn < self.epoch_start_lsn { self.persist_control_file(self.state.clone()).await?; } @@ -902,7 +908,6 @@ where // Now we know that we are in the same term as the proposer, // processing the message. - self.epoch_start_lsn = msg.h.epoch_start_lsn; self.inmem.proposer_uuid = msg.h.proposer_uuid; // do the job From b332268cec58839026ec6d5eaf3e45d3b7329e51 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 5 Sep 2023 14:36:37 +0300 Subject: [PATCH 34/49] Introduce safekeeper peer recovery. Implements fetching of WAL by safekeeper from another safekeeper by imitating behaviour of last elected leader. This allows to avoid WAL accumulation on compute and facilitates faster compute startup as it doesn't need to download any WAL. Actually removing WAL download in walproposer is a matter of another patch though. There is a per timeline task which always runs, checking regularly if it should start recovery frome someone, meaning there is something to fetch and there is no streaming compute. It then proceeds with fetching, finishing when there is nothing more to receive. Implements https://github.com/neondatabase/neon/pull/4875 --- safekeeper/src/bin/safekeeper.rs | 6 +- safekeeper/src/handler.rs | 9 +- safekeeper/src/http/routes.rs | 15 +- safekeeper/src/lib.rs | 2 + safekeeper/src/receive_wal.rs | 60 +++- safekeeper/src/recovery.rs | 406 ++++++++++++++++++++++- safekeeper/src/safekeeper.rs | 134 +++++++- safekeeper/src/send_wal.rs | 5 +- safekeeper/src/timeline.rs | 127 ++++++- test_runner/regress/test_wal_acceptor.py | 86 ++++- 10 files changed, 797 insertions(+), 53 deletions(-) diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 763dcd9eb8..202343fca1 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -3,7 +3,7 @@ // use anyhow::{bail, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; -use clap::Parser; +use clap::{ArgAction, Parser}; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; use futures::{FutureExt, StreamExt}; @@ -105,6 +105,9 @@ struct Args { /// it during this period passed as a human readable duration. #[arg(long, value_parser= humantime::parse_duration, default_value = DEFAULT_HEARTBEAT_TIMEOUT, verbatim_doc_comment)] heartbeat_timeout: Duration, + /// Enable/disable peer recovery. + #[arg(long, default_value = "false", action=ArgAction::Set)] + peer_recovery: bool, /// Remote storage configuration for WAL backup (offloading to s3) as TOML /// inline table, e.g. /// {"max_concurrent_syncs" = 17, "max_sync_errors": 13, "bucket_name": "", "bucket_region":"", "concurrency_limit": 119} @@ -265,6 +268,7 @@ async fn main() -> anyhow::Result<()> { broker_endpoint: args.broker_endpoint, broker_keepalive_interval: args.broker_keepalive_interval, heartbeat_timeout: args.heartbeat_timeout, + peer_recovery_enabled: args.peer_recovery, remote_storage: args.remote_storage, max_offloader_lag_bytes: args.max_offloader_lag, wal_backup_enabled: !args.disable_wal_backup, diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index 134331c673..71b99ab1d8 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -372,6 +372,13 @@ impl SafekeeperPostgresHandler { /// from a walproposer recovery function. This connection gets a special handling: /// safekeeper must stream all local WAL till the flush_lsn, whether committed or not. pub fn is_walproposer_recovery(&self) -> bool { - self.appname == Some("wal_proposer_recovery".to_string()) + match &self.appname { + None => false, + Some(appname) => { + appname == "wal_proposer_recovery" || + // set by safekeeper peer recovery + appname.starts_with("safekeeper") + } + } } } diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index f4c3a4aa03..940ac82df6 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -16,8 +16,8 @@ use tokio::io::AsyncReadExt; use utils::http::endpoint::request_span; use crate::receive_wal::WalReceiverState; -use crate::safekeeper::ServerInfo; use crate::safekeeper::Term; +use crate::safekeeper::{ServerInfo, TermLsn}; use crate::send_wal::WalSenderState; use crate::timeline::PeerInfo; use crate::{debug_dump, pull_timeline}; @@ -60,16 +60,25 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { .as_ref() } -/// Same as TermSwitchEntry, but serializes LSN using display serializer +/// Same as TermLsn, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. #[serde_as] -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct TermSwitchApiEntry { pub term: Term, #[serde_as(as = "DisplayFromStr")] pub lsn: Lsn, } +impl From for TermLsn { + fn from(api_val: TermSwitchApiEntry) -> Self { + TermLsn { + term: api_val.term, + lsn: api_val.lsn, + } + } +} + /// Augment AcceptorState with epoch for convenience #[derive(Debug, Serialize, Deserialize)] pub struct AcceptorStateStatus { diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 00aa405047..aa70cee591 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -62,6 +62,7 @@ pub struct SafeKeeperConf { pub broker_endpoint: Uri, pub broker_keepalive_interval: Duration, pub heartbeat_timeout: Duration, + pub peer_recovery_enabled: bool, pub remote_storage: Option, pub max_offloader_lag_bytes: u64, pub backup_parallel_jobs: usize, @@ -100,6 +101,7 @@ impl SafeKeeperConf { .parse() .expect("failed to parse default broker endpoint"), broker_keepalive_interval: Duration::from_secs(5), + peer_recovery_enabled: true, wal_backup_enabled: true, backup_parallel_jobs: 1, pg_auth: None, diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index cf63eace21..934dda10b2 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -55,9 +55,12 @@ impl WalReceivers { /// Register new walreceiver. Returned guard provides access to the slot and /// automatically deregisters in Drop. - pub fn register(self: &Arc) -> WalReceiverGuard { + pub fn register(self: &Arc, conn_id: Option) -> WalReceiverGuard { let slots = &mut self.mutex.lock().slots; - let walreceiver = WalReceiverState::Voting; + let walreceiver = WalReceiverState { + conn_id, + status: WalReceiverStatus::Voting, + }; // find empty slot or create new one let pos = if let Some(pos) = slots.iter().position(|s| s.is_none()) { slots[pos] = Some(walreceiver); @@ -96,6 +99,18 @@ impl WalReceivers { self.mutex.lock().slots.iter().flatten().cloned().collect() } + /// Get number of streaming walreceivers (normally 0 or 1) from compute. + pub fn get_num_streaming(self: &Arc) -> usize { + self.mutex + .lock() + .slots + .iter() + .flatten() + // conn_id.is_none skips recovery which also registers here + .filter(|s| s.conn_id.is_some() && matches!(s.status, WalReceiverStatus::Streaming)) + .count() + } + /// Unregister walsender. fn unregister(self: &Arc, id: WalReceiverId) { let mut shared = self.mutex.lock(); @@ -108,10 +123,17 @@ struct WalReceiversShared { slots: Vec>, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WalReceiverState { + /// None means it is recovery initiated by us (this safekeeper). + pub conn_id: Option, + pub status: WalReceiverStatus, +} + /// Walreceiver status. Currently only whether it passed voting stage and /// started receiving the stream, but it is easy to add more if needed. #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum WalReceiverState { +pub enum WalReceiverStatus { Voting, Streaming, } @@ -136,8 +158,8 @@ impl Drop for WalReceiverGuard { } } -const MSG_QUEUE_SIZE: usize = 256; -const REPLY_QUEUE_SIZE: usize = 16; +pub const MSG_QUEUE_SIZE: usize = 256; +pub const REPLY_QUEUE_SIZE: usize = 16; impl SafekeeperPostgresHandler { /// Wrapper around handle_start_wal_push_guts handling result. Error is @@ -261,7 +283,7 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { tli.clone(), msg_rx, reply_tx, - self.conn_id, + Some(self.conn_id), )); // Forward all messages to WalAcceptor @@ -317,31 +339,41 @@ async fn network_write( // even when it writes a steady stream of messages. const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(1); -/// Takes messages from msg_rx, processes and pushes replies to reply_tx. -struct WalAcceptor { +/// Encapsulates a task which takes messages from msg_rx, processes and pushes +/// replies to reply_tx; reading from socket and writing to disk in parallel is +/// beneficial for performance, this struct provides writing to disk part. +pub struct WalAcceptor { tli: Arc, msg_rx: Receiver, reply_tx: Sender, + conn_id: Option, } impl WalAcceptor { - /// Spawn thread with WalAcceptor running, return handle to it. - fn spawn( + /// Spawn task with WalAcceptor running, return handle to it. Task returns + /// Ok(()) if either of channels has closed, and Err if any error during + /// message processing is encountered. + /// + /// conn_id None means WalAcceptor is used by recovery initiated at this safekeeper. + pub fn spawn( tli: Arc, msg_rx: Receiver, reply_tx: Sender, - conn_id: ConnectionId, + conn_id: Option, ) -> JoinHandle> { task::spawn(async move { let mut wa = WalAcceptor { tli, msg_rx, reply_tx, + conn_id, }; let span_ttid = wa.tli.ttid; // satisfy borrow checker wa.run() - .instrument(info_span!("WAL acceptor", cid = %conn_id, ttid = %span_ttid)) + .instrument( + info_span!("WAL acceptor", cid = %conn_id.unwrap_or(0), ttid = %span_ttid), + ) .await }) } @@ -355,7 +387,7 @@ impl WalAcceptor { let _compute_conn_guard = ComputeConnectionGuard { timeline: Arc::clone(&self.tli), }; - let walreceiver_guard = self.tli.get_walreceivers().register(); + let walreceiver_guard = self.tli.get_walreceivers().register(self.conn_id); self.tli.update_status_notify().await?; // After this timestamp we will stop processing AppendRequests and send a response @@ -372,7 +404,7 @@ impl WalAcceptor { // Update walreceiver state in shmem for reporting. if let ProposerAcceptorMessage::Elected(_) = &next_msg { - *walreceiver_guard.get() = WalReceiverState::Streaming; + walreceiver_guard.get().status = WalReceiverStatus::Streaming; } let reply_msg = if matches!(next_msg, ProposerAcceptorMessage::AppendRequest(_)) { diff --git a/safekeeper/src/recovery.rs b/safekeeper/src/recovery.rs index 90ba3d2e16..e8fa6c55f4 100644 --- a/safekeeper/src/recovery.rs +++ b/safekeeper/src/recovery.rs @@ -1,17 +1,41 @@ //! This module implements pulling WAL from peer safekeepers if compute can't //! provide it, i.e. safekeeper lags too much. -use std::sync::Arc; +use std::time::SystemTime; +use std::{fmt, pin::pin, sync::Arc}; -use tokio::{select, time::sleep, time::Duration}; -use tracing::{info, instrument}; +use anyhow::{bail, Context}; +use futures::StreamExt; +use postgres_protocol::message::backend::ReplicationMessage; +use tokio::sync::mpsc::{channel, Receiver, Sender}; +use tokio::time::timeout; +use tokio::{ + select, + time::sleep, + time::{self, Duration}, +}; +use tokio_postgres::replication::ReplicationStream; +use tokio_postgres::types::PgLsn; +use tracing::*; +use utils::{id::NodeId, lsn::Lsn, postgres_client::wal_stream_connection_config}; -use crate::{timeline::Timeline, SafeKeeperConf}; +use crate::receive_wal::{WalAcceptor, REPLY_QUEUE_SIZE}; +use crate::safekeeper::{AppendRequest, AppendRequestHeader}; +use crate::{ + http::routes::TimelineStatus, + receive_wal::MSG_QUEUE_SIZE, + safekeeper::{ + AcceptorProposerMessage, ProposerAcceptorMessage, ProposerElected, Term, TermHistory, + TermLsn, VoteRequest, + }, + timeline::{PeerInfo, Timeline}, + SafeKeeperConf, +}; /// Entrypoint for per timeline task which always runs, checking whether /// recovery for this safekeeper is needed and starting it if so. #[instrument(name = "recovery task", skip_all, fields(ttid = %tli.ttid))] -pub async fn recovery_main(tli: Arc, _conf: SafeKeeperConf) { +pub async fn recovery_main(tli: Arc, conf: SafeKeeperConf) { info!("started"); let mut cancellation_rx = match tli.get_cancellation_rx() { Ok(rx) => rx, @@ -22,19 +46,387 @@ pub async fn recovery_main(tli: Arc, _conf: SafeKeeperConf) { }; select! { - _ = recovery_main_loop(tli) => { unreachable!() } + _ = recovery_main_loop(tli, conf) => { unreachable!() } _ = cancellation_rx.changed() => { info!("stopped"); } } } +/// Result of Timeline::recovery_needed, contains donor(s) if recovery needed and +/// fields to explain the choice. +#[derive(Debug)] +pub struct RecoveryNeededInfo { + /// my term + pub term: Term, + /// my last_log_term + pub last_log_term: Term, + /// my flush_lsn + pub flush_lsn: Lsn, + /// peers from which we can fetch WAL, for observability. + pub peers: Vec, + /// for observability + pub num_streaming_computes: usize, + pub donors: Vec, +} + +// Custom to omit not important fields from PeerInfo. +impl fmt::Display for RecoveryNeededInfo { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{{")?; + write!( + f, + "term: {}, last_log_term: {}, flush_lsn: {}, peers: {{", + self.term, self.last_log_term, self.flush_lsn + )?; + for p in self.peers.iter() { + write!( + f, + "PeerInfo {{ sk_id: {}, term: {}, last_log_term: {}, flush_lsn: {} }}, ", + p.sk_id, p.term, p.last_log_term, p.flush_lsn + )?; + } + write!( + f, + "}} num_streaming_computes: {}, donors: {:?}", + self.num_streaming_computes, self.donors + ) + } +} + +#[derive(Clone, Debug)] +pub struct Donor { + pub sk_id: NodeId, + /// equals to last_log_term + pub term: Term, + pub flush_lsn: Lsn, + pub pg_connstr: String, + pub http_connstr: String, +} + +impl From<&PeerInfo> for Donor { + fn from(p: &PeerInfo) -> Self { + Donor { + sk_id: p.sk_id, + term: p.term, + flush_lsn: p.flush_lsn, + pg_connstr: p.pg_connstr.clone(), + http_connstr: p.http_connstr.clone(), + } + } +} + const CHECK_INTERVAL_MS: u64 = 2000; /// Check regularly whether we need to start recovery. -async fn recovery_main_loop(_tli: Arc) { +async fn recovery_main_loop(tli: Arc, conf: SafeKeeperConf) { let check_duration = Duration::from_millis(CHECK_INTERVAL_MS); loop { + let recovery_needed_info = tli.recovery_needed(conf.heartbeat_timeout).await; + match recovery_needed_info.donors.first() { + Some(donor) => { + info!( + "starting recovery from donor {}: {}", + donor.sk_id, recovery_needed_info + ); + match recover(tli.clone(), donor, &conf).await { + // Note: 'write_wal rewrites WAL written before' error is + // expected here and might happen if compute and recovery + // concurrently write the same data. Eventually compute + // should win. + Err(e) => warn!("recovery failed: {:#}", e), + Ok(msg) => info!("recovery finished: {}", msg), + } + } + None => { + trace!( + "recovery not needed or not possible: {}", + recovery_needed_info + ); + } + } sleep(check_duration).await; } } + +/// Recover from the specified donor. Returns message explaining normal finish +/// reason or error. +async fn recover( + tli: Arc, + donor: &Donor, + conf: &SafeKeeperConf, +) -> anyhow::Result { + // Learn donor term switch history to figure out starting point. + let client = reqwest::Client::new(); + let timeline_info: TimelineStatus = client + .get(format!( + "http://{}/v1/tenant/{}/timeline/{}", + donor.http_connstr, tli.ttid.tenant_id, tli.ttid.timeline_id + )) + .send() + .await? + .json() + .await?; + if timeline_info.acceptor_state.term != donor.term { + bail!( + "donor term changed from {} to {}", + donor.term, + timeline_info.acceptor_state.term + ); + } + // convert from API TermSwitchApiEntry into TermLsn. + let donor_th = TermHistory( + timeline_info + .acceptor_state + .term_history + .iter() + .map(|tl| Into::::into(*tl)) + .collect(), + ); + + // Now understand our term history. + let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: donor.term }); + let vote_response = match tli + .process_msg(&vote_request) + .await + .context("VoteRequest handling")? + { + Some(AcceptorProposerMessage::VoteResponse(vr)) => vr, + _ => { + bail!("unexpected VoteRequest response"); // unreachable + } + }; + if vote_response.term != donor.term { + bail!( + "our term changed from {} to {}", + donor.term, + vote_response.term + ); + } + + let last_common_point = match TermHistory::find_highest_common_point( + &donor_th, + &vote_response.term_history, + vote_response.flush_lsn, + ) { + None => bail!( + "couldn't find common point in histories, donor {:?}, sk {:?}", + donor_th, + vote_response.term_history, + ), + Some(lcp) => lcp, + }; + info!("found last common point at {:?}", last_common_point); + + // truncate WAL locally + let pe = ProposerAcceptorMessage::Elected(ProposerElected { + term: donor.term, + start_streaming_at: last_common_point.lsn, + term_history: donor_th, + timeline_start_lsn: Lsn::INVALID, + }); + // Successful ProposerElected handling always returns None. If term changed, + // we'll find out that during the streaming. Note: it is expected to get + // 'refusing to overwrite correct WAL' here if walproposer reconnected + // concurrently, restart helps here. + tli.process_msg(&pe) + .await + .context("ProposerElected handling")?; + + recovery_stream(tli, donor, last_common_point.lsn, conf).await +} + +// Pull WAL from donor, assuming handshake is already done. +async fn recovery_stream( + tli: Arc, + donor: &Donor, + start_streaming_at: Lsn, + conf: &SafeKeeperConf, +) -> anyhow::Result { + // TODO: pass auth token + let cfg = wal_stream_connection_config(tli.ttid, &donor.pg_connstr, None, None)?; + let mut cfg = cfg.to_tokio_postgres_config(); + // It will make safekeeper give out not committed WAL (up to flush_lsn). + cfg.application_name(&format!("safekeeper_{}", conf.my_id)); + cfg.replication_mode(tokio_postgres::config::ReplicationMode::Physical); + + let connect_timeout = Duration::from_millis(10000); + let (client, connection) = match time::timeout(connect_timeout, cfg.connect(postgres::NoTls)) + .await + { + Ok(client_and_conn) => client_and_conn?, + Err(_elapsed) => { + bail!("timed out while waiting {connect_timeout:?} for connection to peer safekeeper to open"); + } + }; + trace!("connected to {:?}", donor); + + // The connection object performs the actual communication with the + // server, spawn it off to run on its own. + let ttid = tli.ttid; + tokio::spawn(async move { + if let Err(e) = connection + .instrument(info_span!("recovery task connection poll", ttid = %ttid)) + .await + { + // This logging isn't very useful as error is anyway forwarded to client. + trace!( + "tokio_postgres connection object finished with error: {}", + e + ); + } + }); + + let query = format!( + "START_REPLICATION PHYSICAL {} (term='{}')", + start_streaming_at, donor.term + ); + + let copy_stream = client.copy_both_simple(&query).await?; + let physical_stream = ReplicationStream::new(copy_stream); + + // As in normal walreceiver, do networking and writing to disk in parallel. + let (msg_tx, msg_rx) = channel(MSG_QUEUE_SIZE); + let (reply_tx, reply_rx) = channel(REPLY_QUEUE_SIZE); + let wa = WalAcceptor::spawn(tli.clone(), msg_rx, reply_tx, None); + + let res = tokio::select! { + r = network_io(physical_stream, msg_tx, donor.clone(), tli.clone(), conf.clone()) => r, + r = read_replies(reply_rx, donor.term) => r.map(|()| None), + }; + + // Join the spawned WalAcceptor. At this point chans to/from it passed to + // network routines are dropped, so it will exit as soon as it touches them. + match wa.await { + Ok(Ok(())) => { + // WalAcceptor finished normally, termination reason is different + match res { + Ok(Some(success_desc)) => Ok(success_desc), + Ok(None) => bail!("unexpected recovery end without error/success"), // can't happen + Err(e) => Err(e), // network error or term change + } + } + Ok(Err(e)) => Err(e), // error while processing message + Err(e) => bail!("WalAcceptor panicked: {}", e), + } +} + +// Perform network part of streaming: read data and push it to msg_tx, send KA +// to make sender hear from us. If there is nothing coming for a while, check +// for termination. +// Returns +// - Ok(None) if channel to WalAcceptor closed -- its task should return error. +// - Ok(Some(String)) if recovery successfully completed. +// - Err if error happened while reading/writing to socket. +async fn network_io( + physical_stream: ReplicationStream, + msg_tx: Sender, + donor: Donor, + tli: Arc, + conf: SafeKeeperConf, +) -> anyhow::Result> { + let mut physical_stream = pin!(physical_stream); + let mut last_received_lsn = Lsn::INVALID; + // tear down connection if no data arrives withing this period + let no_data_timeout = Duration::from_millis(30000); + + loop { + let msg = match timeout(no_data_timeout, physical_stream.next()).await { + Ok(next) => match next { + None => bail!("unexpected end of replication stream"), + Some(msg) => msg.context("get replication message")?, + }, + Err(_) => bail!("no message received within {:?}", no_data_timeout), + }; + + match msg { + ReplicationMessage::XLogData(xlog_data) => { + let ar_hdr = AppendRequestHeader { + term: donor.term, + epoch_start_lsn: Lsn::INVALID, // unused + begin_lsn: Lsn(xlog_data.wal_start()), + end_lsn: Lsn(xlog_data.wal_start()) + xlog_data.data().len() as u64, + commit_lsn: Lsn::INVALID, // do not attempt to advance, peer communication anyway does it + truncate_lsn: Lsn::INVALID, // do not attempt to advance + proposer_uuid: [0; 16], + }; + let ar = AppendRequest { + h: ar_hdr, + wal_data: xlog_data.into_data(), + }; + trace!( + "processing AppendRequest {}-{}, len {}", + ar.h.begin_lsn, + ar.h.end_lsn, + ar.wal_data.len() + ); + last_received_lsn = ar.h.end_lsn; + if msg_tx + .send(ProposerAcceptorMessage::AppendRequest(ar)) + .await + .is_err() + { + return Ok(None); // chan closed, WalAcceptor terminated + } + } + ReplicationMessage::PrimaryKeepAlive(_) => { + // keepalive means nothing is being streamed for a while. Check whether we need to stop. + let recovery_needed_info = tli.recovery_needed(conf.heartbeat_timeout).await; + // do current donors still contain one we currently connected to? + if !recovery_needed_info + .donors + .iter() + .any(|d| d.sk_id == donor.sk_id) + { + // Most likely it means we are caughtup. + // note: just exiting makes tokio_postgres send CopyFail to the far end. + return Ok(Some(format!( + "terminating at {} as connected safekeeper {} with term {} is not a donor anymore: {}", + last_received_lsn, donor.sk_id, donor.term, recovery_needed_info + ))); + } + } + _ => {} + } + // Send reply to each message to keep connection alive. Ideally we + // should do that once in a while instead, but this again requires + // stream split or similar workaround, and recovery is anyway not that + // performance critical. + // + // We do not know here real write/flush LSNs (need to take mutex again + // or check replies which are read in different future), but neither + // sender much cares about them, so just send last received. + physical_stream + .as_mut() + .standby_status_update( + PgLsn::from(last_received_lsn.0), + PgLsn::from(last_received_lsn.0), + PgLsn::from(last_received_lsn.0), + SystemTime::now(), + 0, + ) + .await?; + } +} + +// Read replies from WalAcceptor. We are not interested much in sending them to +// donor safekeeper, so don't route them anywhere. However, we should check if +// term changes and exit if it does. +// Returns Ok(()) if channel closed, Err in case of term change. +async fn read_replies( + mut reply_rx: Receiver, + donor_term: Term, +) -> anyhow::Result<()> { + loop { + match reply_rx.recv().await { + Some(msg) => { + if let AcceptorProposerMessage::AppendResponse(ar) = msg { + if ar.term != donor_term { + bail!("donor term changed from {} to {}", donor_term, ar.term); + } + } + } + None => return Ok(()), // chan closed, WalAcceptor terminated + } + } +} diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 9c6efeb349..1437bdb50e 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -91,6 +91,69 @@ impl TermHistory { } TermHistory(res) } + + /// Find point of divergence between leader (walproposer) term history and + /// safekeeper. Arguments are not symmetrics as proposer history ends at + /// +infinity while safekeeper at flush_lsn. + /// C version is at walproposer SendProposerElected. + pub fn find_highest_common_point( + prop_th: &TermHistory, + sk_th: &TermHistory, + sk_wal_end: Lsn, + ) -> Option { + let (prop_th, sk_th) = (&prop_th.0, &sk_th.0); // avoid .0 below + + if let Some(sk_th_last) = sk_th.last() { + assert!( + sk_th_last.lsn <= sk_wal_end, + "safekeeper term history end {:?} LSN is higher than WAL end {:?}", + sk_th_last, + sk_wal_end + ); + } + + // find last common term, if any... + let mut last_common_idx = None; + for i in 0..min(sk_th.len(), prop_th.len()) { + if prop_th[i].term != sk_th[i].term { + break; + } + // If term is the same, LSN must be equal as well. + assert!( + prop_th[i].lsn == sk_th[i].lsn, + "same term {} has different start LSNs: prop {}, sk {}", + prop_th[i].term, + prop_th[i].lsn, + sk_th[i].lsn + ); + last_common_idx = Some(i); + } + let last_common_idx = match last_common_idx { + None => return None, // no common point + Some(lci) => lci, + }; + // Now find where it ends at both prop and sk and take min. End of + // (common) term is the start of the next except it is the last one; + // there it is flush_lsn in case of safekeeper or, in case of proposer + // +infinity, so we just take flush_lsn then. + if last_common_idx == prop_th.len() - 1 { + Some(TermLsn { + term: prop_th[last_common_idx].term, + lsn: sk_wal_end, + }) + } else { + let prop_common_term_end = prop_th[last_common_idx + 1].lsn; + let sk_common_term_end = if last_common_idx + 1 < sk_th.len() { + sk_th[last_common_idx + 1].lsn + } else { + sk_wal_end + }; + Some(TermLsn { + term: prop_th[last_common_idx].term, + lsn: min(prop_common_term_end, sk_common_term_end), + }) + } + } } /// Display only latest entries for Debug. @@ -305,19 +368,19 @@ pub struct AcceptorGreeting { /// Vote request sent from proposer to safekeepers #[derive(Debug, Deserialize)] pub struct VoteRequest { - term: Term, + pub term: Term, } /// Vote itself, sent from safekeeper to proposer #[derive(Debug, Serialize)] pub struct VoteResponse { - term: Term, // safekeeper's current term; if it is higher than proposer's, the compute is out of date. + pub term: Term, // safekeeper's current term; if it is higher than proposer's, the compute is out of date. vote_given: u64, // fixme u64 due to padding // Safekeeper flush_lsn (end of WAL) + history of term switches allow // proposer to choose the most advanced one. - flush_lsn: Lsn, + pub flush_lsn: Lsn, truncate_lsn: Lsn, - term_history: TermHistory, + pub term_history: TermHistory, timeline_start_lsn: Lsn, } @@ -760,7 +823,7 @@ where bail!("refusing ProposerElected which is going to overwrite correct WAL: term={}, flush_lsn={}, start_streaming_at={}; restarting the handshake should help", msg.term, self.flush_lsn(), msg.start_streaming_at) } - // Otherwise this shouldn't happen. + // Otherwise we must never attempt to truncate committed data. assert!( msg.start_streaming_at >= self.inmem.commit_lsn, "attempt to truncate committed data: start_streaming_at={}, commit_lsn={}", @@ -1190,4 +1253,65 @@ mod tests { sk.wal_store.truncate_wal(Lsn(3)).await.unwrap(); // imitate the complete record at 3 %) assert_eq!(sk.get_epoch(), 1); } + + #[test] + fn test_find_highest_common_point_none() { + let prop_th = TermHistory(vec![(0, Lsn(1)).into()]); + let sk_th = TermHistory(vec![(1, Lsn(1)).into(), (2, Lsn(2)).into()]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(3),), + None + ); + } + + #[test] + fn test_find_highest_common_point_middle() { + let prop_th = TermHistory(vec![ + (1, Lsn(10)).into(), + (2, Lsn(20)).into(), + (4, Lsn(40)).into(), + ]); + let sk_th = TermHistory(vec![ + (1, Lsn(10)).into(), + (2, Lsn(20)).into(), + (3, Lsn(30)).into(), // sk ends last common term 2 at 30 + ]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(40),), + Some(TermLsn { + term: 2, + lsn: Lsn(30), + }) + ); + } + + #[test] + fn test_find_highest_common_point_sk_end() { + let prop_th = TermHistory(vec![ + (1, Lsn(10)).into(), + (2, Lsn(20)).into(), // last common term 2, sk will end it at 32 sk_end_lsn + (4, Lsn(40)).into(), + ]); + let sk_th = TermHistory(vec![(1, Lsn(10)).into(), (2, Lsn(20)).into()]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(32),), + Some(TermLsn { + term: 2, + lsn: Lsn(32), + }) + ); + } + + #[test] + fn test_find_highest_common_point_walprop() { + let prop_th = TermHistory(vec![(1, Lsn(10)).into(), (2, Lsn(20)).into()]); + let sk_th = TermHistory(vec![(1, Lsn(10)).into(), (2, Lsn(20)).into()]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(32),), + Some(TermLsn { + term: 2, + lsn: Lsn(32), + }) + ); + } } diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index b684083446..cd765dcbce 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -418,10 +418,11 @@ impl SafekeeperPostgresHandler { } info!( - "starting streaming from {:?}, available WAL ends at {}, recovery={}", + "starting streaming from {:?}, available WAL ends at {}, recovery={}, appname={:?}", start_pos, end_pos, - matches!(end_watch, EndWatch::Flush(_)) + matches!(end_watch, EndWatch::Flush(_)), + appname ); // switch to copy diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 6e00e27b91..96d844fa7a 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -11,6 +11,7 @@ use tokio::fs; use serde_with::DisplayFromStr; use std::cmp::max; use std::sync::Arc; +use std::time::Duration; use tokio::sync::{Mutex, MutexGuard}; use tokio::{ sync::{mpsc::Sender, watch}, @@ -27,7 +28,7 @@ use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use crate::receive_wal::WalReceivers; -use crate::recovery::recovery_main; +use crate::recovery::{recovery_main, Donor, RecoveryNeededInfo}; use crate::safekeeper::{ AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState, SafekeeperMemState, ServerInfo, Term, TermLsn, INVALID_TERM, @@ -45,11 +46,12 @@ use crate::{debug_dump, wal_storage}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerInfo { pub sk_id: NodeId, + pub term: Term, /// Term of the last entry. - _last_log_term: Term, + pub last_log_term: Term, /// LSN of the last record. #[serde_as(as = "DisplayFromStr")] - _flush_lsn: Lsn, + pub flush_lsn: Lsn, #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, /// Since which LSN safekeeper has WAL. TODO: remove this once we fill new @@ -61,16 +63,21 @@ pub struct PeerInfo { #[serde(skip)] #[serde(default = "Instant::now")] ts: Instant, + pub pg_connstr: String, + pub http_connstr: String, } impl PeerInfo { fn from_sk_info(sk_info: &SafekeeperTimelineInfo, ts: Instant) -> PeerInfo { PeerInfo { sk_id: NodeId(sk_info.safekeeper_id), - _last_log_term: sk_info.last_log_term, - _flush_lsn: Lsn(sk_info.flush_lsn), + term: sk_info.term, + last_log_term: sk_info.last_log_term, + flush_lsn: Lsn(sk_info.flush_lsn), commit_lsn: Lsn(sk_info.commit_lsn), local_start_lsn: Lsn(sk_info.local_start_lsn), + pg_connstr: sk_info.safekeeper_connstr.clone(), + http_connstr: sk_info.http_connstr.clone(), ts, } } @@ -262,6 +269,20 @@ impl SharedState { availability_zone: conf.availability_zone.clone(), } } + + /// Get our latest view of alive peers status on the timeline. + /// We pass our own info through the broker as well, so when we don't have connection + /// to the broker returned vec is empty. + fn get_peers(&self, heartbeat_timeout: Duration) -> Vec { + let now = Instant::now(); + self.peers_info + .0 + .iter() + // Regard peer as absent if we haven't heard from it within heartbeat_timeout. + .filter(|p| now.duration_since(p.ts) <= heartbeat_timeout) + .cloned() + .collect() + } } #[derive(Debug, thiserror::Error)] @@ -443,7 +464,9 @@ impl Timeline { /// Bootstrap new or existing timeline starting background stasks. pub fn bootstrap(self: &Arc, conf: &SafeKeeperConf) { // Start recovery task which always runs on the timeline. - tokio::spawn(recovery_main(self.clone(), conf.clone())); + if conf.peer_recovery_enabled { + tokio::spawn(recovery_main(self.clone(), conf.clone())); + } } /// Delete timeline from disk completely, by removing timeline directory. Background @@ -677,20 +700,88 @@ impl Timeline { Ok(()) } - /// Get our latest view of alive peers status on the timeline. - /// We pass our own info through the broker as well, so when we don't have connection - /// to the broker returned vec is empty. pub async fn get_peers(&self, conf: &SafeKeeperConf) -> Vec { let shared_state = self.write_shared_state().await; - let now = Instant::now(); - shared_state - .peers_info - .0 - .iter() - // Regard peer as absent if we haven't heard from it within heartbeat_timeout. - .filter(|p| now.duration_since(p.ts) <= conf.heartbeat_timeout) - .cloned() - .collect() + shared_state.get_peers(conf.heartbeat_timeout) + } + + /// Should we start fetching WAL from a peer safekeeper, and if yes, from + /// which? Answer is yes, i.e. .donors is not empty if 1) there is something + /// to fetch, and we can do that without running elections; 2) there is no + /// actively streaming compute, as we don't want to compete with it. + /// + /// If donor(s) are choosen, theirs last_log_term is guaranteed to be equal + /// to its last_log_term so we are sure such a leader ever had been elected. + /// + /// All possible donors are returned so that we could keep connection to the + /// current one if it is good even if it slightly lags behind. + /// + /// Note that term conditions above might be not met, but safekeepers are + /// still not aligned on last flush_lsn. Generally in this case until + /// elections are run it is not possible to say which safekeeper should + /// recover from which one -- history which would be committed is different + /// depending on assembled quorum (e.g. classic picture 8 from Raft paper). + /// Thus we don't try to predict it here. + pub async fn recovery_needed(&self, heartbeat_timeout: Duration) -> RecoveryNeededInfo { + let ss = self.write_shared_state().await; + let term = ss.sk.state.acceptor_state.term; + let last_log_term = ss.sk.get_epoch(); + let flush_lsn = ss.sk.flush_lsn(); + // note that peers contain myself, but that's ok -- we are interested only in peers which are strictly ahead of us. + let mut peers = ss.get_peers(heartbeat_timeout); + // Sort by pairs. + peers.sort_by(|p1, p2| { + let tl1 = TermLsn { + term: p1.last_log_term, + lsn: p1.flush_lsn, + }; + let tl2 = TermLsn { + term: p2.last_log_term, + lsn: p2.flush_lsn, + }; + tl2.cmp(&tl1) // desc + }); + let num_streaming_computes = self.walreceivers.get_num_streaming(); + let donors = if num_streaming_computes > 0 { + vec![] // If there is a streaming compute, don't try to recover to not intervene. + } else { + peers + .iter() + .filter_map(|candidate| { + // Are we interested in this candidate? + let candidate_tl = TermLsn { + term: candidate.last_log_term, + lsn: candidate.flush_lsn, + }; + let my_tl = TermLsn { + term: last_log_term, + lsn: flush_lsn, + }; + if my_tl < candidate_tl { + // Yes, we are interested. Can we pull from it without + // (re)running elections? It is possible if 1) his term + // is equal to his last_log_term so we could act on + // behalf of leader of this term (we must be sure he was + // ever elected) and 2) our term is not higher, or we'll refuse data. + if candidate.term == candidate.last_log_term && candidate.term >= term { + Some(Donor::from(candidate)) + } else { + None + } + } else { + None + } + }) + .collect() + }; + RecoveryNeededInfo { + term, + last_log_term, + flush_lsn, + peers, + num_streaming_computes, + donors, + } } pub fn get_walsenders(&self) -> &Arc { diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index aaac4570b0..0b97ddf048 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -400,8 +400,11 @@ def test_wal_removal(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): def wait(f, desc, timeout=30, wait_f=None): started_at = time.time() while True: - if f(): - break + try: + if f(): + break + except Exception: + pass elapsed = time.time() - started_at if elapsed > timeout: raise RuntimeError(f"timed out waiting {elapsed:.0f}s for {desc}") @@ -984,6 +987,85 @@ def test_restart_endpoint(neon_env_builder: NeonEnvBuilder): endpoint.start() +# is timeline flush_lsn equal on provided safekeepers? +def is_flush_lsn_aligned(sk1_http_cli, sk2_http_cli, tenant_id, timeline_id): + status1 = sk1_http_cli.timeline_status(tenant_id, timeline_id) + status2 = sk2_http_cli.timeline_status(tenant_id, timeline_id) + log.info( + f"waiting for flush_lsn alignment, sk1.flush_lsn={status1.flush_lsn}, sk2.flush_lsn={status2.flush_lsn}" + ) + return status1.flush_lsn == status2.flush_lsn + + +# Test behaviour with one safekeeper down and missing a lot of WAL. Namely, that +# 1) walproposer can't recover node if it misses WAL written by previous computes, but +# still starts up and functions normally if two other sks are ok. +# 2) walproposer doesn't keep WAL after some threshold (pg_wal bloat is limited), but functions +# normally if two other sks are ok. +# 3) Lagged safekeeper can still recover by peer recovery. +def test_one_sk_down(neon_env_builder: NeonEnvBuilder): + pass + + +# Smaller version of test_one_sk_down testing peer recovery in isolation: that +# it works without compute at all. +def test_peer_recovery(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_peer_recovery") + endpoint = env.endpoints.create_start("test_peer_recovery") + + endpoint.safe_psql("create table t(key int, value text)") + sk1 = env.safekeepers[0] + sk2 = env.safekeepers[1] + sk1_http_cli = sk1.http_client() + sk2_http_cli = sk2.http_client() + # ensure tli gets created on sk1, peer recovery won't do that + wait( + partial(is_flush_lsn_aligned, sk1_http_cli, sk2_http_cli, tenant_id, timeline_id), + "flush_lsn to get aligned", + ) + + sk1 = env.safekeepers[0] + sk1.stop() + + # roughly fills one segment + endpoint.safe_psql("insert into t select generate_series(1,250000), 'payload'") + + endpoint.stop() # stop compute + + # now start safekeeper, but with peer recovery disabled; it should lag for about a segment + sk1.start(extra_opts=["--peer-recovery=false"]) + sk1_tli_status = sk1_http_cli.timeline_status(tenant_id, timeline_id) + sk2_tli_status = sk2_http_cli.timeline_status(tenant_id, timeline_id) + log.info( + f"flush_lsns after insertion: sk1={sk1_tli_status.flush_lsn}, sk2={sk2_tli_status.flush_lsn}" + ) + assert sk2_tli_status.flush_lsn - sk1_tli_status.flush_lsn >= 16 * 1024 * 1024 + + # wait a bit, lsns shouldn't change + # time.sleep(5) + sk1_tli_status = sk1_http_cli.timeline_status(tenant_id, timeline_id) + sk2_tli_status = sk2_http_cli.timeline_status(tenant_id, timeline_id) + log.info( + f"flush_lsns after waiting: sk1={sk1_tli_status.flush_lsn}, sk2={sk2_tli_status.flush_lsn}" + ) + assert sk2_tli_status.flush_lsn - sk1_tli_status.flush_lsn >= 16 * 1024 * 1024 + + # now restart safekeeper with peer recovery enabled and wait for recovery + sk1.stop().start(extra_opts=["--peer-recovery=true"]) + wait( + partial(is_flush_lsn_aligned, sk1_http_cli, sk2_http_cli, tenant_id, timeline_id), + "flush_lsn to get aligned", + ) + # stop one of safekeepers which weren't recovering and insert a bit more + env.safekeepers[2].stop() + endpoint = env.endpoints.create_start("test_peer_recovery") + endpoint.safe_psql("insert into t select generate_series(1,100), 'payload'") + + class SafekeeperEnv: def __init__( self, From 1b53b3e200c7516f111ab074b41b4ca994777a6b Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Sun, 17 Sep 2023 07:21:34 +0300 Subject: [PATCH 35/49] Make test_pageserver_http_get_wal_receiver_success not wait for keepalive. --- test_runner/regress/test_pageserver_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_pageserver_api.py b/test_runner/regress/test_pageserver_api.py index f5bcfd52f9..2d83788193 100644 --- a/test_runner/regress/test_pageserver_api.py +++ b/test_runner/regress/test_pageserver_api.py @@ -157,6 +157,8 @@ def test_pageserver_http_get_wal_receiver_success(neon_simple_env: NeonEnv): tenant_id, timeline_id = env.neon_cli.create_tenant() endpoint = env.endpoints.create_start(DEFAULT_BRANCH_NAME, tenant_id=tenant_id) + # insert something to force sk -> ps message + endpoint.safe_psql("CREATE TABLE t(key int primary key, value text)") # Wait to make sure that we get a latest WAL receiver data. # We need to wait here because it's possible that we don't have access to # the latest WAL yet, when the `timeline_detail` API is first called. @@ -168,7 +170,7 @@ def test_pageserver_http_get_wal_receiver_success(neon_simple_env: NeonEnv): ) # Make a DB modification then expect getting a new WAL receiver's data. - endpoint.safe_psql("CREATE TABLE t(key int primary key, value text)") + endpoint.safe_psql("INSERT INTO t VALUES (1, 'hey')") wait_until( number_of_iterations=5, interval=1, From 702382e99a7d71ab5c4e2f4d4e33a0e0dc9e42d8 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 20 Sep 2023 13:34:44 +0300 Subject: [PATCH 36/49] Add check that WAL segments are identical after recovery. --- test_runner/fixtures/neon_fixtures.py | 14 +++++++++++ test_runner/regress/test_wal_acceptor.py | 32 +++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index a9fe5e376e..2d73972eba 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2758,6 +2758,20 @@ class Safekeeper: def data_dir(self) -> str: return os.path.join(self.env.repo_dir, "safekeepers", f"sk{self.id}") + def timeline_dir(self, tenant_id, timeline_id) -> str: + return os.path.join(self.data_dir(), str(tenant_id), str(timeline_id)) + + def list_segments(self, tenant_id, timeline_id) -> List[str]: + """ + Get list of segment names of the given timeline. + """ + tli_dir = self.timeline_dir(tenant_id, timeline_id) + segments = [] + for _, _, filenames in os.walk(tli_dir): + segments.extend([f for f in filenames if f != "safekeeper.control"]) + segments.sort() + return segments + @dataclass class SafekeeperTimelineStatus: diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 0b97ddf048..fd6baa22f3 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -1,3 +1,4 @@ +import filecmp import os import pathlib import random @@ -1060,7 +1061,36 @@ def test_peer_recovery(neon_env_builder: NeonEnvBuilder): partial(is_flush_lsn_aligned, sk1_http_cli, sk2_http_cli, tenant_id, timeline_id), "flush_lsn to get aligned", ) - # stop one of safekeepers which weren't recovering and insert a bit more + + # check that WALs are identic after recovery + segs = sk1.list_segments(tenant_id, timeline_id) + log.info(f"segs are {segs}") + + (_, mismatch, not_regular) = filecmp.cmpfiles( + sk1.timeline_dir(tenant_id, timeline_id), + sk2.timeline_dir(tenant_id, timeline_id), + segs, + shallow=False, + ) + log.info( + f"filecmp result mismatch and not regular files:\n\t mismatch={mismatch}\n\t not_regular={not_regular}" + ) + + for f in mismatch: + f1 = os.path.join(sk1.timeline_dir(tenant_id, timeline_id), f) + f2 = os.path.join(sk2.timeline_dir(tenant_id, timeline_id), f) + stdout_filename = "{}.filediff".format(f2) + + with open(stdout_filename, "w") as stdout_f: + subprocess.run("xxd {} > {}.hex ".format(f1, f1), shell=True) + subprocess.run("xxd {} > {}.hex ".format(f2, f2), shell=True) + + cmd = "diff {}.hex {}.hex".format(f1, f2) + subprocess.run([cmd], stdout=stdout_f, shell=True) + + assert (mismatch, not_regular) == ([], []) + + # stop one of safekeepers which weren't recovering and insert a bit more to check we can commit env.safekeepers[2].stop() endpoint = env.endpoints.create_start("test_peer_recovery") endpoint.safe_psql("insert into t select generate_series(1,100), 'payload'") From 2fbd5ab0755c060a8ae12d2d933b022ff9fde7bf Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 4 Oct 2023 12:50:11 +0300 Subject: [PATCH 37/49] Add safekeeper test_late_init. --- test_runner/regress/test_wal_acceptor.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index fd6baa22f3..05c60eb102 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -988,6 +988,33 @@ def test_restart_endpoint(neon_env_builder: NeonEnvBuilder): endpoint.start() +# Test that we can create timeline with one safekeeper down and initialize it +# later when some data already had been written. +def test_late_init(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + + sk1 = env.safekeepers[0] + sk1.stop() + + # create and insert smth while safekeeper is down... + env.neon_cli.create_branch("test_late_init") + endpoint = env.endpoints.create_start("test_late_init") + endpoint.safe_psql("create table t(key int, value text)") + endpoint.safe_psql("insert into t select generate_series(1, 1000), 'payload'") + log.info("insert with safekeeper down done") + endpoint.stop() # stop compute + + # stop another safekeeper, and start one which missed timeline creation + sk2 = env.safekeepers[1] + sk2.stop() + sk1.start() + + # insert some more + endpoint = env.endpoints.create_start("test_late_init") + endpoint.safe_psql("insert into t select generate_series(1,100), 'payload'") + + # is timeline flush_lsn equal on provided safekeepers? def is_flush_lsn_aligned(sk1_http_cli, sk2_http_cli, tenant_id, timeline_id): status1 = sk1_http_cli.timeline_status(tenant_id, timeline_id) From a8899e1e0f51b11882d70e2c7c7a176b9beaf173 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 20 Oct 2023 09:15:34 +0100 Subject: [PATCH 38/49] pageserver: apply timeout when waiting for tenant loads (#5601) ## Problem Loading tenants shouldn't hang. However, if it does, we shouldn't let one hung tenant prevent the entire process from starting background jobs. ## Summary of changes Generalize the timeout mechanism that we already applied to loading initial logical sizes: each phase in startup where we wait for a barrier is subject to a timeout, and startup will proceed if it doesn't complete within timeout. Startup metrics will still reflect the time when a phase actually completed, rather than when we skipped it. The code isn't the most beautiful, but that kind of reflects the awkwardness of await'ing on a future and then stashing it to await again later if we time out. I could imagine making this cleaner in future by waiting on a structure that doesn't self-destruct on wait() the way Barrier does, then make InitializationOrder into a structure that manages the series of waits etc. --- pageserver/src/bin/pageserver.rs | 165 ++++++++++++++++++++++--------- 1 file changed, 118 insertions(+), 47 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 76cb0e8ec6..798b9f258b 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -2,6 +2,7 @@ use std::env::{var, VarError}; use std::sync::Arc; +use std::time::Duration; use std::{env, ops::ControlFlow, str::FromStr}; use anyhow::{anyhow, Context}; @@ -200,6 +201,51 @@ fn initialize_config( }) } +struct WaitForPhaseResult { + timeout_remaining: Duration, + skipped: Option, +} + +/// During startup, we apply a timeout to our waits for readiness, to avoid +/// stalling the whole service if one Tenant experiences some problem. Each +/// phase may consume some of the timeout: this function returns the updated +/// timeout for use in the next call. +async fn wait_for_phase(phase: &str, mut fut: F, timeout: Duration) -> WaitForPhaseResult +where + F: std::future::Future + Unpin, +{ + let initial_t = Instant::now(); + let skipped = match tokio::time::timeout(timeout, &mut fut).await { + Ok(_) => None, + Err(_) => { + tracing::info!( + timeout_millis = timeout.as_millis(), + %phase, + "Startup phase timed out, proceeding anyway" + ); + Some(fut) + } + }; + + WaitForPhaseResult { + timeout_remaining: timeout + .checked_sub(Instant::now().duration_since(initial_t)) + .unwrap_or(Duration::ZERO), + skipped, + } +} + +fn startup_checkpoint(started_at: Instant, phase: &str, human_phase: &str) { + let elapsed = started_at.elapsed(); + let secs = elapsed.as_secs_f64(); + STARTUP_DURATION.with_label_values(&[phase]).set(secs); + + info!( + elapsed_ms = elapsed.as_millis(), + "{human_phase} ({secs:.3}s since start)" + ) +} + fn start_pageserver( launch_ts: &'static LaunchTimestamp, conf: &'static PageServerConf, @@ -207,16 +253,6 @@ fn start_pageserver( // Monotonic time for later calculating startup duration let started_startup_at = Instant::now(); - let startup_checkpoint = move |phase: &str, human_phase: &str| { - let elapsed = started_startup_at.elapsed(); - let secs = elapsed.as_secs_f64(); - STARTUP_DURATION.with_label_values(&[phase]).set(secs); - info!( - elapsed_ms = elapsed.as_millis(), - "{human_phase} ({secs:.3}s since start)" - ) - }; - // Print version and launch timestamp to the log, // and expose them as prometheus metrics. // A changed version string indicates changed software. @@ -341,7 +377,7 @@ fn start_pageserver( // Up to this point no significant I/O has been done: this should have been fast. Record // duration prior to starting I/O intensive phase of startup. - startup_checkpoint("initial", "Starting loading tenants"); + startup_checkpoint(started_startup_at, "initial", "Starting loading tenants"); STARTUP_IS_LOADING.set(1); // Startup staging or optimizing: @@ -388,58 +424,93 @@ fn start_pageserver( let shutdown_pageserver = shutdown_pageserver.clone(); let drive_init = async move { // NOTE: unlike many futures in pageserver, this one is cancellation-safe - let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial load completed")); + let guard = scopeguard::guard_on_success((), |_| { + tracing::info!("Cancelled before initial load completed") + }); - init_remote_done_rx.wait().await; - startup_checkpoint("initial_tenant_load_remote", "Remote part of initial load completed"); + let timeout = conf.background_task_maximum_delay; - init_done_rx.wait().await; - startup_checkpoint("initial_tenant_load", "Initial load completed"); - STARTUP_IS_LOADING.set(0); + let init_remote_done = std::pin::pin!(async { + init_remote_done_rx.wait().await; + startup_checkpoint( + started_startup_at, + "initial_tenant_load_remote", + "Remote part of initial load completed", + ); + }); + + let WaitForPhaseResult { + timeout_remaining: timeout, + skipped: init_remote_skipped, + } = wait_for_phase("initial_tenant_load_remote", init_remote_done, timeout).await; + + let init_load_done = std::pin::pin!(async { + init_done_rx.wait().await; + startup_checkpoint( + started_startup_at, + "initial_tenant_load", + "Initial load completed", + ); + STARTUP_IS_LOADING.set(0); + }); + + let WaitForPhaseResult { + timeout_remaining: timeout, + skipped: init_load_skipped, + } = wait_for_phase("initial_tenant_load", init_load_done, timeout).await; // initial logical sizes can now start, as they were waiting on init_done_rx. scopeguard::ScopeGuard::into_inner(guard); - let mut init_sizes_done = std::pin::pin!(init_logical_size_done_rx.wait()); + let guard = scopeguard::guard_on_success((), |_| { + tracing::info!("Cancelled before initial logical sizes completed") + }); - let timeout = conf.background_task_maximum_delay; + let logical_sizes_done = std::pin::pin!(async { + init_logical_size_done_rx.wait().await; + startup_checkpoint( + started_startup_at, + "initial_logical_sizes", + "Initial logical sizes completed", + ); + }); - let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial logical sizes completed")); - - let init_sizes_done = match tokio::time::timeout(timeout, &mut init_sizes_done).await { - Ok(_) => { - startup_checkpoint("initial_logical_sizes", "Initial logical sizes completed"); - None - } - Err(_) => { - tracing::info!( - timeout_millis = timeout.as_millis(), - "Initial logical size timeout elapsed; starting background jobs" - ); - Some(init_sizes_done) - } - }; + let WaitForPhaseResult { + timeout_remaining: _, + skipped: logical_sizes_skipped, + } = wait_for_phase("initial_logical_sizes", logical_sizes_done, timeout).await; scopeguard::ScopeGuard::into_inner(guard); - // allow background jobs to start + // allow background jobs to start: we either completed prior stages, or they reached timeout + // and were skipped. It is important that we do not let them block background jobs indefinitely, + // because things like consumption metrics for billing are blocked by this barrier. drop(background_jobs_can_start); - startup_checkpoint("background_jobs_can_start", "Starting background jobs"); - - if let Some(init_sizes_done) = init_sizes_done { - // ending up here is not a bug; at the latest logical sizes will be queried by - // consumption metrics. - let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial logical sizes completed")); - init_sizes_done.await; - - scopeguard::ScopeGuard::into_inner(guard); - - startup_checkpoint("initial_logical_sizes", "Initial logical sizes completed after timeout (background jobs already started)"); + startup_checkpoint( + started_startup_at, + "background_jobs_can_start", + "Starting background jobs", + ); + // We are done. If we skipped any phases due to timeout, run them to completion here so that + // they will eventually update their startup_checkpoint, and so that we do not declare the + // 'complete' stage until all the other stages are really done. + let guard = scopeguard::guard_on_success((), |_| { + tracing::info!("Cancelled before waiting for skipped phases done") + }); + if let Some(f) = init_remote_skipped { + f.await; } + if let Some(f) = init_load_skipped { + f.await; + } + if let Some(f) = logical_sizes_skipped { + f.await; + } + scopeguard::ScopeGuard::into_inner(guard); - startup_checkpoint("complete", "Startup complete"); + startup_checkpoint(started_startup_at, "complete", "Startup complete"); }; async move { From b1a11261527c740f8abb222bbcaf8852d62b9b17 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 20 Oct 2023 21:29:17 +0300 Subject: [PATCH 39/49] Grant replication permission to newly created users (#5615) ## Problem ## Summary of changes ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist Co-authored-by: Konstantin Knizhnik --- compute_tools/src/spec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 591cbe90c0..f7ca2eb33c 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -302,7 +302,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { } RoleAction::Create => { let mut query: String = format!( - "CREATE ROLE {} CREATEROLE CREATEDB BYPASSRLS IN ROLE neon_superuser", + "CREATE ROLE {} CREATEROLE CREATEDB BYPASSRLS REPLICATION IN ROLE neon_superuser", name.pg_quote() ); info!("role create query: '{}'", &query); From 11e523f503d1639e1283925fc909902c44e70448 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 23 Oct 2023 10:00:13 +0200 Subject: [PATCH 40/49] walredo: fix EGAGAIN/"os error 11" false page reconstruction failures (#5560) Stacked atop https://github.com/neondatabase/neon/pull/5559 Before this PR, there was the following race condition: ``` T1: polls for writeable stdin T1: writes to stdin T1: enters poll for stdout/stderr T2: enters poll for stdin write WALREDO: writes to stderr KERNEL: wakes up T1 and T2 Tx: reads stderr and prints it Ty: reads stderr and gets EAGAIN (valid values for (x, y) are (1, 2) or (2, 1)) ``` The concrete symptom that we observed repeatedly was with PG16, which started logging `registered custom resource manager` to stderr always, during startup, thereby giving us repeated opportunity to hit above race condition. PG14 and PG15 didn't log anything to stderr, hence we could have only hit this race condition if there was an actual error happening. This PR fixes the race by moving the reading of stderr into a tokio task. It exits when the stderr is closed by the child process, which in turn happens when the child exits, either by itself or because we killed it. The downside is that the async scheduling can reorder the log messages, which can be seen in the new `test_stderr`, which runs in a single-threaded runtime. I included the output below. Overall I think we should move the entire walredo to async, as Joonas proposed many months ago. This PR's asyncification is just the first step to resolve these false page reconstruction errors. After this is fixed, we should stop printing that annoying stderr message on walredo startup; it causes noise in the pageserver logs. That work is tracked in #5399 . ``` 2023-10-13T19:05:21.878858Z ERROR apply_wal_records{tenant_id=d546fb76ba529195392fb4d19e243991 pid=753986}: failed to write out the walredo errored input: No such file or directory (os error 2) target=walredo-1697223921878-1132-0.walredo length=1132 2023-10-13T19:05:21.878932Z DEBUG postgres applied 2 WAL records (1062 bytes) in 114666 us to reconstruct page image at LSN 0/0 2023-10-13T19:05:21.878942Z ERROR error applying 2 WAL records 0/16A9388..0/16D4080 (1062 bytes) to base image with LSN 0/0 to reconstruct page image at LSN 0/0 n_attempts=0: apply_wal_records Caused by: WAL redo process closed its stdout unexpectedly 2023-10-13T19:05:21.879027Z INFO kill_and_wait_impl{pid=753986}: wait successful exit_status=signal: 11 (SIGSEGV) (core dumped) 2023-10-13T19:05:21.879079Z DEBUG wal-redo-postgres-stderr{pid=753986 tenant_id=d546fb76ba529195392fb4d19e243991 pg_version=16}: wal-redo-postgres stderr_logger_task started 2023-10-13T19:05:21.879104Z ERROR wal-redo-postgres-stderr{pid=753986 tenant_id=d546fb76ba529195392fb4d19e243991 pg_version=16}: received output output="2023-10-13 19:05:21.769 GMT [753986] LOG: registered custom resource manager \"neon\" with ID 134\n" 2023-10-13T19:05:21.879116Z DEBUG wal-redo-postgres-stderr{pid=753986 tenant_id=d546fb76ba529195392fb4d19e243991 pg_version=16}: wal-redo-postgres stderr_logger_task finished 2023-10-13T19:05:22.004439Z ERROR apply_wal_records{tenant_id=d546fb76ba529195392fb4d19e243991 pid=754000}: failed to write out the walredo errored input: No such file or directory (os error 2) target=walredo-1697223922004-1132-0.walredo length=1132 2023-10-13T19:05:22.004493Z DEBUG postgres applied 2 WAL records (1062 bytes) in 125344 us to reconstruct page image at LSN 0/0 2023-10-13T19:05:22.004501Z ERROR error applying 2 WAL records 0/16A9388..0/16D4080 (1062 bytes) to base image with LSN 0/0 to reconstruct page image at LSN 0/0 n_attempts=1: apply_wal_records Caused by: WAL redo process closed its stdout unexpectedly 2023-10-13T19:05:22.004588Z INFO kill_and_wait_impl{pid=754000}: wait successful exit_status=signal: 11 (SIGSEGV) (core dumped) 2023-10-13T19:05:22.004624Z DEBUG wal-redo-postgres-stderr{pid=754000 tenant_id=d546fb76ba529195392fb4d19e243991 pg_version=16}: wal-redo-postgres stderr_logger_task started 2023-10-13T19:05:22.004653Z ERROR wal-redo-postgres-stderr{pid=754000 tenant_id=d546fb76ba529195392fb4d19e243991 pg_version=16}: received output output="2023-10-13 19:05:21.884 GMT [754000] LOG: registered custom resource manager \"neon\" with ID 134\n" 2023-10-13T19:05:22.004666Z DEBUG wal-redo-postgres-stderr{pid=754000 tenant_id=d546fb76ba529195392fb4d19e243991 pg_version=16}: wal-redo-postgres stderr_logger_task finished ``` --- pageserver/src/tenant.rs | 22 +-- pageserver/src/walredo.rs | 197 +++++++++++++++----------- test_runner/fixtures/neon_fixtures.py | 2 +- 3 files changed, 130 insertions(+), 91 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 4dae183aea..8b74515efc 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3668,17 +3668,21 @@ pub(crate) mod harness { static LOG_HANDLE: OnceCell<()> = OnceCell::new(); + pub(crate) fn setup_logging() { + LOG_HANDLE.get_or_init(|| { + logging::init( + logging::LogFormat::Test, + // enable it in case the tests exercise code paths that use + // debug_assert_current_span_has_tenant_and_timeline_id + logging::TracingErrorLayerEnablement::EnableWithRustLogFilter, + ) + .expect("Failed to init test logging") + }); + } + impl TenantHarness { pub fn create(test_name: &'static str) -> anyhow::Result { - LOG_HANDLE.get_or_init(|| { - logging::init( - logging::LogFormat::Test, - // enable it in case in case the tests exercise code paths that use - // debug_assert_current_span_has_tenant_and_timeline_id - logging::TracingErrorLayerEnablement::EnableWithRustLogFilter, - ) - .expect("Failed to init test logging") - }); + setup_logging(); let repo_dir = PageServerConf::test_repo_dir(test_name); let _ = fs::remove_dir_all(&repo_dir); diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 7e61a1dc37..cffd912e16 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -27,13 +27,14 @@ use std::collections::VecDeque; use std::io; use std::io::prelude::*; use std::ops::{Deref, DerefMut}; -use std::os::unix::io::{AsRawFd, RawFd}; +use std::os::unix::io::AsRawFd; use std::os::unix::prelude::CommandExt; use std::process::Stdio; -use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; +use std::process::{Child, ChildStdin, ChildStdout, Command}; use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use std::time::Duration; use std::time::Instant; +use tokio_util::sync::CancellationToken; use tracing::*; use utils::{bin_ser::BeSer, id::TenantId, lsn::Lsn, nonblock::set_nonblock}; @@ -47,7 +48,6 @@ use crate::metrics::{ }; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; use crate::repository::Key; -use crate::task_mgr::BACKGROUND_RUNTIME; use crate::walrecord::NeonWalRecord; use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::pg_constants; @@ -72,8 +72,6 @@ pub(crate) struct BufferTag { struct ProcessInput { stdin: ChildStdin, - stderr_fd: RawFd, - stdout_fd: RawFd, n_requests: usize, } @@ -121,6 +119,7 @@ impl PostgresRedoManager { /// The WAL redo is handled by a separate thread, so this just sends a request /// to the thread and waits for response. /// + /// CANCEL SAFETY: NOT CANCEL SAFE. pub async fn request_redo( &self, key: Key, @@ -153,6 +152,7 @@ impl PostgresRedoManager { self.conf.wal_redo_timeout, pg_version, ) + .await }; img = Some(result?); @@ -173,6 +173,7 @@ impl PostgresRedoManager { self.conf.wal_redo_timeout, pg_version, ) + .await } } } @@ -194,7 +195,7 @@ impl PostgresRedoManager { /// Process one request for WAL redo using wal-redo postgres /// #[allow(clippy::too_many_arguments)] - fn apply_batch_postgres( + async fn apply_batch_postgres( &self, key: Key, lsn: Lsn, @@ -283,19 +284,20 @@ impl PostgresRedoManager { ); // Avoid concurrent callers hitting the same issue. // We can't prevent it from happening because we want to enable parallelism. - let mut guard = self.redo_process.write().unwrap(); - match &*guard { - Some(current_field_value) => { - if Arc::ptr_eq(current_field_value, &proc) { - // We're the first to observe an error from `proc`, it's our job to take it out of rotation. - *guard = None; + { + let mut guard = self.redo_process.write().unwrap(); + match &*guard { + Some(current_field_value) => { + if Arc::ptr_eq(current_field_value, &proc) { + // We're the first to observe an error from `proc`, it's our job to take it out of rotation. + *guard = None; + } + } + None => { + // Another thread was faster to observe the error, and already took the process out of rotation. } } - None => { - // Another thread was faster to observe the error, and already took the process out of rotation. - } } - drop(guard); // NB: there may still be other concurrent threads using `proc`. // The last one will send SIGKILL when the underlying Arc reaches refcount 0. // NB: it's important to drop(proc) after drop(guard). Otherwise we'd keep @@ -308,7 +310,12 @@ impl PostgresRedoManager { // than we can SIGKILL & `wait` for them to exit. By doing it the way we do here, // we limit this risk of run-away to at most $num_runtimes * $num_executor_threads. // This probably needs revisiting at some later point. + let mut wait_done = proc.stderr_logger_task_done.clone(); drop(proc); + wait_done + .wait_for(|v| *v) + .await + .expect("we use scopeguard to ensure we always send `true` to the channel before dropping the sender"); } else if n_attempts != 0 { info!(n_attempts, "retried walredo succeeded"); } @@ -619,7 +626,8 @@ struct WalRedoProcess { child: Option, stdout: Mutex, stdin: Mutex, - stderr: Mutex, + stderr_logger_cancel: CancellationToken, + stderr_logger_task_done: tokio::sync::watch::Receiver, /// Counter to separate same sized walredo inputs failing at the same millisecond. #[cfg(feature = "testing")] dump_sequence: AtomicUsize, @@ -668,7 +676,6 @@ impl WalRedoProcess { let stdin = child.stdin.take().unwrap(); let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); - macro_rules! set_nonblock_or_log_err { ($file:ident) => {{ let res = set_nonblock($file.as_raw_fd()); @@ -682,16 +689,73 @@ impl WalRedoProcess { set_nonblock_or_log_err!(stdout)?; set_nonblock_or_log_err!(stderr)?; + let mut stderr = tokio::io::unix::AsyncFd::new(stderr).context("AsyncFd::with_interest")?; + // all fallible operations post-spawn are complete, so get rid of the guard let child = scopeguard::ScopeGuard::into_inner(child); + let stderr_logger_cancel = CancellationToken::new(); + let (stderr_logger_task_done_tx, stderr_logger_task_done_rx) = + tokio::sync::watch::channel(false); + tokio::spawn({ + let stderr_logger_cancel = stderr_logger_cancel.clone(); + async move { + scopeguard::defer! { + debug!("wal-redo-postgres stderr_logger_task finished"); + let _ = stderr_logger_task_done_tx.send(true); + } + debug!("wal-redo-postgres stderr_logger_task started"); + loop { + // NB: we purposefully don't do a select! for the cancellation here. + // The cancellation would likely cause us to miss stderr messages. + // We can rely on this to return from .await because when we SIGKILL + // the child, the writing end of the stderr pipe gets closed. + match stderr.readable_mut().await { + Ok(mut guard) => { + let mut errbuf = [0; 16384]; + let res = guard.try_io(|fd| { + use std::io::Read; + fd.get_mut().read(&mut errbuf) + }); + match res { + Ok(Ok(0)) => { + // it closed the stderr pipe + break; + } + Ok(Ok(n)) => { + // The message might not be split correctly into lines here. But this is + // good enough, the important thing is to get the message to the log. + let output = String::from_utf8_lossy(&errbuf[0..n]).to_string(); + error!(output, "received output"); + }, + Ok(Err(e)) => { + error!(error = ?e, "read() error, waiting for cancellation"); + stderr_logger_cancel.cancelled().await; + error!(error = ?e, "read() error, cancellation complete"); + break; + } + Err(e) => { + let _e: tokio::io::unix::TryIoError = e; + // the read() returned WouldBlock, that's expected + } + } + } + Err(e) => { + error!(error = ?e, "read() error, waiting for cancellation"); + stderr_logger_cancel.cancelled().await; + error!(error = ?e, "read() error, cancellation complete"); + break; + } + } + } + }.instrument(tracing::info_span!(parent: None, "wal-redo-postgres-stderr", pid = child.id(), tenant_id = %tenant_id, %pg_version)) + }); + Ok(Self { conf, tenant_id, child: Some(child), stdin: Mutex::new(ProcessInput { - stdout_fd: stdout.as_raw_fd(), - stderr_fd: stderr.as_raw_fd(), stdin, n_requests: 0, }), @@ -700,7 +764,8 @@ impl WalRedoProcess { pending_responses: VecDeque::new(), n_processed_responses: 0, }), - stderr: Mutex::new(stderr), + stderr_logger_cancel, + stderr_logger_task_done: stderr_logger_task_done_rx, #[cfg(feature = "testing")] dump_sequence: AtomicUsize::default(), }) @@ -774,19 +839,11 @@ impl WalRedoProcess { let mut proc = { input }; // TODO: remove this legacy rename, but this keep the patch small. let mut nwrite = 0usize; - // Prepare for calling poll() - let mut pollfds = [ - PollFd::new(proc.stdin.as_raw_fd(), PollFlags::POLLOUT), - PollFd::new(proc.stderr_fd, PollFlags::POLLIN), - PollFd::new(proc.stdout_fd, PollFlags::POLLIN), - ]; + let mut stdin_pollfds = [PollFd::new(proc.stdin.as_raw_fd(), PollFlags::POLLOUT)]; - // We do two things simultaneously: send the old base image and WAL records to - // the child process's stdin and forward any logging - // information that the child writes to its stderr to the page server's log. while nwrite < writebuf.len() { let n = loop { - match nix::poll::poll(&mut pollfds[0..2], wal_redo_timeout.as_millis() as i32) { + match nix::poll::poll(&mut stdin_pollfds[..], wal_redo_timeout.as_millis() as i32) { Err(nix::errno::Errno::EINTR) => continue, res => break res, } @@ -796,31 +853,8 @@ impl WalRedoProcess { anyhow::bail!("WAL redo timed out"); } - // If we have some messages in stderr, forward them to the log. - let err_revents = pollfds[1].revents().unwrap(); - if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { - let mut errbuf: [u8; 16384] = [0; 16384]; - let mut stderr = self.stderr.lock().unwrap(); - let len = stderr.read(&mut errbuf)?; - - // The message might not be split correctly into lines here. But this is - // good enough, the important thing is to get the message to the log. - if len > 0 { - error!( - "wal-redo-postgres: {}", - String::from_utf8_lossy(&errbuf[0..len]) - ); - - // To make sure we capture all log from the process if it fails, keep - // reading from the stderr, before checking the stdout. - continue; - } - } else if err_revents.contains(PollFlags::POLLHUP) { - anyhow::bail!("WAL redo process closed its stderr unexpectedly"); - } - // If 'stdin' is writeable, do write. - let in_revents = pollfds[0].revents().unwrap(); + let in_revents = stdin_pollfds[0].revents().unwrap(); if in_revents & (PollFlags::POLLERR | PollFlags::POLLOUT) != PollFlags::empty() { nwrite += proc.stdin.write(&writebuf[nwrite..])?; } else if in_revents.contains(PollFlags::POLLHUP) { @@ -845,6 +879,7 @@ impl WalRedoProcess { // advancing processed responses number. let mut output = self.stdout.lock().unwrap(); + let mut stdout_pollfds = [PollFd::new(output.stdout.as_raw_fd(), PollFlags::POLLIN)]; let n_processed_responses = output.n_processed_responses; while n_processed_responses + output.pending_responses.len() <= request_no { // We expect the WAL redo process to respond with an 8k page image. We read it @@ -855,7 +890,10 @@ impl WalRedoProcess { // We do two things simultaneously: reading response from stdout // and forward any logging information that the child writes to its stderr to the page server's log. let n = loop { - match nix::poll::poll(&mut pollfds[1..3], wal_redo_timeout.as_millis() as i32) { + match nix::poll::poll( + &mut stdout_pollfds[..], + wal_redo_timeout.as_millis() as i32, + ) { Err(nix::errno::Errno::EINTR) => continue, res => break res, } @@ -865,31 +903,8 @@ impl WalRedoProcess { anyhow::bail!("WAL redo timed out"); } - // If we have some messages in stderr, forward them to the log. - let err_revents = pollfds[1].revents().unwrap(); - if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { - let mut errbuf: [u8; 16384] = [0; 16384]; - let mut stderr = self.stderr.lock().unwrap(); - let len = stderr.read(&mut errbuf)?; - - // The message might not be split correctly into lines here. But this is - // good enough, the important thing is to get the message to the log. - if len > 0 { - error!( - "wal-redo-postgres: {}", - String::from_utf8_lossy(&errbuf[0..len]) - ); - - // To make sure we capture all log from the process if it fails, keep - // reading from the stderr, before checking the stdout. - continue; - } - } else if err_revents.contains(PollFlags::POLLHUP) { - anyhow::bail!("WAL redo process closed its stderr unexpectedly"); - } - // If we have some data in stdout, read it to the result buffer. - let out_revents = pollfds[2].revents().unwrap(); + let out_revents = stdout_pollfds[0].revents().unwrap(); if out_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { nresult += output.stdout.read(&mut resultbuf[nresult..])?; } else if out_revents.contains(PollFlags::POLLHUP) { @@ -985,6 +1000,8 @@ impl Drop for WalRedoProcess { .take() .expect("we only do this once") .kill_and_wait(); + self.stderr_logger_cancel.cancel(); + // no way to wait for stderr_logger_task from Drop because that is async only } } @@ -1066,7 +1083,7 @@ impl Drop for NoLeakChild { // Offload the kill+wait of the child process into the background. // If someone stops the runtime, we'll leak the child process. // We can ignore that case because we only stop the runtime on pageserver exit. - BACKGROUND_RUNTIME.spawn(async move { + tokio::runtime::Handle::current().spawn(async move { tokio::task::spawn_blocking(move || { // Intentionally don't inherit the tracing context from whoever is dropping us. // This thread here is going to outlive of our dropper. @@ -1199,6 +1216,22 @@ mod tests { assert_eq!(page, crate::ZERO_PAGE); } + #[tokio::test] + async fn test_stderr() { + let h = RedoHarness::new().unwrap(); + h + .manager + .request_redo( + Key::from_i128(0), + Lsn::INVALID, + None, + short_records(), + 16, /* 16 currently produces stderr output on startup, which adds a nice extra edge */ + ) + .await + .unwrap_err(); + } + #[allow(clippy::octal_escapes)] fn short_records() -> Vec<(Lsn, NeonWalRecord)> { vec![ @@ -1227,6 +1260,8 @@ mod tests { impl RedoHarness { fn new() -> anyhow::Result { + crate::tenant::harness::setup_logging(); + let repo_dir = camino_tempfile::tempdir()?; let conf = PageServerConf::dummy_conf(repo_dir.path().to_path_buf()); let conf = Box::leak(Box::new(conf)); diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 2d73972eba..38e17985ac 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1631,7 +1631,7 @@ class NeonPageserver(PgProtocol): ".*took more than expected to complete.*", # these can happen during shutdown, but it should not be a reason to fail a test ".*completed, took longer than expected.*", - '.*registered custom resource manager "neon".*', + '.*registered custom resource manager \\\\"neon\\\\".*', # AWS S3 may emit 500 errors for keys in a DeleteObjects response: we retry these # and it is not a failure of our code when it happens. ".*DeleteObjects.*We encountered an internal error. Please try again.*", From 41ee75bc7149688659999c429e8da6847e3e0f36 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 23 Oct 2023 09:19:01 +0100 Subject: [PATCH 41/49] pageserver: do config writes in a spawn_blocking (#5603) ## Problem We now persist tenant configuration every time we spawn a tenant. The persist_tenant_config function is doing a series of non-async filesystem I/O, because `crashsafe::` isn't async yet. This isn't a demonstrated problem, but is a source of uncertainty when reasoning about what's happening with our startup times. ## Summary of changes - Wrap `crashsafe_overwrite` in `spawn_blocking`. - Although I think this change makes sense, it does not have a measurable impact on load time when testing with 10k tenants. - This can be reverted when we have full async I/O --- pageserver/src/tenant.rs | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 8b74515efc..82f82dcd09 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -18,6 +18,7 @@ use pageserver_api::models::TimelineState; use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; use storage_broker::BrokerClientChannel; +use tokio::runtime::Handle; use tokio::sync::watch; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; @@ -2614,6 +2615,7 @@ impl Tenant { ) -> anyhow::Result<()> { let legacy_config_path = conf.tenant_config_path(tenant_id); let config_path = conf.tenant_location_config_path(tenant_id); + Self::persist_tenant_config_at(tenant_id, &config_path, &legacy_config_path, location_conf) .await } @@ -2652,12 +2654,20 @@ impl Tenant { // Convert the config to a toml file. conf_content += &toml_edit::ser::to_string_pretty(&location_conf)?; - let conf_content = conf_content.as_bytes(); - let temp_path = path_with_suffix_extension(config_path, TEMP_FILE_SUFFIX); - VirtualFile::crashsafe_overwrite(config_path, &temp_path, conf_content) - .await - .with_context(|| format!("write tenant {tenant_id} config to {config_path}"))?; + + let tenant_id = *tenant_id; + let config_path = config_path.to_owned(); + tokio::task::spawn_blocking(move || { + Handle::current().block_on(async move { + let conf_content = conf_content.as_bytes(); + VirtualFile::crashsafe_overwrite(&config_path, &temp_path, conf_content) + .await + .with_context(|| format!("write tenant {tenant_id} config to {config_path}")) + }) + }) + .await??; + Ok(()) } @@ -2679,12 +2689,21 @@ impl Tenant { // Convert the config to a toml file. conf_content += &toml_edit::ser::to_string(&tenant_conf)?; - let conf_content = conf_content.as_bytes(); - let temp_path = path_with_suffix_extension(target_config_path, TEMP_FILE_SUFFIX); - VirtualFile::crashsafe_overwrite(target_config_path, &temp_path, conf_content) - .await - .with_context(|| format!("write tenant {tenant_id} config to {target_config_path}"))?; + + let tenant_id = *tenant_id; + let target_config_path = target_config_path.to_owned(); + tokio::task::spawn_blocking(move || { + Handle::current().block_on(async move { + let conf_content = conf_content.as_bytes(); + VirtualFile::crashsafe_overwrite(&target_config_path, &temp_path, conf_content) + .await + .with_context(|| { + format!("write tenant {tenant_id} config to {target_config_path}") + }) + }) + }) + .await??; Ok(()) } From 7d17f1719f599ebbe30a6a4f816cb1b732608fa1 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 23 Oct 2023 14:12:41 +0100 Subject: [PATCH 42/49] reduce cancel map contention (#5555) ## Problem Every database request locks this cancel map rwlock. At high requests per second this would have high contention ## Summary of changes Switch to dashmap which has a sharded rwlock to reduce contention --- proxy/src/cancellation.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/proxy/src/cancellation.rs b/proxy/src/cancellation.rs index 8d16f202e9..a5eb3544b4 100644 --- a/proxy/src/cancellation.rs +++ b/proxy/src/cancellation.rs @@ -1,5 +1,5 @@ -use anyhow::{anyhow, Context}; -use hashbrown::HashMap; +use anyhow::{bail, Context}; +use dashmap::DashMap; use pq_proto::CancelKeyData; use std::net::SocketAddr; use tokio::net::TcpStream; @@ -8,7 +8,7 @@ use tracing::info; /// Enables serving `CancelRequest`s. #[derive(Default)] -pub struct CancelMap(parking_lot::RwLock>>); +pub struct CancelMap(DashMap>); impl CancelMap { /// Cancel a running query for the corresponding connection. @@ -16,7 +16,6 @@ impl CancelMap { // NB: we should immediately release the lock after cloning the token. let cancel_closure = self .0 - .read() .get(&key) .and_then(|x| x.clone()) .with_context(|| format!("query cancellation key not found: {key}"))?; @@ -40,15 +39,19 @@ impl CancelMap { // Random key collisions are unlikely to happen here, but they're still possible, // which is why we have to take care not to rewrite an existing key. - self.0 - .write() - .try_insert(key, None) - .map_err(|_| anyhow!("query cancellation key already exists: {key}"))?; + match self.0.entry(key) { + dashmap::mapref::entry::Entry::Occupied(_) => { + bail!("query cancellation key already exists: {key}") + } + dashmap::mapref::entry::Entry::Vacant(e) => { + e.insert(None); + } + } // This will guarantee that the session gets dropped // as soon as the future is finished. scopeguard::defer! { - self.0.write().remove(&key); + self.0.remove(&key); info!("dropped query cancellation key {key}"); } @@ -59,12 +62,12 @@ impl CancelMap { #[cfg(test)] fn contains(&self, session: &Session) -> bool { - self.0.read().contains_key(&session.key) + self.0.contains_key(&session.key) } #[cfg(test)] fn is_empty(&self) -> bool { - self.0.read().is_empty() + self.0.is_empty() } } @@ -113,10 +116,7 @@ impl Session<'_> { /// This enables query cancellation in `crate::proxy::prepare_client_connection`. pub fn enable_query_cancellation(self, cancel_closure: CancelClosure) -> CancelKeyData { info!("enabling query cancellation for this session"); - self.cancel_map - .0 - .write() - .insert(self.key, Some(cancel_closure)); + self.cancel_map.0.insert(self.key, Some(cancel_closure)); self.key } From b514da90cb36e6c7aa0ed95c39c43aa1848983bd Mon Sep 17 00:00:00 2001 From: khanova <32508607+khanova@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:11:05 +0200 Subject: [PATCH 43/49] Set up timeout for scram protocol execution (#5551) ## Problem Context: https://github.com/neondatabase/neon/issues/5511#issuecomment-1759649679 Some of out scram protocol execution timed out only after 17 minutes. ## Summary of changes Make timeout for scram execution meaningful and configurable. --- proxy/src/auth/backend.rs | 9 ++++++--- proxy/src/auth/backend/classic.rs | 13 ++++++++++++- proxy/src/bin/proxy.rs | 9 ++++++++- proxy/src/config.rs | 5 +++++ proxy/src/proxy.rs | 7 ++++--- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index 03c9029862..aa19acf65a 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -6,6 +6,7 @@ pub use link::LinkAuthError; use crate::{ auth::{self, ClientCredentials}, + config::AuthenticationConfig, console::{ self, provider::{CachedNodeInfo, ConsoleReqExtra}, @@ -124,6 +125,7 @@ async fn auth_quirks( creds: &mut ClientCredentials<'_>, client: &mut stream::PqStream, allow_cleartext: bool, + config: &'static AuthenticationConfig, ) -> auth::Result> { // If there's no project so far, that entails that client doesn't // support SNI or other means of passing the endpoint (project) name. @@ -145,7 +147,7 @@ async fn auth_quirks( } // Finally, proceed with the main auth flow (SCRAM-based). - classic::authenticate(api, extra, creds, client).await + classic::authenticate(api, extra, creds, client, config).await } impl BackendType<'_, ClientCredentials<'_>> { @@ -180,6 +182,7 @@ impl BackendType<'_, ClientCredentials<'_>> { extra: &ConsoleReqExtra<'_>, client: &mut stream::PqStream, allow_cleartext: bool, + config: &'static AuthenticationConfig, ) -> auth::Result> { use BackendType::*; @@ -192,7 +195,7 @@ impl BackendType<'_, ClientCredentials<'_>> { ); let api = api.as_ref(); - auth_quirks(api, extra, creds, client, allow_cleartext).await? + auth_quirks(api, extra, creds, client, allow_cleartext, config).await? } Postgres(api, creds) => { info!( @@ -202,7 +205,7 @@ impl BackendType<'_, ClientCredentials<'_>> { ); let api = api.as_ref(); - auth_quirks(api, extra, creds, client, allow_cleartext).await? + auth_quirks(api, extra, creds, client, allow_cleartext, config).await? } // NOTE: this auth backend doesn't use client credentials. Link(url) => { diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs index 9a056f1445..af628efc5e 100644 --- a/proxy/src/auth/backend/classic.rs +++ b/proxy/src/auth/backend/classic.rs @@ -4,6 +4,7 @@ use super::AuthSuccess; use crate::{ auth::{self, AuthFlow, ClientCredentials}, compute, + config::AuthenticationConfig, console::{self, AuthInfo, CachedNodeInfo, ConsoleReqExtra}, proxy::{handle_try_wake, retry_after}, sasl, scram, @@ -17,6 +18,7 @@ pub(super) async fn authenticate( extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials<'_>, client: &mut PqStream, + config: &'static AuthenticationConfig, ) -> auth::Result> { info!("fetching user's authentication info"); let info = api.get_auth_info(extra, creds).await?.unwrap_or_else(|| { @@ -42,7 +44,16 @@ pub(super) async fn authenticate( error })?; - let auth_outcome = auth_flow.authenticate().await.map_err(|error| { + let auth_outcome = tokio::time::timeout( + config.scram_protocol_timeout, + auth_flow.authenticate(), + ) + .await + .map_err(|error| { + warn!("error processing scram messages error = authentication timed out, execution time exeeded {} seconds", config.scram_protocol_timeout.as_secs()); + auth::io::Error::new(auth::io::ErrorKind::TimedOut, error) + })? + .map_err(|error| { warn!(?error, "error processing scram messages"); error })?; diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index b5970b22c9..9b54f98402 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -1,5 +1,6 @@ use futures::future::Either; use proxy::auth; +use proxy::config::AuthenticationConfig; use proxy::config::HttpConfig; use proxy::console; use proxy::http; @@ -83,7 +84,9 @@ struct ProxyCliArgs { /// timeout for http connections #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] sql_over_http_timeout: tokio::time::Duration, - + /// timeout for scram authentication protocol + #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] + scram_protocol_timeout: tokio::time::Duration, /// Require that all incoming requests have a Proxy Protocol V2 packet **and** have an IP address associated. #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] require_client_ip: bool, @@ -231,12 +234,16 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { let http_config = HttpConfig { sql_over_http_timeout: args.sql_over_http_timeout, }; + let authentication_config = AuthenticationConfig { + scram_protocol_timeout: args.scram_protocol_timeout, + }; let config = Box::leak(Box::new(ProxyConfig { tls_config, auth_backend, metric_collection, allow_self_signed_compute: args.allow_self_signed_compute, http_config, + authentication_config, require_client_ip: args.require_client_ip, })); diff --git a/proxy/src/config.rs b/proxy/src/config.rs index b5ff3b64bd..96856ba051 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -14,6 +14,7 @@ pub struct ProxyConfig { pub metric_collection: Option, pub allow_self_signed_compute: bool, pub http_config: HttpConfig, + pub authentication_config: AuthenticationConfig, pub require_client_ip: bool, } @@ -32,6 +33,10 @@ pub struct HttpConfig { pub sql_over_http_timeout: tokio::time::Duration, } +pub struct AuthenticationConfig { + pub scram_protocol_timeout: tokio::time::Duration, +} + impl TlsConfig { pub fn to_server_config(&self) -> Arc { self.config.clone() diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 907a061f96..12bf1b6a69 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -5,7 +5,7 @@ use crate::{ auth::{self, backend::AuthSuccess}, cancellation::{self, CancelMap}, compute::{self, PostgresConnection}, - config::{ProxyConfig, TlsConfig}, + config::{AuthenticationConfig, ProxyConfig, TlsConfig}, console::{self, errors::WakeComputeError, messages::MetricsAuxInfo, Api}, http::StatusCode, metrics::{Ids, USAGE_METRICS}, @@ -340,7 +340,7 @@ pub async fn handle_client( mode.allow_self_signed_compute(config), ); cancel_map - .with_session(|session| client.connect_to_db(session, mode)) + .with_session(|session| client.connect_to_db(session, mode, &config.authentication_config)) .await } @@ -818,6 +818,7 @@ impl Client<'_, S> { self, session: cancellation::Session<'_>, mode: ClientMode, + config: &'static AuthenticationConfig, ) -> anyhow::Result<()> { let Self { mut stream, @@ -835,7 +836,7 @@ impl Client<'_, S> { let latency_timer = LatencyTimer::new(mode.protocol_label()); let auth_result = match creds - .authenticate(&extra, &mut stream, mode.allow_cleartext()) + .authenticate(&extra, &mut stream, mode.allow_cleartext(), config) .await { Ok(auth_result) => auth_result, From 94b4e76e13a8ad0aed031db091ad664092ce9d11 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 23 Oct 2023 15:17:28 +0100 Subject: [PATCH 44/49] proxy: latency connect outcome (#5588) ## Problem I recently updated the latency timers to include cache miss and pool miss, as well as connection protocol. By moving the latency timer to start before authentication, we count a lot more failures and it's messed up the latency dashboard. ## Summary of changes Add another label to LatencyTimer metrics for outcome. Explicitly report on success --- proxy/src/http/conn_pool.rs | 3 ++- proxy/src/proxy.rs | 28 ++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/proxy/src/http/conn_pool.rs b/proxy/src/http/conn_pool.rs index ed84b0f2bf..7a5f00d627 100644 --- a/proxy/src/http/conn_pool.rs +++ b/proxy/src/http/conn_pool.rs @@ -194,9 +194,10 @@ impl GlobalConnPool { info!("pool: cached connection '{conn_info}' is closed, opening a new one"); connect_to_compute(self.proxy_config, conn_info, session_id, latency_timer).await } else { - latency_timer.pool_hit(); info!("pool: reusing connection '{conn_info}'"); client.session.send(session_id)?; + latency_timer.pool_hit(); + latency_timer.success(); return Ok(Client { inner: Some(client), span: Span::current(), diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 12bf1b6a69..ec878b696e 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -96,7 +96,9 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { register_histogram_vec!( "proxy_compute_connection_latency_seconds", "Time it took for proxy to establish a connection to the compute endpoint", - &["protocol", "cache_miss", "pool_miss"], + // http/ws/tcp, true/false, true/false, success/failure + // 3 * 2 * 2 * 2 = 24 counters + &["protocol", "cache_miss", "pool_miss", "outcome"], // largest bucket = 2^16 * 0.5ms = 32s exponential_buckets(0.0005, 2.0, 16).unwrap(), ) @@ -105,19 +107,22 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { pub struct LatencyTimer { start: Instant, - pool_miss: bool, - cache_miss: bool, protocol: &'static str, + cache_miss: bool, + pool_miss: bool, + outcome: &'static str, } impl LatencyTimer { pub fn new(protocol: &'static str) -> Self { Self { start: Instant::now(), + protocol, cache_miss: false, // by default we don't do pooling pool_miss: true, - protocol, + // assume failed unless otherwise specified + outcome: "failed", } } @@ -128,6 +133,10 @@ impl LatencyTimer { pub fn pool_hit(&mut self) { self.pool_miss = false; } + + pub fn success(mut self) { + self.outcome = "success"; + } } impl Drop for LatencyTimer { @@ -138,6 +147,7 @@ impl Drop for LatencyTimer { self.protocol, bool_to_str(self.cache_miss), bool_to_str(self.pool_miss), + self.outcome, ]) .observe(duration) } @@ -547,7 +557,10 @@ where // try once let (config, err) = match mechanism.connect_once(&node_info, CONNECT_TIMEOUT).await { - Ok(res) => return Ok(res), + Ok(res) => { + latency_timer.success(); + return Ok(res); + } Err(e) => { error!(error = ?e, "could not connect to compute node"); (invalidate_cache(node_info), e) @@ -601,7 +614,10 @@ where info!("wake_compute success. attempting to connect"); loop { match mechanism.connect_once(&node_info, CONNECT_TIMEOUT).await { - Ok(res) => return Ok(res), + Ok(res) => { + latency_timer.success(); + return Ok(res); + } Err(e) => { let retriable = e.should_retry(num_retries); if !retriable { From c6ca1d76d2dc255cb237f82053dc6d74e2d7206e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 23 Oct 2023 16:31:38 +0200 Subject: [PATCH 45/49] consumption_metrics: fix periodicness behavior & reporting (#5625) Before this PR, the ticker was running at default miss behavior `Delay`. For example, here is the startup output with 25k tenants: ``` 2023-10-19T09:57:21.682466Z INFO synthetic_size_worker: starting calculate_synthetic_size_worker 2023-10-19T10:50:44.678202Z WARN synthetic_size_worker: task iteration took longer than the configured period elapsed=3202.995707156s period=10m task=ConsumptionMetricsSyntheticSizeWorker 2023-10-19T10:52:17.408056Z WARN synthetic_size_worker: task iteration took longer than the configured period elapsed=2695.72556035s period=10m task=ConsumptionMetricsSyntheticSizeWorker ``` The first message's `elapsed` value is correct. It matches the delta between the log line timestamps. The second one is logged ca 1.5min after, though, but reports a much larger `elapsed` than 1.5min. This PR fixes the behavior by copying what `eviction_task.rs` does. --- pageserver/src/consumption_metrics.rs | 45 ++++++++++++++++----------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 13f7977946..061045eb76 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -11,6 +11,7 @@ use reqwest::Url; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, SystemTime}; +use tokio::time::Instant; use tracing::*; use utils::id::NodeId; @@ -88,22 +89,12 @@ pub async fn collect_metrics( let node_id = node_id.to_string(); - // reminder: ticker is ready immediatedly - let mut ticker = tokio::time::interval(metric_collection_interval); - loop { - let tick_at = tokio::select! { - _ = cancel.cancelled() => return Ok(()), - tick_at = ticker.tick() => tick_at, - }; + let started_at = Instant::now(); // these are point in time, with variable "now" let metrics = metrics::collect_all_metrics(&cached_metrics, &ctx).await; - if metrics.is_empty() { - continue; - } - let metrics = Arc::new(metrics); // why not race cancellation here? because we are one of the last tasks, and if we are @@ -142,10 +133,19 @@ pub async fn collect_metrics( let (_, _) = tokio::join!(flush, upload); crate::tenant::tasks::warn_when_period_overrun( - tick_at.elapsed(), + started_at.elapsed(), metric_collection_interval, BackgroundLoopKind::ConsumptionMetricsCollectMetrics, ); + + let res = tokio::time::timeout_at( + started_at + metric_collection_interval, + task_mgr::shutdown_token().cancelled(), + ) + .await; + if res.is_ok() { + return Ok(()); + } } } @@ -244,16 +244,14 @@ async fn calculate_synthetic_size_worker( ctx: &RequestContext, ) -> anyhow::Result<()> { info!("starting calculate_synthetic_size_worker"); + scopeguard::defer! { + info!("calculate_synthetic_size_worker stopped"); + }; - // reminder: ticker is ready immediatedly - let mut ticker = tokio::time::interval(synthetic_size_calculation_interval); let cause = LogicalSizeCalculationCause::ConsumptionMetricsSyntheticSize; loop { - let tick_at = tokio::select! { - _ = task_mgr::shutdown_watcher() => return Ok(()), - tick_at = ticker.tick() => tick_at, - }; + let started_at = Instant::now(); let tenants = match mgr::list_tenants().await { Ok(tenants) => tenants, @@ -281,9 +279,18 @@ async fn calculate_synthetic_size_worker( } crate::tenant::tasks::warn_when_period_overrun( - tick_at.elapsed(), + started_at.elapsed(), synthetic_size_calculation_interval, BackgroundLoopKind::ConsumptionMetricsSyntheticSizeWorker, ); + + let res = tokio::time::timeout_at( + started_at + synthetic_size_calculation_interval, + task_mgr::shutdown_token().cancelled(), + ) + .await; + if res.is_ok() { + return Ok(()); + } } } From 7e805200bb94c4826d1b41d86910987fde97a5c5 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 23 Oct 2023 15:32:34 +0100 Subject: [PATCH 46/49] pageserver: parallel load of configs (#5607) ## Problem When the number of tenants is large, sequentially issuing the open/read calls for their config files is a ~1000ms delay during startup. It's not a lot, but it's simple to fix. ## Summary of changes Put all the config loads into spawn_blocking() tasks and run them in a JoinSet. We can simplify this a bit later when we have full async disk I/O. --------- Co-authored-by: Shany Pozin --- pageserver/src/tenant/mgr.rs | 146 +++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 65 deletions(-) diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 3b6039db93..380b610a4c 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -1,7 +1,7 @@ //! This module acts as a switchboard to access different repositories managed by this //! page server. -use camino::{Utf8Path, Utf8PathBuf}; +use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; use rand::{distributions::Alphanumeric, Rng}; use std::collections::{hash_map, HashMap}; use std::sync::Arc; @@ -256,83 +256,99 @@ async fn init_load_generations( Ok(Some(generations)) } +/// Given a directory discovered in the pageserver's tenants/ directory, attempt +/// to load a tenant config from it. +/// +/// If file is missing, return Ok(None) +fn load_tenant_config( + conf: &'static PageServerConf, + dentry: Utf8DirEntry, +) -> anyhow::Result)>> { + let tenant_dir_path = dentry.path().to_path_buf(); + if crate::is_temporary(&tenant_dir_path) { + info!("Found temporary tenant directory, removing: {tenant_dir_path}"); + // No need to use safe_remove_tenant_dir_all because this is already + // a temporary path + if let Err(e) = std::fs::remove_dir_all(&tenant_dir_path) { + error!( + "Failed to remove temporary directory '{}': {:?}", + tenant_dir_path, e + ); + } + return Ok(None); + } + + // This case happens if we crash during attachment before writing a config into the dir + let is_empty = tenant_dir_path + .is_empty_dir() + .with_context(|| format!("Failed to check whether {tenant_dir_path:?} is an empty dir"))?; + if is_empty { + info!("removing empty tenant directory {tenant_dir_path:?}"); + if let Err(e) = std::fs::remove_dir(&tenant_dir_path) { + error!( + "Failed to remove empty tenant directory '{}': {e:#}", + tenant_dir_path + ) + } + return Ok(None); + } + + let tenant_ignore_mark_file = tenant_dir_path.join(IGNORED_TENANT_FILE_NAME); + if tenant_ignore_mark_file.exists() { + info!("Found an ignore mark file {tenant_ignore_mark_file:?}, skipping the tenant"); + return Ok(None); + } + + let tenant_id = match tenant_dir_path + .file_name() + .unwrap_or_default() + .parse::() + { + Ok(id) => id, + Err(_) => { + warn!("Invalid tenant path (garbage in our repo directory?): {tenant_dir_path}",); + return Ok(None); + } + }; + + Ok(Some(( + tenant_id, + Tenant::load_tenant_config(conf, &tenant_id), + ))) +} + /// Initial stage of load: walk the local tenants directory, clean up any temp files, /// and load configurations for the tenants we found. +/// +/// Do this in parallel, because we expect 10k+ tenants, so serial execution can take +/// seconds even on reasonably fast drives. async fn init_load_tenant_configs( conf: &'static PageServerConf, ) -> anyhow::Result>> { let tenants_dir = conf.tenants_path(); - let mut dir_entries = tenants_dir - .read_dir_utf8() - .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?; + let dentries = tokio::task::spawn_blocking(move || -> anyhow::Result> { + let dir_entries = tenants_dir + .read_dir_utf8() + .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?; + + Ok(dir_entries.collect::, std::io::Error>>()?) + }) + .await??; let mut configs = HashMap::new(); - loop { - match dir_entries.next() { - None => break, - Some(Ok(dentry)) => { - let tenant_dir_path = dentry.path().to_path_buf(); - if crate::is_temporary(&tenant_dir_path) { - info!("Found temporary tenant directory, removing: {tenant_dir_path}"); - // No need to use safe_remove_tenant_dir_all because this is already - // a temporary path - if let Err(e) = fs::remove_dir_all(&tenant_dir_path).await { - error!( - "Failed to remove temporary directory '{}': {:?}", - tenant_dir_path, e - ); - } - continue; - } + let mut join_set = JoinSet::new(); + for dentry in dentries { + join_set.spawn_blocking(move || load_tenant_config(conf, dentry)); + } - // This case happens if we: - // * crash during attach before creating the attach marker file - // * crash during tenant delete before removing tenant directory - let is_empty = tenant_dir_path.is_empty_dir().with_context(|| { - format!("Failed to check whether {tenant_dir_path:?} is an empty dir") - })?; - if is_empty { - info!("removing empty tenant directory {tenant_dir_path:?}"); - if let Err(e) = fs::remove_dir(&tenant_dir_path).await { - error!( - "Failed to remove empty tenant directory '{}': {e:#}", - tenant_dir_path - ) - } - continue; - } - - let tenant_ignore_mark_file = tenant_dir_path.join(IGNORED_TENANT_FILE_NAME); - if tenant_ignore_mark_file.exists() { - info!("Found an ignore mark file {tenant_ignore_mark_file:?}, skipping the tenant"); - continue; - } - - let tenant_id = match tenant_dir_path - .file_name() - .unwrap_or_default() - .parse::() - { - Ok(id) => id, - Err(_) => { - warn!( - "Invalid tenant path (garbage in our repo directory?): {tenant_dir_path}", - ); - continue; - } - }; - - configs.insert(tenant_id, Tenant::load_tenant_config(conf, &tenant_id)); - } - Some(Err(e)) => { - // An error listing the top level directory indicates serious problem - // with local filesystem: we will fail to load, and fail to start. - anyhow::bail!(e); - } + while let Some(r) = join_set.join_next().await { + if let Some((tenant_id, tenant_config)) = r?? { + configs.insert(tenant_id, tenant_config); } } + Ok(configs) } From 188f67e1dffe119b6468cd5e0d86360fd1b4b6e4 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 23 Oct 2023 16:51:38 +0100 Subject: [PATCH 47/49] pageserver: forward compat: be tolerant of deletion marker in `timelines/` (#5632) ## Problem https://github.com/neondatabase/neon/pull/5580 will move the remote deletion marker into the `timelines/` path. This would cause old pageserver code to fail loading the tenant due to an apparently invalid timeline ID. That would be a problem if we had to roll back after deploying #5580 ## Summary of changes If a `deleted` file is in `timelines/` just ignore it. --- pageserver/src/tenant/remote_timeline_client/download.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index ef8d217be4..ca6e3293d6 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -18,7 +18,7 @@ use crate::config::PageServerConf; use crate::tenant::remote_timeline_client::{remote_layer_path, remote_timelines_path}; use crate::tenant::storage_layer::LayerFileName; use crate::tenant::timeline::span::debug_assert_current_span_has_tenant_and_timeline_id; -use crate::tenant::Generation; +use crate::tenant::{Generation, TENANT_DELETED_MARKER_FILE_NAME}; use remote_storage::{DownloadError, GenericRemoteStorage}; use utils::crashsafe::path_with_suffix_extension; use utils::id::{TenantId, TimelineId}; @@ -190,6 +190,12 @@ pub async fn list_remote_timelines( let mut timeline_ids = HashSet::new(); for timeline_remote_storage_key in timelines { + if timeline_remote_storage_key.object_name() == Some(TENANT_DELETED_MARKER_FILE_NAME) { + // A `deleted` key within `timelines/` is a marker file, not a timeline. Ignore it. + // This code will be removed in https://github.com/neondatabase/neon/pull/5580 + continue; + } + let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| { anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}") })?; From eaaa18f6eda7b05d88a8b4d32ff7756504ef2113 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 23 Oct 2023 17:30:25 +0100 Subject: [PATCH 48/49] attachment_service: graceful SIGQUIT (#5626) `attachment_service` doesn't explicitly handle signals, which causes a backtrace when `neon_local` kills it with SIGQUIT. Closes: https://github.com/neondatabase/neon/issues/5613 --- control_plane/src/bin/attachment_service.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index d4bca59c7b..65ed78d20b 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use std::{collections::HashMap, sync::Arc}; use utils::logging::{self, LogFormat}; +use utils::signals::{ShutdownSignals, Signal}; use utils::{ http::{ @@ -268,7 +269,16 @@ async fn main() -> anyhow::Result<()> { let server = hyper::Server::from_tcp(http_listener)?.serve(service); tracing::info!("Serving on {0}", args.listen); - server.await?; + + tokio::task::spawn(server); + + ShutdownSignals::handle(|signal| match signal { + Signal::Interrupt | Signal::Terminate | Signal::Quit => { + tracing::info!("Got {}. Terminating", signal.name()); + // We're just a test helper: no graceful shutdown. + std::process::exit(0); + } + })?; Ok(()) } From 1e250cd90a79902f954740a71f1144039676027b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 23 Oct 2023 20:08:56 +0200 Subject: [PATCH 49/49] Cleanup in azure_upload_download_works test (#5636) The `azure_upload_download_works` test is not cleaning up after itself, leaving behind the files it is uploading. I found these files when looking at the contents of the bucket in #5627. We now clean up the file we uploaded before, like the other tests do it as well. Follow-up of #5546 --- libs/remote_storage/tests/test_real_azure.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/remote_storage/tests/test_real_azure.rs b/libs/remote_storage/tests/test_real_azure.rs index 5ebbd9e95b..0338270aaf 100644 --- a/libs/remote_storage/tests/test_real_azure.rs +++ b/libs/remote_storage/tests/test_real_azure.rs @@ -267,6 +267,12 @@ async fn azure_upload_download_works(ctx: &mut MaybeEnabledAzure) -> anyhow::Res let buf = download_and_compare(dl).await?; assert_eq!(buf, data); + debug!("Cleanup: deleting file at path {path:?}"); + ctx.client + .delete(&path) + .await + .with_context(|| format!("{path:?} removal"))?; + Ok(()) }