From c6b56d2967b188ec84b461805d1466a3bd50b7ba Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Tue, 17 Jan 2023 22:07:38 +0200 Subject: [PATCH 001/102] Add more io::Error context when fail to operate on a path (#3254) I have a test failure that shows ``` Caused by: 0: Failed to reconstruct a page image: 1: Directory not empty (os error 39) ``` but does not really show where exactly that happens. https://neon-github-public-dev.s3.amazonaws.com/reports/pr-3227/release/3823785365/index.html#categories/c0057473fc9ec8fb70876fd29a171ce8/7088dab272f2c7b7/?attachment=60fe6ed2add4d82d The PR aims to add more context in debugging that issue. --- pageserver/src/walredo.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index a552c05d63..fd0524016f 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -626,24 +626,20 @@ impl PostgresRedoProcess { // Create empty data directory for wal-redo postgres, deleting old one first. if datadir.exists() { - info!( - "old temporary datadir {} exists, removing", - datadir.display() - ); - fs::remove_dir_all(&datadir)?; + info!("old temporary datadir {datadir:?} exists, removing"); + fs::remove_dir_all(&datadir).map_err(|e| { + Error::new( + e.kind(), + format!("Old temporary dir {datadir:?} removal failure: {e}"), + ) + })?; } - let pg_bin_dir_path = conf.pg_bin_dir(pg_version).map_err(|e| { - Error::new( - ErrorKind::Other, - format!("incorrect pg_bin_dir path: {}", e), - ) - })?; - let pg_lib_dir_path = conf.pg_lib_dir(pg_version).map_err(|e| { - Error::new( - ErrorKind::Other, - format!("incorrect pg_lib_dir path: {}", e), - ) - })?; + let pg_bin_dir_path = conf + .pg_bin_dir(pg_version) + .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_bin_dir path: {e}")))?; + let pg_lib_dir_path = conf + .pg_lib_dir(pg_version) + .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_lib_dir path: {e}")))?; info!("running initdb in {}", datadir.display()); let initdb = Command::new(pg_bin_dir_path.join("initdb")) From 3b58c61b337f7d735dc45f30201289d2015c985e Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 18 Jan 2023 02:29:05 +0200 Subject: [PATCH 002/102] If an error happens while checking for core dumps, don't panic. If we panic, we skip the 30s wait in 'main', and don't give the console a chance to observe the error. Which is not nice. Spotted by @ololobus at https://github.com/neondatabase/neon/pull/3352#discussion_r1072806981 --- compute_tools/src/compute.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index c2c9ab2230..d652084e00 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -23,7 +23,7 @@ use std::sync::RwLock; use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; -use log::{info, warn}; +use log::{error, info, warn}; use postgres::{Client, NoTls}; use serde::{Serialize, Serializer}; @@ -311,8 +311,9 @@ impl ComputeNode { .wait() .expect("failed to start waiting on Postgres process"); - self.check_for_core_dumps() - .expect("failed to check for core dumps"); + if let Err(err) = self.check_for_core_dumps() { + error!("error while checking for core dumps: {err:?}"); + } Ok(ecode) } From 506086a3e2fe96318aecb66bdcef176a5834a003 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Wed, 18 Jan 2023 14:58:55 +0200 Subject: [PATCH 003/102] Fix metric_collection_endpoint for prod. It was incorrectly set to staging url --- .github/ansible/production.hosts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ansible/production.hosts.yaml b/.github/ansible/production.hosts.yaml index 22bace5ade..ecb847bd61 100644 --- a/.github/ansible/production.hosts.yaml +++ b/.github/ansible/production.hosts.yaml @@ -7,7 +7,7 @@ storage: broker_endpoint: http://storage-broker.prod.local:50051 pageserver_config_stub: pg_distrib_dir: /usr/local - metric_collection_endpoint: http://console-staging.local/billing/api/v1/usage_events + metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events metric_collection_interval: 10min remote_storage: bucket_name: "{{ bucket_name }}" From e59d32ac5d483673083b7a178c76da9128f370e7 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Wed, 18 Jan 2023 14:49:59 +0100 Subject: [PATCH 004/102] Change SENTRY_ENVIRONMENT from "development" to "staging" --- .github/ansible/staging.eu-west-1.hosts.yaml | 2 +- .github/ansible/staging.us-east-2.hosts.yaml | 2 +- .github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml | 2 +- .github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml | 2 +- .github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml | 2 +- .../helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml | 2 +- .github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml | 2 +- .github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/ansible/staging.eu-west-1.hosts.yaml b/.github/ansible/staging.eu-west-1.hosts.yaml index fce450ed39..f28dc8e07b 100644 --- a/.github/ansible/staging.eu-west-1.hosts.yaml +++ b/.github/ansible/staging.eu-west-1.hosts.yaml @@ -18,7 +18,7 @@ storage: ansible_aws_ssm_region: eu-west-1 ansible_aws_ssm_bucket_name: neon-dev-storage-eu-west-1 console_region_id: aws-eu-west-1 - sentry_environment: development + sentry_environment: staging children: pageservers: diff --git a/.github/ansible/staging.us-east-2.hosts.yaml b/.github/ansible/staging.us-east-2.hosts.yaml index 1d1b8dbfa4..4891875369 100644 --- a/.github/ansible/staging.us-east-2.hosts.yaml +++ b/.github/ansible/staging.us-east-2.hosts.yaml @@ -18,7 +18,7 @@ storage: ansible_aws_ssm_region: us-east-2 ansible_aws_ssm_bucket_name: neon-staging-storage-us-east-2 console_region_id: aws-us-east-2 - sentry_environment: development + sentry_environment: staging children: pageservers: diff --git a/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml b/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml index 47924456ba..c49b8d2009 100644 --- a/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml +++ b/.github/helm-values/dev-eu-west-1-zeta.neon-proxy-scram.yaml @@ -8,7 +8,7 @@ settings: authBackend: "console" authEndpoint: "http://console-staging.local/management/api/v2" domain: "*.eu-west-1.aws.neon.build" - sentryEnvironment: "development" + sentryEnvironment: "staging" wssPort: 8443 metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events" metricCollectionInterval: "1min" diff --git a/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml b/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml index c6e682f571..ccf701f52d 100644 --- a/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml +++ b/.github/helm-values/dev-eu-west-1-zeta.neon-storage-broker.yaml @@ -49,4 +49,4 @@ extraManifests: - "{{ .Release.Namespace }}" settings: - sentryEnvironment: "development" + sentryEnvironment: "staging" diff --git a/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml b/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml index eb8fd50c0f..cb062f705d 100644 --- a/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml +++ b/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml @@ -8,7 +8,7 @@ settings: authBackend: "link" authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/" uri: "https://console.stage.neon.tech/psql_session/" - sentryEnvironment: "development" + sentryEnvironment: "staging" metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events" metricCollectionInterval: "1min" diff --git a/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml b/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml index 8a08738d5f..99b67d75c1 100644 --- a/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml +++ b/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram-legacy.yaml @@ -8,7 +8,7 @@ settings: authBackend: "console" authEndpoint: "http://console-staging.local/management/api/v2" domain: "*.cloud.stage.neon.tech" - sentryEnvironment: "development" + sentryEnvironment: "staging" wssPort: 8443 metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events" metricCollectionInterval: "1min" diff --git a/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml b/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml index b02d46917c..764bb25b64 100644 --- a/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml +++ b/.github/helm-values/dev-us-east-2-beta.neon-proxy-scram.yaml @@ -8,7 +8,7 @@ settings: authBackend: "console" authEndpoint: "http://console-staging.local/management/api/v2" domain: "*.us-east-2.aws.neon.build" - sentryEnvironment: "development" + sentryEnvironment: "staging" wssPort: 8443 metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events" metricCollectionInterval: "1min" diff --git a/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml b/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml index c7682d24c0..69363c5f13 100644 --- a/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml +++ b/.github/helm-values/dev-us-east-2-beta.neon-storage-broker.yaml @@ -49,4 +49,4 @@ extraManifests: - "{{ .Release.Namespace }}" settings: - sentryEnvironment: "development" + sentryEnvironment: "staging" From 826e89b9ce43ce2c4d046b2c5d6376c3de8dbbac Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sat, 7 Jan 2023 00:23:35 +0200 Subject: [PATCH 005/102] Use actual temporary dir for pageserver unit tests --- .gitignore | 2 - control_plane/.gitignore | 1 - pageserver/src/config.rs | 5 - pageserver/src/tenant.rs | 105 ++++++++---------- pageserver/src/tenant/ephemeral_file.rs | 36 +++--- .../src/tenant/remote_timeline_client.rs | 2 +- pageserver/src/virtual_file.rs | 8 +- pageserver/src/walingest.rs | 12 +- .../src/walreceiver/connection_manager.rs | 16 +-- test_runner/sql_regress/.gitignore | 1 - 10 files changed, 80 insertions(+), 108 deletions(-) delete mode 100644 control_plane/.gitignore diff --git a/.gitignore b/.gitignore index f1afdee599..2e241ee8cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ /pg_install /target -/tmp_check -/tmp_check_cli __pycache__/ test_output/ .vscode diff --git a/control_plane/.gitignore b/control_plane/.gitignore deleted file mode 100644 index c1e54a6bcb..0000000000 --- a/control_plane/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tmp_check/ diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 51d1664e52..f3e5fb8c1a 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -693,11 +693,6 @@ impl PageServerConf { Ok(t_conf) } - #[cfg(test)] - pub fn test_repo_dir(test_name: &str) -> PathBuf { - PathBuf::from(format!("../tmp_check/test_{test_name}")) - } - pub fn dummy_conf(repo_dir: PathBuf) -> Self { let pg_distrib_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install"); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 1d0d6b66ab..0dd6735993 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2626,9 +2626,9 @@ where #[cfg(test)] pub mod harness { use bytes::{Bytes, BytesMut}; - use once_cell::sync::Lazy; - use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; + use std::sync::Arc; use std::{fs, path::PathBuf}; + use tempfile::TempDir; use utils::lsn::Lsn; use crate::{ @@ -2659,8 +2659,6 @@ pub mod harness { buf.freeze() } - static LOCK: Lazy> = Lazy::new(|| RwLock::new(())); - impl From for TenantConfOpt { fn from(tenant_conf: TenantConf) -> Self { Self { @@ -2681,36 +2679,27 @@ pub mod harness { } } - pub struct TenantHarness<'a> { + /// The harness saves some boilerplate and provides a way to create functional tenant + /// without running pageserver binary. It uses temporary directory to store data in it. + /// Tempdir gets removed on harness drop. + pub struct TenantHarness { + // keep the struct to not to remove tmp dir during the test + _temp_repo_dir: TempDir, pub conf: &'static PageServerConf, pub tenant_conf: TenantConf, pub tenant_id: TenantId, - - pub lock_guard: ( - Option>, - Option>, - ), } - impl<'a> TenantHarness<'a> { - pub fn create(test_name: &'static str) -> anyhow::Result { - Self::create_internal(test_name, false) - } - pub fn create_exclusive(test_name: &'static str) -> anyhow::Result { - Self::create_internal(test_name, true) - } - fn create_internal(test_name: &'static str, exclusive: bool) -> anyhow::Result { - let lock_guard = if exclusive { - (None, Some(LOCK.write().unwrap())) - } else { - (Some(LOCK.read().unwrap()), None) - }; + static LOG_HANDLE: OnceCell<()> = OnceCell::new(); - let repo_dir = PageServerConf::test_repo_dir(test_name); - let _ = fs::remove_dir_all(&repo_dir); - fs::create_dir_all(&repo_dir)?; + impl TenantHarness { + pub fn new() -> anyhow::Result { + let temp_repo_dir = tempfile::tempdir()?; + // `TempDir` uses a randomly generated subdirectory of a system tmp dir, + // so far it's enough to take care of concurrently running tests. + let repo_dir = temp_repo_dir.path(); - let conf = PageServerConf::dummy_conf(repo_dir); + let conf = PageServerConf::dummy_conf(repo_dir.to_path_buf()); // Make a static copy of the config. This can never be free'd, but that's // OK in a test. let conf: &'static PageServerConf = Box::leak(Box::new(conf)); @@ -2728,10 +2717,10 @@ pub mod harness { fs::create_dir_all(conf.timelines_path(&tenant_id))?; Ok(Self { + _temp_repo_dir: temp_repo_dir, conf, tenant_conf, tenant_id, - lock_guard, }) } @@ -2825,7 +2814,8 @@ mod tests { #[tokio::test] async fn test_basic() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_basic")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2858,9 +2848,8 @@ mod tests { #[tokio::test] async fn no_duplicate_timelines() -> anyhow::Result<()> { - let tenant = TenantHarness::create("no_duplicate_timelines")? - .load() - .await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let _ = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2891,7 +2880,8 @@ mod tests { /// #[tokio::test] async fn test_branch() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_branch")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2988,10 +2978,8 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_garbage_collected_data() -> anyhow::Result<()> { - let tenant = - TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? - .load() - .await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3026,9 +3014,8 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? - .load() - .await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; tenant .create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION)? @@ -3077,9 +3064,8 @@ mod tests { #[tokio::test] async fn test_retain_data_in_parent_which_is_needed_for_child() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? - .load() - .await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3101,9 +3087,8 @@ mod tests { } #[tokio::test] async fn test_parent_keeps_data_forever_after_branching() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_parent_keeps_data_forever_after_branching")? - .load() - .await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3134,8 +3119,7 @@ mod tests { #[tokio::test] async fn timeline_load() -> anyhow::Result<()> { - const TEST_NAME: &str = "timeline_load"; - let harness = TenantHarness::create(TEST_NAME)?; + let harness = TenantHarness::new()?; { let tenant = harness.load().await; let tline = tenant @@ -3154,8 +3138,7 @@ mod tests { #[tokio::test] async fn timeline_load_with_ancestor() -> anyhow::Result<()> { - const TEST_NAME: &str = "timeline_load_with_ancestor"; - let harness = TenantHarness::create(TEST_NAME)?; + let harness = TenantHarness::new()?; // create two timelines { let tenant = harness.load().await; @@ -3193,8 +3176,7 @@ mod tests { #[tokio::test] async fn corrupt_metadata() -> anyhow::Result<()> { - const TEST_NAME: &str = "corrupt_metadata"; - let harness = TenantHarness::create(TEST_NAME)?; + let harness = TenantHarness::new()?; let tenant = harness.load().await; tenant @@ -3235,7 +3217,8 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_images")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3302,7 +3285,8 @@ mod tests { // #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_bulk_insert")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3346,7 +3330,8 @@ mod tests { #[tokio::test] async fn test_random_updates() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_random_updates")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3419,9 +3404,8 @@ mod tests { #[tokio::test] async fn test_traverse_branches() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_traverse_branches")? - .load() - .await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let mut tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3505,9 +3489,8 @@ mod tests { #[tokio::test] async fn test_traverse_ancestors() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_traverse_ancestors")? - .load() - .await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let mut tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index c433e65ad2..0debeaff1c 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -76,7 +76,7 @@ impl EphemeralFile { }) } - fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), io::Error> { + fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> io::Result<()> { let mut off = 0; while off < PAGE_SZ { let n = self @@ -277,7 +277,7 @@ impl Drop for EphemeralFile { } } -pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> Result<(), io::Error> { +pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> io::Result<()> { if let Some(file) = EPHEMERAL_FILES.read().unwrap().files.get(&file_id) { match file.write_all_at(buf, blkno as u64 * PAGE_SZ as u64) { Ok(_) => Ok(()), @@ -332,25 +332,17 @@ mod tests { use super::*; use crate::tenant::blob_io::{BlobCursor, BlobWriter}; use crate::tenant::block_io::BlockCursor; + use crate::tenant::harness::TenantHarness; use rand::{seq::SliceRandom, thread_rng, RngCore}; use std::fs; use std::str::FromStr; - fn harness( - test_name: &str, - ) -> Result<(&'static PageServerConf, TenantId, TimelineId), io::Error> { - let repo_dir = PageServerConf::test_repo_dir(test_name); - let _ = fs::remove_dir_all(&repo_dir); - let conf = PageServerConf::dummy_conf(repo_dir); - // Make a static copy of the config. This can never be free'd, but that's - // OK in a test. - let conf: &'static PageServerConf = Box::leak(Box::new(conf)); - - let tenant_id = TenantId::from_str("11000000000000000000000000000000").unwrap(); + fn harness() -> Result<(TenantHarness, TimelineId), io::Error> { + let harness = TenantHarness::new().expect("Failed to create tenant harness"); let timeline_id = TimelineId::from_str("22000000000000000000000000000000").unwrap(); - fs::create_dir_all(conf.timeline_path(&timeline_id, &tenant_id))?; + fs::create_dir_all(harness.timeline_path(&timeline_id))?; - Ok((conf, tenant_id, timeline_id)) + Ok((harness, timeline_id)) } // Helper function to slurp contents of a file, starting at the current position, @@ -367,10 +359,10 @@ mod tests { } #[test] - fn test_ephemeral_files() -> Result<(), io::Error> { - let (conf, tenant_id, timeline_id) = harness("ephemeral_files")?; + fn test_ephemeral_files() -> io::Result<()> { + let (harness, timeline_id) = harness()?; - let file_a = EphemeralFile::create(conf, tenant_id, timeline_id)?; + let file_a = EphemeralFile::create(harness.conf, harness.tenant_id, timeline_id)?; file_a.write_all_at(b"foo", 0)?; assert_eq!("foo", read_string(&file_a, 0, 20)?); @@ -381,7 +373,7 @@ mod tests { // Open a lot of files, enough to cause some page evictions. let mut efiles = Vec::new(); for fileno in 0..100 { - let efile = EphemeralFile::create(conf, tenant_id, timeline_id)?; + let efile = EphemeralFile::create(harness.conf, harness.tenant_id, timeline_id)?; efile.write_all_at(format!("file {}", fileno).as_bytes(), 0)?; assert_eq!(format!("file {}", fileno), read_string(&efile, 0, 10)?); efiles.push((fileno, efile)); @@ -398,10 +390,10 @@ mod tests { } #[test] - fn test_ephemeral_blobs() -> Result<(), io::Error> { - let (conf, tenant_id, timeline_id) = harness("ephemeral_blobs")?; + fn test_ephemeral_blobs() -> io::Result<()> { + let (harness, timeline_id) = harness()?; - let mut file = EphemeralFile::create(conf, tenant_id, timeline_id)?; + let mut file = EphemeralFile::create(harness.conf, harness.tenant_id, timeline_id)?; let pos_foo = file.write_blob(b"foo")?; assert_eq!(b"foo", file.block_cursor().read_blob(pos_foo)?.as_slice()); diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 013591caee..58b7eea1eb 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1064,7 +1064,7 @@ mod tests { // Test scheduling #[test] fn upload_scheduling() -> anyhow::Result<()> { - let harness = TenantHarness::create("upload_scheduling")?; + let harness = TenantHarness::new()?; let timeline_path = harness.timeline_path(&TIMELINE_ID); std::fs::create_dir_all(&timeline_path)?; diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index fb216123c1..3ad049cc21 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -525,12 +525,13 @@ mod tests { }) } - fn test_files(testname: &str, openfunc: OF) -> Result<(), Error> + fn test_files(test_name: &str, openfunc: OF) -> Result<(), Error> where FD: Read + Write + Seek + FileExt, OF: Fn(&Path, &OpenOptions) -> Result, { - let testdir = crate::config::PageServerConf::test_repo_dir(testname); + let temp_repo_dir = tempfile::tempdir()?; + let testdir = temp_repo_dir.path().join(test_name); std::fs::create_dir_all(&testdir)?; let path_a = testdir.join("file_a"); @@ -632,7 +633,8 @@ mod tests { const THREADS: usize = 100; const SAMPLE: [u8; SIZE] = [0xADu8; SIZE]; - let testdir = crate::config::PageServerConf::test_repo_dir("vfile_concurrency"); + let temp_repo_dir = tempfile::tempdir()?; + let testdir = temp_repo_dir.path().join("vfile_concurrency"); std::fs::create_dir_all(&testdir)?; // Create a test file. diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 0de2e6654d..77fce95160 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1146,7 +1146,8 @@ mod tests { #[tokio::test] async fn test_relsize() -> Result<()> { - let tenant = TenantHarness::create("test_relsize")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; @@ -1323,7 +1324,8 @@ mod tests { // and then created it again within the same layer. #[tokio::test] async fn test_drop_extend() -> Result<()> { - let tenant = TenantHarness::create("test_drop_extend")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; @@ -1376,7 +1378,8 @@ mod tests { // and then extended it again within the same layer. #[tokio::test] async fn test_truncate_extend() -> Result<()> { - let tenant = TenantHarness::create("test_truncate_extend")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; @@ -1497,7 +1500,8 @@ mod tests { /// split into multiple 1 GB segments in Postgres. #[tokio::test] async fn test_large_rel() -> Result<()> { - let tenant = TenantHarness::create("test_large_rel")?.load().await; + let harness = TenantHarness::new()?; + let tenant = harness.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; diff --git a/pageserver/src/walreceiver/connection_manager.rs b/pageserver/src/walreceiver/connection_manager.rs index 8b60e59305..be58aa0e07 100644 --- a/pageserver/src/walreceiver/connection_manager.rs +++ b/pageserver/src/walreceiver/connection_manager.rs @@ -846,7 +846,7 @@ mod tests { #[tokio::test] async fn no_connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("no_connection_no_candidate")?; + let harness = TenantHarness::new()?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -879,7 +879,7 @@ mod tests { #[tokio::test] async fn connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("connection_no_candidate")?; + let harness = TenantHarness::new()?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -942,7 +942,7 @@ mod tests { #[tokio::test] async fn no_connection_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("no_connection_candidate")?; + let harness = TenantHarness::new()?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1001,7 +1001,7 @@ mod tests { #[tokio::test] async fn candidate_with_many_connection_failures() -> anyhow::Result<()> { - let harness = TenantHarness::create("candidate_with_many_connection_failures")?; + let harness = TenantHarness::new()?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1041,7 +1041,7 @@ mod tests { #[tokio::test] async fn lsn_wal_over_threshhold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?; + let harness = TenantHarness::new()?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1105,7 +1105,7 @@ mod tests { #[tokio::test] async fn timeout_connection_threshhold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("timeout_connection_threshhold_current_candidate")?; + let harness = TenantHarness::new()?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1166,7 +1166,7 @@ mod tests { #[tokio::test] async fn timeout_wal_over_threshhold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("timeout_wal_over_threshhold_current_candidate")?; + let harness = TenantHarness::new()?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let new_lsn = Lsn(100_100).align(); @@ -1232,7 +1232,7 @@ mod tests { const DUMMY_SAFEKEEPER_HOST: &str = "safekeeper_connstr"; - async fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState { + async fn dummy_state(harness: &TenantHarness) -> WalreceiverState { WalreceiverState { id: TenantTimelineId { tenant_id: harness.tenant_id, diff --git a/test_runner/sql_regress/.gitignore b/test_runner/sql_regress/.gitignore index 89129d7358..83186b5c86 100644 --- a/test_runner/sql_regress/.gitignore +++ b/test_runner/sql_regress/.gitignore @@ -2,7 +2,6 @@ /pg_regress # Generated subdirectories -/tmp_check/ /results/ /log/ From 90f66aa51b90a601868caa4e7f5bfd053632009e Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 12 Jan 2023 15:14:04 +0200 Subject: [PATCH 006/102] Enable logs in unit tests --- libs/utils/src/logging.rs | 2 ++ pageserver/src/tenant.rs | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/libs/utils/src/logging.rs b/libs/utils/src/logging.rs index 3b1a1f5aff..82c9267f4a 100644 --- a/libs/utils/src/logging.rs +++ b/libs/utils/src/logging.rs @@ -8,6 +8,7 @@ use strum_macros::{EnumString, EnumVariantNames}; pub enum LogFormat { Plain, Json, + Test, } impl LogFormat { @@ -39,6 +40,7 @@ pub fn init(log_format: LogFormat) -> anyhow::Result<()> { match log_format { LogFormat::Json => base_logger.json().init(), LogFormat::Plain => base_logger.init(), + LogFormat::Test => base_logger.with_test_writer().init(), } Ok(()) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 0dd6735993..c53c9bc3e1 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2626,9 +2626,11 @@ where #[cfg(test)] pub mod harness { use bytes::{Bytes, BytesMut}; + use once_cell::sync::OnceCell; use std::sync::Arc; use std::{fs, path::PathBuf}; use tempfile::TempDir; + use utils::logging; use utils::lsn::Lsn; use crate::{ @@ -2694,6 +2696,10 @@ pub mod harness { impl TenantHarness { pub fn new() -> anyhow::Result { + LOG_HANDLE.get_or_init(|| { + logging::init(logging::LogFormat::Test).expect("Failed to init test logging") + }); + let temp_repo_dir = tempfile::tempdir()?; // `TempDir` uses a randomly generated subdirectory of a system tmp dir, // so far it's enough to take care of concurrently running tests. From e5cc2f92c4f611f1bebba3e6c1151b0c4f36c5bd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 18 Jan 2023 11:30:02 +0200 Subject: [PATCH 007/102] Switch to 'tracing' for logging, restructure code to make use of spans. Refactors Compute::prepare_and_run. It's split into subroutines differently, to make it easier to attach tracing spans to the different stages. The high-level logic for waiting for Postgres to exit is moved to the caller. Replace 'env_logger' with 'tracing', and add `#instrument` directives to different stages fo the startup process. This is a fairly mechanical change, except for the changes in 'spec.rs'. 'spec.rs' contained some complicated formatting, where parts of log messages were printed directly to stdout with `print`s. That was a bit messed up because the log normally goes to stderr, but those lines were printed to stdout. In our docker images, stderr and stdout both go to the same place so you wouldn't notice, but I don't think it was intentional. This changes the log format to the default 'tracing_subscriber::format' format. It's different from the Postgres log format, however, and because both compute_tools and Postgres print to the same log, it's now a mix of two different formats. I'm not sure how the Grafana log parsing pipeline can handle that. If it's a problem, we can build custom formatter to change the compute_tools log format to be the same as Postgres's, like it was before this commit, or we can change the Postgres log format to match tracing_formatter's, or we can start printing compute_tool's log output to a different destination than Postgres --- Cargo.lock | 6 +- compute_tools/Cargo.toml | 4 +- compute_tools/src/bin/compute_ctl.rs | 55 +++++--- compute_tools/src/checker.rs | 4 +- compute_tools/src/compute.rs | 77 ++++++----- compute_tools/src/http/api.rs | 2 +- compute_tools/src/informant.rs | 2 +- compute_tools/src/logger.rs | 44 ++---- compute_tools/src/monitor.rs | 2 +- compute_tools/src/pg_helpers.rs | 12 +- compute_tools/src/spec.rs | 171 ++++++++++++++---------- test_runner/regress/test_compute_ctl.py | 2 +- workspace_hack/Cargo.toml | 5 +- 13 files changed, 210 insertions(+), 176 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59adf696a7..d8aba9ba68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -833,10 +833,8 @@ dependencies = [ "anyhow", "chrono", "clap 4.0.32", - "env_logger", "futures", "hyper", - "log", "notify", "postgres", "regex", @@ -845,6 +843,8 @@ dependencies = [ "tar", "tokio", "tokio-postgres", + "tracing", + "tracing-subscriber", "url", "workspace_hack", ] @@ -1954,7 +1954,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", - "serde", ] [[package]] @@ -4565,6 +4564,7 @@ dependencies = [ "tower", "tracing", "tracing-core", + "tracing-subscriber", "url", ] diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 1e0aee81d7..4536604bdf 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -8,10 +8,8 @@ license.workspace = true anyhow.workspace = true chrono.workspace = true clap.workspace = true -env_logger.workspace = true futures.workspace = true hyper = { workspace = true, features = ["full"] } -log = { workspace = true, features = ["std", "serde"] } notify.workspace = true postgres.workspace = true regex.workspace = true @@ -20,6 +18,8 @@ serde_json.workspace = true tar.workspace = true tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } tokio-postgres.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true url.workspace = true workspace_hack.workspace = true diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 307300cfd8..e5ab8eb153 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -40,7 +40,7 @@ use std::{thread, time::Duration}; use anyhow::{Context, Result}; use chrono::Utc; use clap::Arg; -use log::{error, info}; +use tracing::{error, info}; use compute_tools::compute::{ComputeMetrics, ComputeNode, ComputeState, ComputeStatus}; use compute_tools::http::api::launch_http_server; @@ -53,7 +53,6 @@ use compute_tools::spec::*; use url::Url; fn main() -> Result<()> { - // TODO: re-use `utils::logging` later init_logger(DEFAULT_LOG_LEVEL)?; let matches = cli().get_matches(); @@ -122,29 +121,45 @@ fn main() -> Result<()> { // Also spawn the thread responsible for handling the VM informant -- if it's present let _vm_informant_handle = spawn_vm_informant_if_present().expect("cannot launch VM informant"); - // Run compute (Postgres) and hang waiting on it. - match compute.prepare_and_run() { - Ok(ec) => { - let code = ec.code().unwrap_or(1); - info!("Postgres exited with code {}, shutting down", code); - exit(code) - } - Err(error) => { - error!("could not start the compute node: {:?}", error); - + // Start Postgres + let mut delay_exit = false; + let mut exit_code = None; + let pg = match compute.start_compute() { + Ok(pg) => Some(pg), + Err(err) => { + error!("could not start the compute node: {:?}", err); let mut state = compute.state.write().unwrap(); - state.error = Some(format!("{:?}", error)); + state.error = Some(format!("{:?}", err)); state.status = ComputeStatus::Failed; drop(state); - - // Keep serving HTTP requests, so the cloud control plane was able to - // get the actual error. - info!("giving control plane 30s to collect the error before shutdown"); - thread::sleep(Duration::from_secs(30)); - info!("shutting down"); - Err(error) + delay_exit = true; + None } + }; + + // Wait for the child Postgres process forever. In this state Ctrl+C will + // propagate to Postgres and it will be shut down as well. + if let Some(mut pg) = pg { + let ecode = pg + .wait() + .expect("failed to start waiting on Postgres process"); + info!("Postgres exited with code {}, shutting down", ecode); + exit_code = ecode.code() } + + if let Err(err) = compute.check_for_core_dumps() { + error!("error while checking for core dumps: {err:?}"); + } + + // If launch failed, keep serving HTTP requests for a while, so the cloud + // control plane can get the actual error. + if delay_exit { + info!("giving control plane 30s to collect the error before shutdown"); + thread::sleep(Duration::from_secs(30)); + info!("shutting down"); + } + + exit(exit_code.unwrap_or(1)) } fn cli() -> clap::Command { diff --git a/compute_tools/src/checker.rs b/compute_tools/src/checker.rs index ee1605c814..b8413de516 100644 --- a/compute_tools/src/checker.rs +++ b/compute_tools/src/checker.rs @@ -1,10 +1,11 @@ use anyhow::{anyhow, Result}; -use log::error; use postgres::Client; use tokio_postgres::NoTls; +use tracing::{error, instrument}; use crate::compute::ComputeNode; +#[instrument(skip_all)] pub fn create_writability_check_data(client: &mut Client) -> Result<()> { let query = " CREATE TABLE IF NOT EXISTS health_check ( @@ -21,6 +22,7 @@ pub fn create_writability_check_data(client: &mut Client) -> Result<()> { Ok(()) } +#[instrument(skip_all)] pub async fn check_writability(compute: &ComputeNode) -> Result<()> { let (client, connection) = tokio_postgres::connect(compute.connstr.as_str(), NoTls).await?; if client.is_closed() { diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index d652084e00..e229bb1cc2 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -17,15 +17,15 @@ use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::Path; -use std::process::{Command, ExitStatus, Stdio}; +use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::RwLock; use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; -use log::{error, info, warn}; use postgres::{Client, NoTls}; use serde::{Serialize, Serializer}; +use tracing::{info, instrument, warn}; use crate::checker::create_writability_check_data; use crate::config; @@ -121,6 +121,7 @@ impl ComputeNode { // Get basebackup from the libpq connection to pageserver using `connstr` and // unarchive it to `pgdata` directory overriding all its previous content. + #[instrument(skip(self))] fn get_basebackup(&self, lsn: &str) -> Result<()> { let start_time = Utc::now(); @@ -154,6 +155,7 @@ impl ComputeNode { // Run `postgres` in a special mode with `--sync-safekeepers` argument // and return the reported LSN back to the caller. + #[instrument(skip(self))] fn sync_safekeepers(&self) -> Result { let start_time = Utc::now(); @@ -196,6 +198,7 @@ impl ComputeNode { /// Do all the preparations like PGDATA directory creation, configuration, /// safekeepers sync, basebackup, etc. + #[instrument(skip(self))] pub fn prepare_pgdata(&self) -> Result<()> { let spec = &self.spec; let pgdata_path = Path::new(&self.pgdata); @@ -229,9 +232,8 @@ impl ComputeNode { /// Start Postgres as a child process and manage DBs/roles. /// After that this will hang waiting on the postmaster process to exit. - pub fn run(&self) -> Result { - let start_time = Utc::now(); - + #[instrument(skip(self))] + pub fn start_postgres(&self) -> Result { let pgdata_path = Path::new(&self.pgdata); // Run postgres as a child process. @@ -242,6 +244,11 @@ impl ComputeNode { wait_for_postgres(&mut pg, pgdata_path)?; + Ok(pg) + } + + #[instrument(skip(self))] + pub fn apply_config(&self) -> Result<()> { // If connection fails, // it may be the old node with `zenith_admin` superuser. // @@ -279,8 +286,34 @@ impl ComputeNode { // 'Close' connection drop(client); - let startup_end_time = Utc::now(); + info!( + "finished configuration of compute for project {}", + self.spec.cluster.cluster_id + ); + + Ok(()) + } + + #[instrument(skip(self))] + pub fn start_compute(&self) -> Result { + info!( + "starting compute for project {}, operation {}, tenant {}, timeline {}", + self.spec.cluster.cluster_id, + self.spec.operation_uuid.as_ref().unwrap(), + self.tenant, + self.timeline, + ); + + self.prepare_pgdata()?; + + let start_time = Utc::now(); + + let pg = self.start_postgres()?; + + self.apply_config()?; + + let startup_end_time = Utc::now(); self.metrics.config_ms.store( startup_end_time .signed_duration_since(start_time) @@ -300,35 +333,7 @@ impl ComputeNode { self.set_status(ComputeStatus::Running); - info!( - "finished configuration of compute for project {}", - self.spec.cluster.cluster_id - ); - - // Wait for child Postgres process basically forever. In this state Ctrl+C - // will propagate to Postgres and it will be shut down as well. - let ecode = pg - .wait() - .expect("failed to start waiting on Postgres process"); - - if let Err(err) = self.check_for_core_dumps() { - error!("error while checking for core dumps: {err:?}"); - } - - Ok(ecode) - } - - pub fn prepare_and_run(&self) -> Result { - info!( - "starting compute for project {}, operation {}, tenant {}, timeline {}", - self.spec.cluster.cluster_id, - self.spec.operation_uuid.as_ref().unwrap(), - self.tenant, - self.timeline, - ); - - self.prepare_pgdata()?; - self.run() + Ok(pg) } // Look for core dumps and collect backtraces. @@ -341,7 +346,7 @@ impl ComputeNode { // // Use that as a default location and pattern, except macos where core dumps are written // to /cores/ directory by default. - fn check_for_core_dumps(&self) -> Result<()> { + pub fn check_for_core_dumps(&self) -> Result<()> { let core_dump_dir = match std::env::consts::OS { "macos" => Path::new("/cores/"), _ => Path::new(&self.pgdata), diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index 44f83e5003..f2a49f332c 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -6,8 +6,8 @@ use std::thread; use anyhow::Result; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; -use log::{error, info}; use serde_json; +use tracing::{error, info}; use crate::compute::ComputeNode; diff --git a/compute_tools/src/informant.rs b/compute_tools/src/informant.rs index 09bd5e3138..8a6e3ab43a 100644 --- a/compute_tools/src/informant.rs +++ b/compute_tools/src/informant.rs @@ -1,8 +1,8 @@ -use log::{info, warn}; use std::path::Path; use std::process; use std::thread; use std::time::Duration; +use tracing::{info, warn}; use anyhow::{Context, Result}; diff --git a/compute_tools/src/logger.rs b/compute_tools/src/logger.rs index dde0a950f8..57e5496e86 100644 --- a/compute_tools/src/logger.rs +++ b/compute_tools/src/logger.rs @@ -1,42 +1,20 @@ -use std::io::Write; - use anyhow::Result; -use chrono::Utc; -use env_logger::{Builder, Env}; - -macro_rules! info_println { - ($($tts:tt)*) => { - if log_enabled!(Level::Info) { - println!($($tts)*); - } - } -} - -macro_rules! info_print { - ($($tts:tt)*) => { - if log_enabled!(Level::Info) { - print!($($tts)*); - } - } -} +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::prelude::*; /// Initialize `env_logger` using either `default_level` or /// `RUST_LOG` environment variable as default log level. pub fn init_logger(default_level: &str) -> Result<()> { - let env = Env::default().filter_or("RUST_LOG", default_level); + let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(default_level)); - Builder::from_env(env) - .format(|buf, record| { - let thread_handle = std::thread::current(); - writeln!( - buf, - "{} [{}] {}: {}", - Utc::now().format("%Y-%m-%d %H:%M:%S%.3f %Z"), - thread_handle.name().unwrap_or("main"), - record.level(), - record.args() - ) - }) + let fmt_layer = tracing_subscriber::fmt::layer() + .with_target(false) + .with_writer(std::io::stderr); + + tracing_subscriber::registry() + .with(env_filter) + .with(fmt_layer) .init(); Ok(()) diff --git a/compute_tools/src/monitor.rs b/compute_tools/src/monitor.rs index c871422e78..7c9878ffcf 100644 --- a/compute_tools/src/monitor.rs +++ b/compute_tools/src/monitor.rs @@ -3,8 +3,8 @@ use std::{thread, time}; use anyhow::Result; use chrono::{DateTime, Utc}; -use log::{debug, info}; use postgres::{Client, NoTls}; +use tracing::{debug, info}; use crate::compute::ComputeNode; diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index ff422f1cf5..921289d7c2 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -11,6 +11,7 @@ use anyhow::{bail, Result}; use notify::{RecursiveMode, Watcher}; use postgres::{Client, Transaction}; use serde::Deserialize; +use tracing::{debug, instrument}; const POSTGRES_WAIT_TIMEOUT: Duration = Duration::from_millis(60 * 1000); // milliseconds @@ -229,6 +230,7 @@ pub fn get_existing_dbs(client: &mut Client) -> Result> { /// Wait for Postgres to become ready to accept connections. It's ready to /// accept connections when the state-field in `pgdata/postmaster.pid` says /// 'ready'. +#[instrument(skip(pg))] pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> { let pid_path = pgdata.join("postmaster.pid"); @@ -287,18 +289,18 @@ pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> { } let res = rx.recv_timeout(Duration::from_millis(100)); - log::debug!("woken up by notify: {res:?}"); + debug!("woken up by notify: {res:?}"); // If there are multiple events in the channel already, we only need to be // check once. Swallow the extra events before we go ahead to check the // pid file. while let Ok(res) = rx.try_recv() { - log::debug!("swallowing extra event: {res:?}"); + debug!("swallowing extra event: {res:?}"); } // Check that we can open pid file first. if let Ok(file) = File::open(&pid_path) { if !postmaster_pid_seen { - log::debug!("postmaster.pid appeared"); + debug!("postmaster.pid appeared"); watcher .unwatch(pgdata) .expect("Failed to remove pgdata dir watch"); @@ -314,7 +316,7 @@ pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> { // Pid file could be there and we could read it, but it could be empty, for example. if let Some(Ok(line)) = last_line { let status = line.trim(); - log::debug!("last line of postmaster.pid: {status:?}"); + debug!("last line of postmaster.pid: {status:?}"); // Now Postgres is ready to accept connections if status == "ready" { @@ -330,7 +332,7 @@ pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> { } } - log::info!("PostgreSQL is now running, continuing to configure it"); + tracing::info!("PostgreSQL is now running, continuing to configure it"); Ok(()) } diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 81e01fe555..40c8366bf4 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -1,12 +1,11 @@ use std::path::Path; use std::str::FromStr; -use std::time::Instant; use anyhow::Result; -use log::{info, log_enabled, warn, Level}; use postgres::config::Config; use postgres::{Client, NoTls}; use serde::Deserialize; +use tracing::{info, info_span, instrument, span_enabled, warn, Level}; use crate::compute::ComputeNode; use crate::config; @@ -80,23 +79,25 @@ pub fn update_pg_hba(pgdata_path: &Path) -> Result<()> { /// Given a cluster spec json and open transaction it handles roles creation, /// deletion and update. +#[instrument(skip_all)] pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { let mut xact = client.transaction()?; let existing_roles: Vec = get_existing_roles(&mut xact)?; // Print a list of existing Postgres roles (only in debug mode) - info!("postgres roles:"); - for r in &existing_roles { - info_println!( - "{} - {}:{}", - " ".repeat(27 + 5), - r.name, - if r.encrypted_password.is_some() { - "[FILTERED]" - } else { - "(null)" - } - ); + if span_enabled!(Level::INFO) { + info!("postgres roles:"); + for r in &existing_roles { + info!( + " - {}:{}", + r.name, + if r.encrypted_password.is_some() { + "[FILTERED]" + } else { + "(null)" + } + ); + } } // Process delta operations first @@ -137,58 +138,68 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { info!("cluster spec roles:"); for role in &spec.cluster.roles { let name = &role.name; - - info_print!( - "{} - {}:{}", - " ".repeat(27 + 5), - name, - if role.encrypted_password.is_some() { - "[FILTERED]" - } else { - "(null)" - } - ); - // XXX: with a limited number of roles it is fine, but consider making it a HashMap let pg_role = existing_roles.iter().find(|r| r.name == *name); - if let Some(r) = pg_role { - let mut update_role = false; - + enum RoleAction { + None, + Update, + Create, + } + let action = if let Some(r) = pg_role { if (r.encrypted_password.is_none() && role.encrypted_password.is_some()) || (r.encrypted_password.is_some() && role.encrypted_password.is_none()) { - update_role = true; + RoleAction::Update } else if let Some(pg_pwd) = &r.encrypted_password { // Check whether password changed or not (trim 'md5:' prefix first) - update_role = pg_pwd[3..] != *role.encrypted_password.as_ref().unwrap(); + if pg_pwd[3..] != *role.encrypted_password.as_ref().unwrap() { + RoleAction::Update + } else { + RoleAction::None + } + } else { + RoleAction::None } + } else { + RoleAction::Create + }; - if update_role { + match action { + RoleAction::None => {} + RoleAction::Update => { let mut query: String = format!("ALTER ROLE {} ", name.pg_quote()); - info_print!(" -> update"); - query.push_str(&role.to_pg_options()); xact.execute(query.as_str(), &[])?; } - } else { - info!("role name: '{}'", &name); - let mut query: String = format!("CREATE ROLE {} ", name.pg_quote()); - info!("role create query: '{}'", &query); - info_print!(" -> create"); + RoleAction::Create => { + let mut query: String = format!("CREATE ROLE {} ", name.pg_quote()); + info!("role create query: '{}'", &query); + query.push_str(&role.to_pg_options()); + xact.execute(query.as_str(), &[])?; - query.push_str(&role.to_pg_options()); - xact.execute(query.as_str(), &[])?; - - let grant_query = format!( - "GRANT pg_read_all_data, pg_write_all_data TO {}", - name.pg_quote() - ); - xact.execute(grant_query.as_str(), &[])?; - info!("role grant query: '{}'", &grant_query); + let grant_query = format!( + "GRANT pg_read_all_data, pg_write_all_data TO {}", + name.pg_quote() + ); + xact.execute(grant_query.as_str(), &[])?; + info!("role grant query: '{}'", &grant_query); + } } - info_print!("\n"); + if span_enabled!(Level::INFO) { + let pwd = if role.encrypted_password.is_some() { + "[FILTERED]" + } else { + "(null)" + }; + let action_str = match action { + RoleAction::None => "", + RoleAction::Create => " -> create", + RoleAction::Update => " -> update", + }; + info!(" - {}:{}{}", name, pwd, action_str); + } } xact.commit()?; @@ -197,6 +208,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { } /// Reassign all dependent objects and delete requested roles. +#[instrument(skip_all)] pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result<()> { if let Some(ops) = &node.spec.delta_operations { // First, reassign all dependent objects to db owners. @@ -261,13 +273,16 @@ fn reassign_owned_objects(node: &ComputeNode, role_name: &PgIdent) -> Result<()> /// like `CREATE DATABASE` and `DROP DATABASE` do not support it. Statement-level /// atomicity should be enough here due to the order of operations and various checks, /// which together provide us idempotency. +#[instrument(skip_all)] pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { let existing_dbs: Vec = get_existing_dbs(client)?; // Print a list of existing Postgres databases (only in debug mode) - info!("postgres databases:"); - for r in &existing_dbs { - info_println!("{} - {}:{}", " ".repeat(27 + 5), r.name, r.owner); + if span_enabled!(Level::INFO) { + info!("postgres databases:"); + for r in &existing_dbs { + info!(" {}:{}", r.name, r.owner); + } } // Process delta operations first @@ -310,13 +325,15 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { for db in &spec.cluster.databases { let name = &db.name; - info_print!("{} - {}:{}", " ".repeat(27 + 5), db.name, db.owner); - // XXX: with a limited number of databases it is fine, but consider making it a HashMap let pg_db = existing_dbs.iter().find(|r| r.name == *name); - let start_time = Instant::now(); - if let Some(r) = pg_db { + enum DatabaseAction { + None, + Update, + Create, + } + let action = if let Some(r) = pg_db { // XXX: db owner name is returned as quoted string from Postgres, // when quoting is needed. let new_owner = if r.owner.starts_with('"') { @@ -326,29 +343,42 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { }; if new_owner != r.owner { + // Update the owner + DatabaseAction::Update + } else { + DatabaseAction::None + } + } else { + DatabaseAction::Create + }; + + match action { + DatabaseAction::None => {} + DatabaseAction::Update => { let query: String = format!( "ALTER DATABASE {} OWNER TO {}", name.pg_quote(), db.owner.pg_quote() ); - info_print!(" -> update"); - + let _ = info_span!("executing", query).entered(); client.execute(query.as_str(), &[])?; - let elapsed = start_time.elapsed().as_millis(); - info_print!(" ({} ms)", elapsed); } - } else { - let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote()); - info_print!(" -> create"); + DatabaseAction::Create => { + let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote()); + query.push_str(&db.to_pg_options()); + let _ = info_span!("executing", query).entered(); + client.execute(query.as_str(), &[])?; + } + }; - query.push_str(&db.to_pg_options()); - client.execute(query.as_str(), &[])?; - - let elapsed = start_time.elapsed().as_millis(); - info_print!(" ({} ms)", elapsed); + if span_enabled!(Level::INFO) { + let action_str = match action { + DatabaseAction::None => "", + DatabaseAction::Create => " -> create", + DatabaseAction::Update => " -> update", + }; + info!(" - {}:{}{}", db.name, db.owner, action_str); } - - info_print!("\n"); } Ok(()) @@ -356,6 +386,7 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { /// Grant CREATE ON DATABASE to the database owner and do some other alters and grants /// to allow users creating trusted extensions and re-creating `public` schema, for example. +#[instrument(skip_all)] pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> { let spec = &node.spec; diff --git a/test_runner/regress/test_compute_ctl.py b/test_runner/regress/test_compute_ctl.py index f973bd8e60..05ac3841dc 100644 --- a/test_runner/regress/test_compute_ctl.py +++ b/test_runner/regress/test_compute_ctl.py @@ -194,7 +194,7 @@ def test_sync_safekeepers_logs(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): ) except TimeoutExpired as exc: ctl_logs = (exc.stderr or b"").decode("utf-8") - log.info("compute_ctl output:\n{ctl_logs}") + log.info(f"compute_ctl stderr:\n{ctl_logs}") with ExternalProcessManager(Path(pgdata) / "postmaster.pid"): start = "starting safekeepers syncing" diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 8addfcf72e..f4b71ae9b7 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -26,7 +26,7 @@ futures-util = { version = "0.3", features = ["channel", "io", "sink"] } indexmap = { version = "1", default-features = false, features = ["std"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["serde", "std"] } +log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } nom = { version = "7" } num-bigint = { version = "0.4" } @@ -45,6 +45,7 @@ tokio-util = { version = "0.7", features = ["codec", "io"] } tower = { version = "0.4", features = ["balance", "buffer", "limit", "retry", "timeout", "util"] } tracing = { version = "0.1", features = ["log"] } tracing-core = { version = "0.1" } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } url = { version = "2", features = ["serde"] } [build-dependencies] @@ -54,7 +55,7 @@ either = { version = "1" } indexmap = { version = "1", default-features = false, features = ["std"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["serde", "std"] } +log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } nom = { version = "7" } prost = { version = "0.11" } From a9bd05760f5d5787286c607e690b603e7f218412 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 19 Jan 2023 10:29:15 -0500 Subject: [PATCH 008/102] Improve layer map docstrings (#3382) --- pageserver/src/tenant/layer_map.rs | 52 ++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index 44bed5959f..01c5359e88 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -250,15 +250,32 @@ where L: ?Sized + Layer, { /// - /// Find the latest layer that covers the given 'key', with lsn < - /// 'end_lsn'. + /// Find the latest layer (by lsn.end) that covers the given + /// 'key', with lsn.start < 'end_lsn'. /// - /// Returns the layer, if any, and an 'lsn_floor' value that - /// indicates which portion of the layer the caller should - /// check. 'lsn_floor' is normally the start-LSN of the layer, but - /// can be greater if there is an overlapping layer that might - /// contain the version, even if it's missing from the returned - /// layer. + /// The caller of this function is the page reconstruction + /// algorithm looking for the next relevant delta layer, or + /// the terminal image layer. The caller will pass the lsn_floor + /// value as end_lsn in the next call to search. + /// + /// If there's an image layer exactly below the given end_lsn, + /// search should return that layer regardless if there are + /// overlapping deltas. + /// + /// If the latest layer is a delta and there is an overlapping + /// image with it below, the lsn_floor returned should be right + /// above that image so we don't skip it in the search. Otherwise + /// the lsn_floor returned should be the bottom of the delta layer + /// because we should make as much progress down the lsn axis + /// as possible. It's fine if this way we skip some overlapping + /// deltas, because the delta we returned would contain the same + /// wal content. + /// + /// TODO: This API is convoluted and inefficient. If the caller + /// makes N search calls, we'll end up finding the same latest + /// image layer N times. We should either cache the latest image + /// layer result, or simplify the api to `get_latest_image` and + /// `get_latest_delta`, and only call `get_latest_image` once. /// /// NOTE: This only searches the 'historic' layers, *not* the /// 'open' and 'frozen' layers! @@ -401,7 +418,9 @@ where NUM_ONDISK_LAYERS.dec(); } - /// Is there a newer image layer for given key- and LSN-range? + /// Is there a newer image layer for given key- and LSN-range? Or a set + /// of image layers within the specified lsn range that cover the entire + /// specified key range? /// /// This is used for garbage collection, to determine if an old layer can /// be deleted. @@ -488,8 +507,8 @@ where /// /// Divide the whole given range of keys into sub-ranges based on the latest - /// image layer that covers each range. (This is used when creating new - /// image layers) + /// image layer that covers each range at the specified lsn (inclusive). + /// This is used when creating new image layers. /// // FIXME: clippy complains that the result type is very complex. She's probably // right... @@ -541,8 +560,15 @@ where Ok(ranges) } - /// Count how many L1 delta layers there are that overlap with the - /// given key and LSN range. + /// Count the height of the tallest stack of deltas in this 2d region. + /// + /// This number is used to compute the largest number of deltas that + /// we'll need to visit for any page reconstruction in this region. + /// We use this heuristic to decide whether to create an image layer. + /// + /// TODO currently we just return the total number of deltas in the + /// region, no matter if they're stacked on top of each other + /// or next to each other. pub fn count_deltas(&self, key_range: &Range, lsn_range: &Range) -> Result { let mut result = 0; if lsn_range.start >= lsn_range.end { From 8ba1699937049b88aa60a55b35a9029557f1f90f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 19 Jan 2023 18:49:36 +0100 Subject: [PATCH 009/102] Revert "Use actual temporary dir for pageserver unit tests" This reverts commit 826e89b9ce43ce2c4d046b2c5d6376c3de8dbbac. The problem with that commit was that it deletes the TempDir while there are still EphemeralFile instances open. At first I thought this could be fixed by simply adding Handle::current().block_on(task_mgr::shutdown(None, Some(tenant_id), None)) to TenantHarness::drop, but it turned out to be insufficient. So, reverting the commit until we find a proper solution. refs https://github.com/neondatabase/neon/issues/3385 --- .gitignore | 2 + control_plane/.gitignore | 1 + pageserver/src/config.rs | 5 + pageserver/src/tenant.rs | 105 +++++++++++------- pageserver/src/tenant/ephemeral_file.rs | 38 ++++--- .../src/tenant/remote_timeline_client.rs | 2 +- pageserver/src/virtual_file.rs | 8 +- pageserver/src/walingest.rs | 12 +- .../src/walreceiver/connection_manager.rs | 16 +-- test_runner/sql_regress/.gitignore | 1 + 10 files changed, 110 insertions(+), 80 deletions(-) create mode 100644 control_plane/.gitignore diff --git a/.gitignore b/.gitignore index 2e241ee8cd..f1afdee599 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ /pg_install /target +/tmp_check +/tmp_check_cli __pycache__/ test_output/ .vscode diff --git a/control_plane/.gitignore b/control_plane/.gitignore new file mode 100644 index 0000000000..c1e54a6bcb --- /dev/null +++ b/control_plane/.gitignore @@ -0,0 +1 @@ +tmp_check/ diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index f3e5fb8c1a..51d1664e52 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -693,6 +693,11 @@ impl PageServerConf { Ok(t_conf) } + #[cfg(test)] + pub fn test_repo_dir(test_name: &str) -> PathBuf { + PathBuf::from(format!("../tmp_check/test_{test_name}")) + } + pub fn dummy_conf(repo_dir: PathBuf) -> Self { let pg_distrib_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install"); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index c53c9bc3e1..c18c645e5b 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2626,10 +2626,10 @@ where #[cfg(test)] pub mod harness { use bytes::{Bytes, BytesMut}; + use once_cell::sync::Lazy; use once_cell::sync::OnceCell; - use std::sync::Arc; + use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::{fs, path::PathBuf}; - use tempfile::TempDir; use utils::logging; use utils::lsn::Lsn; @@ -2661,6 +2661,8 @@ pub mod harness { buf.freeze() } + static LOCK: Lazy> = Lazy::new(|| RwLock::new(())); + impl From for TenantConfOpt { fn from(tenant_conf: TenantConf) -> Self { Self { @@ -2681,31 +2683,42 @@ pub mod harness { } } - /// The harness saves some boilerplate and provides a way to create functional tenant - /// without running pageserver binary. It uses temporary directory to store data in it. - /// Tempdir gets removed on harness drop. - pub struct TenantHarness { - // keep the struct to not to remove tmp dir during the test - _temp_repo_dir: TempDir, + pub struct TenantHarness<'a> { pub conf: &'static PageServerConf, pub tenant_conf: TenantConf, pub tenant_id: TenantId, + + pub lock_guard: ( + Option>, + Option>, + ), } static LOG_HANDLE: OnceCell<()> = OnceCell::new(); - impl TenantHarness { - pub fn new() -> anyhow::Result { + impl<'a> TenantHarness<'a> { + pub fn create(test_name: &'static str) -> anyhow::Result { + Self::create_internal(test_name, false) + } + pub fn create_exclusive(test_name: &'static str) -> anyhow::Result { + Self::create_internal(test_name, true) + } + fn create_internal(test_name: &'static str, exclusive: bool) -> anyhow::Result { + let lock_guard = if exclusive { + (None, Some(LOCK.write().unwrap())) + } else { + (Some(LOCK.read().unwrap()), None) + }; + LOG_HANDLE.get_or_init(|| { logging::init(logging::LogFormat::Test).expect("Failed to init test logging") }); - let temp_repo_dir = tempfile::tempdir()?; - // `TempDir` uses a randomly generated subdirectory of a system tmp dir, - // so far it's enough to take care of concurrently running tests. - let repo_dir = temp_repo_dir.path(); + let repo_dir = PageServerConf::test_repo_dir(test_name); + let _ = fs::remove_dir_all(&repo_dir); + fs::create_dir_all(&repo_dir)?; - let conf = PageServerConf::dummy_conf(repo_dir.to_path_buf()); + let conf = PageServerConf::dummy_conf(repo_dir); // Make a static copy of the config. This can never be free'd, but that's // OK in a test. let conf: &'static PageServerConf = Box::leak(Box::new(conf)); @@ -2723,10 +2736,10 @@ pub mod harness { fs::create_dir_all(conf.timelines_path(&tenant_id))?; Ok(Self { - _temp_repo_dir: temp_repo_dir, conf, tenant_conf, tenant_id, + lock_guard, }) } @@ -2820,8 +2833,7 @@ mod tests { #[tokio::test] async fn test_basic() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_basic")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2854,8 +2866,9 @@ mod tests { #[tokio::test] async fn no_duplicate_timelines() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("no_duplicate_timelines")? + .load() + .await; let _ = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2886,8 +2899,7 @@ mod tests { /// #[tokio::test] async fn test_branch() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_branch")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -2984,8 +2996,10 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_garbage_collected_data() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = + TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? + .load() + .await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3020,8 +3034,9 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? + .load() + .await; tenant .create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION)? @@ -3070,8 +3085,9 @@ mod tests { #[tokio::test] async fn test_retain_data_in_parent_which_is_needed_for_child() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? + .load() + .await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3093,8 +3109,9 @@ mod tests { } #[tokio::test] async fn test_parent_keeps_data_forever_after_branching() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_parent_keeps_data_forever_after_branching")? + .load() + .await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3125,7 +3142,8 @@ mod tests { #[tokio::test] async fn timeline_load() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + const TEST_NAME: &str = "timeline_load"; + let harness = TenantHarness::create(TEST_NAME)?; { let tenant = harness.load().await; let tline = tenant @@ -3144,7 +3162,8 @@ mod tests { #[tokio::test] async fn timeline_load_with_ancestor() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + const TEST_NAME: &str = "timeline_load_with_ancestor"; + let harness = TenantHarness::create(TEST_NAME)?; // create two timelines { let tenant = harness.load().await; @@ -3182,7 +3201,8 @@ mod tests { #[tokio::test] async fn corrupt_metadata() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + const TEST_NAME: &str = "corrupt_metadata"; + let harness = TenantHarness::create(TEST_NAME)?; let tenant = harness.load().await; tenant @@ -3223,8 +3243,7 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_images")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3291,8 +3310,7 @@ mod tests { // #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_bulk_insert")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3336,8 +3354,7 @@ mod tests { #[tokio::test] async fn test_random_updates() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_random_updates")?.load().await; let tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3410,8 +3427,9 @@ mod tests { #[tokio::test] async fn test_traverse_branches() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_traverse_branches")? + .load() + .await; let mut tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; @@ -3495,8 +3513,9 @@ mod tests { #[tokio::test] async fn test_traverse_ancestors() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_traverse_ancestors")? + .load() + .await; let mut tline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? .initialize()?; diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index 0debeaff1c..c433e65ad2 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -76,7 +76,7 @@ impl EphemeralFile { }) } - fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> io::Result<()> { + fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), io::Error> { let mut off = 0; while off < PAGE_SZ { let n = self @@ -277,7 +277,7 @@ impl Drop for EphemeralFile { } } -pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> io::Result<()> { +pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> Result<(), io::Error> { if let Some(file) = EPHEMERAL_FILES.read().unwrap().files.get(&file_id) { match file.write_all_at(buf, blkno as u64 * PAGE_SZ as u64) { Ok(_) => Ok(()), @@ -332,17 +332,25 @@ mod tests { use super::*; use crate::tenant::blob_io::{BlobCursor, BlobWriter}; use crate::tenant::block_io::BlockCursor; - use crate::tenant::harness::TenantHarness; use rand::{seq::SliceRandom, thread_rng, RngCore}; use std::fs; use std::str::FromStr; - fn harness() -> Result<(TenantHarness, TimelineId), io::Error> { - let harness = TenantHarness::new().expect("Failed to create tenant harness"); - let timeline_id = TimelineId::from_str("22000000000000000000000000000000").unwrap(); - fs::create_dir_all(harness.timeline_path(&timeline_id))?; + fn harness( + test_name: &str, + ) -> Result<(&'static PageServerConf, TenantId, TimelineId), io::Error> { + let repo_dir = PageServerConf::test_repo_dir(test_name); + let _ = fs::remove_dir_all(&repo_dir); + let conf = PageServerConf::dummy_conf(repo_dir); + // Make a static copy of the config. This can never be free'd, but that's + // OK in a test. + let conf: &'static PageServerConf = Box::leak(Box::new(conf)); - Ok((harness, timeline_id)) + let tenant_id = TenantId::from_str("11000000000000000000000000000000").unwrap(); + let timeline_id = TimelineId::from_str("22000000000000000000000000000000").unwrap(); + fs::create_dir_all(conf.timeline_path(&timeline_id, &tenant_id))?; + + Ok((conf, tenant_id, timeline_id)) } // Helper function to slurp contents of a file, starting at the current position, @@ -359,10 +367,10 @@ mod tests { } #[test] - fn test_ephemeral_files() -> io::Result<()> { - let (harness, timeline_id) = harness()?; + fn test_ephemeral_files() -> Result<(), io::Error> { + let (conf, tenant_id, timeline_id) = harness("ephemeral_files")?; - let file_a = EphemeralFile::create(harness.conf, harness.tenant_id, timeline_id)?; + let file_a = EphemeralFile::create(conf, tenant_id, timeline_id)?; file_a.write_all_at(b"foo", 0)?; assert_eq!("foo", read_string(&file_a, 0, 20)?); @@ -373,7 +381,7 @@ mod tests { // Open a lot of files, enough to cause some page evictions. let mut efiles = Vec::new(); for fileno in 0..100 { - let efile = EphemeralFile::create(harness.conf, harness.tenant_id, timeline_id)?; + let efile = EphemeralFile::create(conf, tenant_id, timeline_id)?; efile.write_all_at(format!("file {}", fileno).as_bytes(), 0)?; assert_eq!(format!("file {}", fileno), read_string(&efile, 0, 10)?); efiles.push((fileno, efile)); @@ -390,10 +398,10 @@ mod tests { } #[test] - fn test_ephemeral_blobs() -> io::Result<()> { - let (harness, timeline_id) = harness()?; + fn test_ephemeral_blobs() -> Result<(), io::Error> { + let (conf, tenant_id, timeline_id) = harness("ephemeral_blobs")?; - let mut file = EphemeralFile::create(harness.conf, harness.tenant_id, timeline_id)?; + let mut file = EphemeralFile::create(conf, tenant_id, timeline_id)?; let pos_foo = file.write_blob(b"foo")?; assert_eq!(b"foo", file.block_cursor().read_blob(pos_foo)?.as_slice()); diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 58b7eea1eb..013591caee 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1064,7 +1064,7 @@ mod tests { // Test scheduling #[test] fn upload_scheduling() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("upload_scheduling")?; let timeline_path = harness.timeline_path(&TIMELINE_ID); std::fs::create_dir_all(&timeline_path)?; diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index 3ad049cc21..fb216123c1 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -525,13 +525,12 @@ mod tests { }) } - fn test_files(test_name: &str, openfunc: OF) -> Result<(), Error> + fn test_files(testname: &str, openfunc: OF) -> Result<(), Error> where FD: Read + Write + Seek + FileExt, OF: Fn(&Path, &OpenOptions) -> Result, { - let temp_repo_dir = tempfile::tempdir()?; - let testdir = temp_repo_dir.path().join(test_name); + let testdir = crate::config::PageServerConf::test_repo_dir(testname); std::fs::create_dir_all(&testdir)?; let path_a = testdir.join("file_a"); @@ -633,8 +632,7 @@ mod tests { const THREADS: usize = 100; const SAMPLE: [u8; SIZE] = [0xADu8; SIZE]; - let temp_repo_dir = tempfile::tempdir()?; - let testdir = temp_repo_dir.path().join("vfile_concurrency"); + let testdir = crate::config::PageServerConf::test_repo_dir("vfile_concurrency"); std::fs::create_dir_all(&testdir)?; // Create a test file. diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 77fce95160..0de2e6654d 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1146,8 +1146,7 @@ mod tests { #[tokio::test] async fn test_relsize() -> Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_relsize")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; @@ -1324,8 +1323,7 @@ mod tests { // and then created it again within the same layer. #[tokio::test] async fn test_drop_extend() -> Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_drop_extend")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; @@ -1378,8 +1376,7 @@ mod tests { // and then extended it again within the same layer. #[tokio::test] async fn test_truncate_extend() -> Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_truncate_extend")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; @@ -1500,8 +1497,7 @@ mod tests { /// split into multiple 1 GB segments in Postgres. #[tokio::test] async fn test_large_rel() -> Result<()> { - let harness = TenantHarness::new()?; - let tenant = harness.load().await; + let tenant = TenantHarness::create("test_large_rel")?.load().await; let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; let mut walingest = init_walingest_test(&tline).await?; diff --git a/pageserver/src/walreceiver/connection_manager.rs b/pageserver/src/walreceiver/connection_manager.rs index be58aa0e07..8b60e59305 100644 --- a/pageserver/src/walreceiver/connection_manager.rs +++ b/pageserver/src/walreceiver/connection_manager.rs @@ -846,7 +846,7 @@ mod tests { #[tokio::test] async fn no_connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("no_connection_no_candidate")?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -879,7 +879,7 @@ mod tests { #[tokio::test] async fn connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("connection_no_candidate")?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -942,7 +942,7 @@ mod tests { #[tokio::test] async fn no_connection_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("no_connection_candidate")?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1001,7 +1001,7 @@ mod tests { #[tokio::test] async fn candidate_with_many_connection_failures() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("candidate_with_many_connection_failures")?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1041,7 +1041,7 @@ mod tests { #[tokio::test] async fn lsn_wal_over_threshhold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1105,7 +1105,7 @@ mod tests { #[tokio::test] async fn timeout_connection_threshhold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("timeout_connection_threshhold_current_candidate")?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1166,7 +1166,7 @@ mod tests { #[tokio::test] async fn timeout_wal_over_threshhold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::new()?; + let harness = TenantHarness::create("timeout_wal_over_threshhold_current_candidate")?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let new_lsn = Lsn(100_100).align(); @@ -1232,7 +1232,7 @@ mod tests { const DUMMY_SAFEKEEPER_HOST: &str = "safekeeper_connstr"; - async fn dummy_state(harness: &TenantHarness) -> WalreceiverState { + async fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState { WalreceiverState { id: TenantTimelineId { tenant_id: harness.tenant_id, diff --git a/test_runner/sql_regress/.gitignore b/test_runner/sql_regress/.gitignore index 83186b5c86..89129d7358 100644 --- a/test_runner/sql_regress/.gitignore +++ b/test_runner/sql_regress/.gitignore @@ -2,6 +2,7 @@ /pg_regress # Generated subdirectories +/tmp_check/ /results/ /log/ From 20b1e26e7412090350ffff33a04fd91912c2023f Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 20 Jan 2023 15:37:24 +0100 Subject: [PATCH 010/102] [compute_ctl] Make role deletion spec processing idempotent (#3380) Previously, we were trying to re-assign owned objects of the already deleted role. This were causing a crash loop in the case when compute was restarted with a spec that includes delta operation for role deletion. To avoid such cases, check that role is still present before calling `reassign_owned_objects`. Resolves neondatabase/cloud#3553 --- compute_tools/src/compute.rs | 3 ++- compute_tools/src/pg_helpers.rs | 4 ++-- compute_tools/src/spec.rs | 14 +++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index e229bb1cc2..c8af8822b7 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -252,7 +252,7 @@ impl ComputeNode { // If connection fails, // it may be the old node with `zenith_admin` superuser. // - // In this case we need to connect with old `zenith_admin`name + // In this case we need to connect with old `zenith_admin` name // and create new user. We cannot simply rename connected user, // but we can create a new one and grant it all privileges. let mut client = match Client::connect(self.connstr.as_str(), NoTls) { @@ -278,6 +278,7 @@ impl ComputeNode { Ok(client) => client, }; + // Proceed with post-startup configuration. Note, that order of operations is important. handle_roles(&self.spec, &mut client)?; handle_databases(&self.spec, &mut client)?; handle_role_deletions(self, &mut client)?; diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index 921289d7c2..6ab2864721 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -130,8 +130,8 @@ impl Role { /// Serialize a list of role parameters into a Postgres-acceptable /// string of arguments. pub fn to_pg_options(&self) -> String { - // XXX: consider putting LOGIN as a default option somewhere higher, e.g. in Rails. - // For now we do not use generic `options` for roles. Once used, add + // XXX: consider putting LOGIN as a default option somewhere higher, e.g. in control-plane. + // For now, we do not use generic `options` for roles. Once used, add // `self.options.as_pg_options()` somewhere here. let mut params: String = "LOGIN".to_string(); diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 40c8366bf4..97cd623052 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -213,8 +213,20 @@ pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result< if let Some(ops) = &node.spec.delta_operations { // First, reassign all dependent objects to db owners. info!("reassigning dependent objects of to-be-deleted roles"); + + // Fetch existing roles. We could've exported and used `existing_roles` from + // `handle_roles()`, but we only make this list there before creating new roles. + // Which is probably fine as we never create to-be-deleted roles, but that'd + // just look a bit untidy. Anyway, the entire `pg_roles` should be in shared + // buffers already, so this shouldn't be a big deal. + let mut xact = client.transaction()?; + let existing_roles: Vec = get_existing_roles(&mut xact)?; + xact.commit()?; + for op in ops { - if op.action == "delete_role" { + // Check that role is still present in Postgres, as this could be a + // restart with the same spec after role deletion. + if op.action == "delete_role" && existing_roles.iter().any(|r| r.name == op.name) { reassign_owned_objects(node, &op.name)?; } } From 58fb6fe861353ef62cfe16b6feca6fac7f158971 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 20 Jan 2023 18:19:52 +0200 Subject: [PATCH 011/102] fix: dont stop pageserver if we fail to calculate synthetic size --- pageserver/src/consumption_metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index f8a0bc6f08..c07026261d 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -59,7 +59,7 @@ pub async fn collect_metrics( None, None, "synthetic size calculation", - true, + false, async move { calculate_synthetic_size_worker(synthetic_size_calculation_interval) .instrument(info_span!("synthetic_size_worker")) From 36f048d6b0c85415a99ced28c9c493fdaa8b281e Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Fri, 20 Jan 2023 20:21:36 +0200 Subject: [PATCH 012/102] Fix tenant size orphans (#3377) Before only the timelines which have passed the `gc_horizon` were processed which failed with orphans at the tree_sort phase. Example input in added `test_branched_empty_timeline_size` test case. The PR changes iteration to happen through all timelines, and in addition to that, any learned branch points will be calculated as they would had been in the original implementation if the ancestor branch had been over the `gc_horizon`. This also changes how tenants where all timelines are below `gc_horizon` are handled. Previously tenant_size 0 was returned, but now they will have approximately `initdb_lsn` worth of tenant_size. The PR also adds several new tenant size tests that describe various corner cases of branching structure and `gc_horizon` setting. They are currently disabled to not consume time during CI. Co-authored-by: Joonas Koivunen Co-authored-by: Anastasia Lubennikova --- pageserver/src/tenant/size.rs | 149 ++++++++++++--- test_runner/fixtures/neon_fixtures.py | 9 +- test_runner/regress/test_tenant_size.py | 244 ++++++++++++++++++++++-- 3 files changed, 360 insertions(+), 42 deletions(-) diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index dd4bf768a7..2181d6d4dc 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -23,7 +23,13 @@ use tracing::*; pub struct ModelInputs { updates: Vec, retention_period: u64, + + /// Relevant lsns per timeline. + /// + /// This field is not required for deserialization purposes, which is mostly used in tests. The + /// LSNs explain the outcome (updates) but are not needed in size calculation. #[serde_as(as = "HashMap")] + #[serde(default)] timeline_inputs: HashMap, } @@ -32,6 +38,8 @@ pub struct ModelInputs { #[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] struct TimelineInputs { + #[serde_as(as = "serde_with::DisplayFromStr")] + ancestor_lsn: Lsn, #[serde_as(as = "serde_with::DisplayFromStr")] last_record: Lsn, #[serde_as(as = "serde_with::DisplayFromStr")] @@ -178,21 +186,13 @@ pub(super) async fn gather_inputs( // our advantage with `?` error handling. let mut joinset = tokio::task::JoinSet::new(); - let timelines = tenant + // refresh is needed to update gc related pitr_cutoff and horizon_cutoff + tenant .refresh_gc_info() .await .context("Failed to refresh gc_info before gathering inputs")?; - if timelines.is_empty() { - // All timelines are below tenant's gc_horizon; alternative would be to use - // Tenant::list_timelines but then those gc_info's would not be updated yet, possibly - // missing GcInfo::retain_lsns or having obsolete values for cutoff's. - return Ok(ModelInputs { - updates: vec![], - retention_period: 0, - timeline_inputs: HashMap::new(), - }); - } + let timelines = tenant.list_timelines(); // record the used/inserted cache keys here, to remove extras not to start leaking // after initial run the cache should be quite stable, but live timelines will eventually @@ -201,13 +201,25 @@ pub(super) async fn gather_inputs( let mut updates = Vec::new(); - // record the per timline values used to determine `retention_period` + // record the per timeline values useful to debug the model inputs, also used to track + // ancestor_lsn without keeping a hold of Timeline let mut timeline_inputs = HashMap::with_capacity(timelines.len()); // used to determine the `retention_period` for the size model let mut max_cutoff_distance = None; + // mapping from (TimelineId, Lsn) => if this branch point has been handled already via + // GcInfo::retain_lsns or if it needs to have its logical_size calculated. + let mut referenced_branch_froms = HashMap::<(TimelineId, Lsn), bool>::new(); + for timeline in timelines { + if !timeline.is_active() { + anyhow::bail!( + "timeline {} is not active, cannot calculate tenant_size now", + timeline.timeline_id + ); + } + let last_record_lsn = timeline.get_last_record_lsn(); let (interesting_lsns, horizon_cutoff, pitr_cutoff, next_gc_cutoff) = { @@ -273,13 +285,30 @@ pub(super) async fn gather_inputs( // all timelines branch from something, because it might be impossible to pinpoint // which is the tenant_size_model's "default" branch. + + let ancestor_lsn = timeline.get_ancestor_lsn(); + updates.push(Update { - lsn: timeline.get_ancestor_lsn(), + lsn: ancestor_lsn, command: Command::BranchFrom(timeline.get_ancestor_timeline_id()), timeline_id: timeline.timeline_id, }); + if let Some(parent_timeline_id) = timeline.get_ancestor_timeline_id() { + // refresh_gc_info will update branchpoints and pitr_cutoff but only do it for branches + // which are over gc_horizon. for example, a "main" branch which never received any + // updates apart from initdb not have branch points recorded. + referenced_branch_froms + .entry((parent_timeline_id, timeline.get_ancestor_lsn())) + .or_default(); + } + for (lsn, _kind) in &interesting_lsns { + // mark this visited so don't need to re-process this parent + *referenced_branch_froms + .entry((timeline.timeline_id, *lsn)) + .or_default() = true; + if let Some(size) = logical_size_cache.get(&(timeline.timeline_id, *lsn)) { updates.push(Update { lsn: *lsn, @@ -295,22 +324,10 @@ pub(super) async fn gather_inputs( } } - // all timelines also have an end point if they have made any progress - if last_record_lsn > timeline.get_ancestor_lsn() - && !interesting_lsns - .iter() - .any(|(lsn, _)| lsn == &last_record_lsn) - { - updates.push(Update { - lsn: last_record_lsn, - command: Command::EndOfBranch, - timeline_id: timeline.timeline_id, - }); - } - timeline_inputs.insert( timeline.timeline_id, TimelineInputs { + ancestor_lsn, last_record: last_record_lsn, // this is not used above, because it might not have updated recently enough latest_gc_cutoff: *timeline.get_latest_gc_cutoff_lsn(), @@ -321,6 +338,80 @@ pub(super) async fn gather_inputs( ); } + // iterate over discovered branch points and make sure we are getting logical sizes at those + // points. + for ((timeline_id, lsn), handled) in referenced_branch_froms.iter() { + if *handled { + continue; + } + + let timeline_id = *timeline_id; + let lsn = *lsn; + + match timeline_inputs.get(&timeline_id) { + Some(inputs) if inputs.ancestor_lsn == lsn => { + // we don't need an update at this branch point which is also point where + // timeline_id branch was branched from. + continue; + } + Some(_) => {} + None => { + // we should have this because we have iterated through all of the timelines + anyhow::bail!("missing timeline_input for {timeline_id}") + } + } + + if let Some(size) = logical_size_cache.get(&(timeline_id, lsn)) { + updates.push(Update { + lsn, + timeline_id, + command: Command::Update(*size), + }); + + needed_cache.insert((timeline_id, lsn)); + } else { + let timeline = tenant + .get_timeline(timeline_id, false) + .context("find referenced ancestor timeline")?; + let parallel_size_calcs = Arc::clone(limit); + joinset.spawn(calculate_logical_size( + parallel_size_calcs, + timeline.clone(), + lsn, + )); + + if let Some(parent_id) = timeline.get_ancestor_timeline_id() { + // we should not find new ones because we iterated tenants all timelines + anyhow::ensure!( + timeline_inputs.contains_key(&parent_id), + "discovered new timeline {parent_id} (parent of {timeline_id})" + ); + } + }; + } + + // finally add in EndOfBranch for all timelines where their last_record_lsn is not a branch + // point. this is needed by the model. + for (timeline_id, inputs) in timeline_inputs.iter() { + let lsn = inputs.last_record; + + if referenced_branch_froms.contains_key(&(*timeline_id, lsn)) { + // this means that the (timeline_id, last_record_lsn) represents a branch point + // we do not want to add EndOfBranch updates for these points because it doesn't fit + // into the current tenant_size_model. + continue; + } + + if lsn > inputs.ancestor_lsn { + // all timelines also have an end point if they have made any progress + updates.push(Update { + lsn, + command: Command::EndOfBranch, + timeline_id: *timeline_id, + }); + } + } + let mut have_any_error = false; while let Some(res) = joinset.join_next().await { @@ -379,6 +470,7 @@ pub(super) async fn gather_inputs( // handled by the variant order in `Command`. // updates.sort_unstable(); + // And another sort to handle Command::BranchFrom ordering // in case when there are multiple branches at the same LSN. let sorted_updates = sort_updates_in_tree_order(updates)?; @@ -574,7 +666,10 @@ fn updates_sort() { fn verify_size_for_multiple_branches() { // this is generated from integration test test_tenant_size_with_multiple_branches, but this way // it has the stable lsn's - let doc = r#"{"updates":[{"lsn":"0/0","command":{"branch_from":null},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"update":25763840},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/1819818","command":{"update":26075136},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/18B5E40","command":{"update":26427392},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"update":26492928},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"230fc9d756f7363574c0d66533564dcc"},{"lsn":"0/220F438","command":{"update":25239552},"timeline_id":"230fc9d756f7363574c0d66533564dcc"}],"retention_period":131072,"timeline_inputs":{"cd9d9409c216e64bf580904facedb01b":{"last_record":"0/18D5E40","latest_gc_cutoff":"0/169ACF0","horizon_cutoff":"0/18B5E40","pitr_cutoff":"0/18B5E40","next_gc_cutoff":"0/18B5E40"},"10b532a550540bc15385eac4edde416a":{"last_record":"0/1839818","latest_gc_cutoff":"0/169ACF0","horizon_cutoff":"0/1819818","pitr_cutoff":"0/1819818","next_gc_cutoff":"0/1819818"},"230fc9d756f7363574c0d66533564dcc":{"last_record":"0/222F438","latest_gc_cutoff":"0/169ACF0","horizon_cutoff":"0/220F438","pitr_cutoff":"0/220F438","next_gc_cutoff":"0/220F438"}}}"#; + // + // timelineinputs have been left out, because those explain the inputs, but don't participate + // in further size calculations. + let doc = r#"{"updates":[{"lsn":"0/0","command":{"branch_from":null},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"update":25763840},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/1819818","command":{"update":26075136},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/18B5E40","command":{"update":26427392},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"update":26492928},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"230fc9d756f7363574c0d66533564dcc"},{"lsn":"0/220F438","command":{"update":25239552},"timeline_id":"230fc9d756f7363574c0d66533564dcc"}],"retention_period":131072}"#; let inputs: ModelInputs = serde_json::from_str(doc).unwrap(); diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index d6c4c32b0b..8476066691 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1206,6 +1206,9 @@ class PageserverHttpClient(requests.Session): return res_json def tenant_size(self, tenant_id: TenantId) -> int: + return self.tenant_size_and_modelinputs(tenant_id)[0] + + def tenant_size_and_modelinputs(self, tenant_id: TenantId) -> Tuple[int, Dict[str, Any]]: """ Returns the tenant size, together with the model inputs as the second tuple item. """ @@ -1216,9 +1219,9 @@ class PageserverHttpClient(requests.Session): assert TenantId(res["id"]) == tenant_id size = res["size"] assert type(size) == int - # there are additional inputs, which are the collected raw information before being fed to the tenant_size_model - # there are no tests for those right now. - return size + inputs = res["inputs"] + assert type(inputs) is dict + return (size, inputs) def timeline_list( self, diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index 5747ae235f..72cfbc9dda 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -1,5 +1,6 @@ -from typing import List, Tuple +from typing import Any, List, Tuple +import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, wait_for_last_flush_lsn from fixtures.types import Lsn @@ -9,28 +10,247 @@ def test_empty_tenant_size(neon_simple_env: NeonEnv): env = neon_simple_env (tenant_id, _) = env.neon_cli.create_tenant() http_client = env.pageserver.http_client() - size = http_client.tenant_size(tenant_id) + initial_size = http_client.tenant_size(tenant_id) - # we should never have zero, because there should be the initdb however - # this is questionable if we should have anything in this case, as the - # gc_cutoff is negative - assert ( - size == 0 - ), "initial implementation returns zero tenant_size before last_record_lsn is past gc_horizon" + # we should never have zero, because there should be the initdb "changes" + assert initial_size > 0, "initial implementation returns ~initdb tenant_size" - with env.postgres.create_start("main", tenant_id=tenant_id) as pg: + main_branch_name = "main" + + with env.postgres.create_start( + main_branch_name, + tenant_id=tenant_id, + config_lines=["autovacuum=off", "checkpoint_timeout=10min"], + ) as pg: with pg.cursor() as cur: cur.execute("SELECT 1") row = cur.fetchone() assert row is not None assert row[0] == 1 size = http_client.tenant_size(tenant_id) - assert size == 0, "starting idle compute should not change the tenant size" + # we've disabled the autovacuum and checkpoint + # so background processes should not change the size. + # If this test will flake we should probably loosen the check + assert size == initial_size, "starting idle compute should not change the tenant size" # the size should be the same, until we increase the size over the # gc_horizon - size = http_client.tenant_size(tenant_id) - assert size == 0, "tenant_size should not be affected by shutdown of compute" + size, inputs = http_client.tenant_size_and_modelinputs(tenant_id) + assert size == initial_size, "tenant_size should not be affected by shutdown of compute" + + expected_commands: List[Any] = [{"branch_from": None}, "end_of_branch"] + actual_commands: List[Any] = list(map(lambda x: x["command"], inputs["updates"])) # type: ignore + assert actual_commands == expected_commands + + +def test_branched_empty_timeline_size(neon_simple_env: NeonEnv): + """ + Issue found in production. Because the ancestor branch was under + gc_horizon, the branchpoint was "dangling" and the computation could not be + done. + + Assuming gc_horizon = 50 + root: I 0---10------>20 + branch: |-------------------I---------->150 + gc_horizon + """ + env = neon_simple_env + (tenant_id, _) = env.neon_cli.create_tenant() + http_client = env.pageserver.http_client() + + initial_size = http_client.tenant_size(tenant_id) + + first_branch_timeline_id = env.neon_cli.create_branch("first-branch", tenant_id=tenant_id) + + with env.postgres.create_start("first-branch", tenant_id=tenant_id) as pg: + with pg.cursor() as cur: + cur.execute( + "CREATE TABLE t0 AS SELECT i::bigint n FROM generate_series(0, 1000000) s(i)" + ) + wait_for_last_flush_lsn(env, pg, tenant_id, first_branch_timeline_id) + + size_after_branching = http_client.tenant_size(tenant_id) + log.info(f"size_after_branching: {size_after_branching}") + + assert size_after_branching > initial_size + + +def test_branched_from_many_empty_parents_size(neon_simple_env: NeonEnv): + """ + More general version of test_branched_empty_timeline_size + + Assuming gc_horizon = 50 + + root: I 0------10 + first: I 10 + nth_0: I 10 + nth_1: I 10 + nth_n: 10------------I--------100 + """ + env = neon_simple_env + (tenant_id, _) = env.neon_cli.create_tenant() + http_client = env.pageserver.http_client() + + initial_size = http_client.tenant_size(tenant_id) + + first_branch_name = "first" + env.neon_cli.create_branch(first_branch_name, tenant_id=tenant_id) + + size_after_branching = http_client.tenant_size(tenant_id) + + # this might be flaky like test_get_tenant_size_with_multiple_branches + # https://github.com/neondatabase/neon/issues/2962 + assert size_after_branching == initial_size + + last_branch_name = first_branch_name + last_branch = None + + for i in range(0, 4): + latest_branch_name = f"nth_{i}" + last_branch = env.neon_cli.create_branch( + latest_branch_name, ancestor_branch_name=last_branch_name, tenant_id=tenant_id + ) + last_branch_name = latest_branch_name + + size_after_branching = http_client.tenant_size(tenant_id) + assert size_after_branching == initial_size + + assert last_branch is not None + + with env.postgres.create_start(last_branch_name, tenant_id=tenant_id) as pg: + with pg.cursor() as cur: + cur.execute( + "CREATE TABLE t0 AS SELECT i::bigint n FROM generate_series(0, 1000000) s(i)" + ) + wait_for_last_flush_lsn(env, pg, tenant_id, last_branch) + + size_after_writes = http_client.tenant_size(tenant_id) + assert size_after_writes > initial_size + + +@pytest.mark.skip("This should work, but is left out because assumed covered by other tests") +def test_branch_point_within_horizon(neon_simple_env: NeonEnv): + """ + gc_horizon = 15 + + main: 0--I-10------>20 + branch: |-------------------I---------->150 + gc_horizon + """ + + env = neon_simple_env + gc_horizon = 20_000 + (tenant_id, main_id) = env.neon_cli.create_tenant(conf={"gc_horizon": str(gc_horizon)}) + http_client = env.pageserver.http_client() + + with env.postgres.create_start("main", tenant_id=tenant_id) as pg: + initdb_lsn = wait_for_last_flush_lsn(env, pg, tenant_id, main_id) + with pg.cursor() as cur: + cur.execute("CREATE TABLE t0 AS SELECT i::bigint n FROM generate_series(0, 1000) s(i)") + flushed_lsn = wait_for_last_flush_lsn(env, pg, tenant_id, main_id) + + size_before_branching = http_client.tenant_size(tenant_id) + + assert flushed_lsn.lsn_int - gc_horizon > initdb_lsn.lsn_int + + branch_id = env.neon_cli.create_branch( + "branch", tenant_id=tenant_id, ancestor_start_lsn=flushed_lsn + ) + + with env.postgres.create_start("branch", tenant_id=tenant_id) as pg: + with pg.cursor() as cur: + cur.execute("CREATE TABLE t1 AS SELECT i::bigint n FROM generate_series(0, 1000) s(i)") + wait_for_last_flush_lsn(env, pg, tenant_id, branch_id) + + size_after = http_client.tenant_size(tenant_id) + + assert size_before_branching < size_after + + +@pytest.mark.skip("This should work, but is left out because assumed covered by other tests") +def test_parent_within_horizon(neon_simple_env: NeonEnv): + """ + gc_horizon = 5 + + main: 0----10----I->20 + branch: |-------------------I---------->150 + gc_horizon + """ + + env = neon_simple_env + gc_horizon = 200_000 + (tenant_id, main_id) = env.neon_cli.create_tenant(conf={"gc_horizon": str(gc_horizon)}) + http_client = env.pageserver.http_client() + + with env.postgres.create_start("main", tenant_id=tenant_id) as pg: + initdb_lsn = wait_for_last_flush_lsn(env, pg, tenant_id, main_id) + with pg.cursor() as cur: + cur.execute("CREATE TABLE t0 AS SELECT i::bigint n FROM generate_series(0, 1000) s(i)") + + flushed_lsn = wait_for_last_flush_lsn(env, pg, tenant_id, main_id) + + with pg.cursor() as cur: + cur.execute("CREATE TABLE t00 AS SELECT i::bigint n FROM generate_series(0, 2000) s(i)") + + wait_for_last_flush_lsn(env, pg, tenant_id, main_id) + + size_before_branching = http_client.tenant_size(tenant_id) + + assert flushed_lsn.lsn_int - gc_horizon > initdb_lsn.lsn_int + + branch_id = env.neon_cli.create_branch( + "branch", tenant_id=tenant_id, ancestor_start_lsn=flushed_lsn + ) + + with env.postgres.create_start("branch", tenant_id=tenant_id) as pg: + with pg.cursor() as cur: + cur.execute("CREATE TABLE t1 AS SELECT i::bigint n FROM generate_series(0, 10000) s(i)") + wait_for_last_flush_lsn(env, pg, tenant_id, branch_id) + + size_after = http_client.tenant_size(tenant_id) + + assert size_before_branching < size_after + + +@pytest.mark.skip("This should work, but is left out because assumed covered by other tests") +def test_only_heads_within_horizon(neon_simple_env: NeonEnv): + """ + gc_horizon = small + + main: 0--------10-----I>20 + first: |-----------------------------I>150 + second: |---------I>30 + """ + + env = neon_simple_env + (tenant_id, main_id) = env.neon_cli.create_tenant(conf={"gc_horizon": "1024"}) + http_client = env.pageserver.http_client() + + initial_size = http_client.tenant_size(tenant_id) + + first_id = env.neon_cli.create_branch("first", tenant_id=tenant_id) + second_id = env.neon_cli.create_branch("second", tenant_id=tenant_id) + + ids = {"main": main_id, "first": first_id, "second": second_id} + + latest_size = None + + # gc is not expected to change the results + + for branch_name, amount in [("main", 2000), ("first", 15000), ("second", 3000)]: + with env.postgres.create_start(branch_name, tenant_id=tenant_id) as pg: + with pg.cursor() as cur: + cur.execute( + f"CREATE TABLE t0 AS SELECT i::bigint n FROM generate_series(0, {amount}) s(i)" + ) + wait_for_last_flush_lsn(env, pg, tenant_id, ids[branch_name]) + size_now = http_client.tenant_size(tenant_id) + if latest_size is not None: + assert size_now > latest_size + else: + assert size_now > initial_size + + latest_size = size_now def test_single_branch_get_tenant_size_grows(neon_env_builder: NeonEnvBuilder): From a3d7ad2d52ec6f13dc532e3034f1024601e541c1 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Fri, 20 Jan 2023 16:10:12 -0500 Subject: [PATCH 013/102] Implement layer map using immutable BST (#2998) --- Cargo.lock | 173 ++-- Cargo.toml | 5 +- pageserver/Cargo.toml | 3 +- pageserver/benches/bench_layer_map.rs | 224 ++--- pageserver/src/repository.rs | 11 + pageserver/src/tenant/layer_map.rs | 877 +++++++++--------- .../layer_map/historic_layer_coverage.rs | 583 ++++++++++++ .../src/tenant/layer_map/layer_coverage.rs | 154 +++ pageserver/src/tenant/storage_layer.rs | 47 + pageserver/src/tenant/timeline.rs | 62 +- workspace_hack/Cargo.toml | 2 +- 11 files changed, 1471 insertions(+), 670 deletions(-) create mode 100644 pageserver/src/tenant/layer_map/historic_layer_coverage.rs create mode 100644 pageserver/src/tenant/layer_map/layer_coverage.rs diff --git a/Cargo.lock b/Cargo.lock index d8aba9ba68..edb739f017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,11 +37,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "amplify_num" -version = "0.4.1" -source = "git+https://github.com/rust-amplify/rust-amplify.git?tag=v4.0.0-beta.1#3ad006cf2804e1862ec7725a7684a493f3023523" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -66,6 +61,15 @@ dependencies = [ "backtrace", ] +[[package]] +name = "archery" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a8da9bc4c4053ee067669762bcaeea6e241841295a2b6c948312dad6ef4cc02" +dependencies = [ + "static_assertions", +] + [[package]] name = "asn1-rs" version = "0.5.1" @@ -137,15 +141,6 @@ dependencies = [ "syn", ] -[[package]] -name = "atomic-polyfill" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ff7eb3f316534d83a8a2c3d1674ace8a5a71198eba31e2e2b597833f699b28" -dependencies = [ - "critical-section", -] - [[package]] name = "atty" version = "0.2.14" @@ -629,9 +624,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -750,13 +745,13 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.32" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "4ec7a4128863c188deefe750ac1d1dfe66c236909f845af04beed823638dc1b2" dependencies = [ "bitflags", "clap_derive", - "clap_lex 0.3.0", + "clap_lex 0.3.1", "is-terminal", "once_cell", "strsim", @@ -765,9 +760,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ "heck", "proc-macro-error", @@ -787,9 +782,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" dependencies = [ "os_str_bytes", ] @@ -832,7 +827,7 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap 4.0.32", + "clap 4.1.1", "futures", "hyper", "notify", @@ -887,7 +882,7 @@ name = "control_plane" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.0.32", + "clap 4.1.1", "comfy-table", "git-version", "nix", @@ -988,12 +983,6 @@ dependencies = [ "itertools", ] -[[package]] -name = "critical-section" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" - [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -1030,12 +1019,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.11" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -1506,15 +1494,6 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" -[[package]] -name = "hash32" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" -dependencies = [ - "byteorder", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1530,19 +1509,6 @@ dependencies = [ "ahash", ] -[[package]] -name = "heapless" -version = "0.7.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" -dependencies = [ - "atomic-polyfill", - "hash32", - "rustc_version", - "spin 0.9.4", - "stable_deref_trait", -] - [[package]] name = "heck" version = "0.4.0" @@ -1804,9 +1770,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", "windows-sys", @@ -1916,12 +1882,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "libm" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" - [[package]] name = "link-cplusplus" version = "1.0.8" @@ -2067,9 +2027,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "nix" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ "bitflags", "cfg-if", @@ -2081,9 +2041,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.2" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -2154,7 +2114,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -2230,14 +2189,13 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" name = "pageserver" version = "0.1.0" dependencies = [ - "amplify_num", "anyhow", "async-stream", "async-trait", "byteorder", "bytes", "chrono", - "clap 4.0.32", + "clap 4.1.1", "close_fds", "const_format", "consumption_metrics", @@ -2269,7 +2227,7 @@ dependencies = [ "regex", "remote_storage", "reqwest", - "rstar", + "rpds", "scopeguard", "serde", "serde_json", @@ -2581,9 +2539,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" dependencies = [ "unicode-ident", ] @@ -2683,7 +2641,7 @@ dependencies = [ "bstr", "bytes", "chrono", - "clap 4.0.32", + "clap 4.1.1", "consumption_metrics", "futures", "git-version", @@ -2742,14 +2700,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", - "rand_hc", ] [[package]] @@ -2771,15 +2728,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core", -] - [[package]] name = "rayon" version = "1.6.1" @@ -2930,7 +2878,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin 0.5.2", + "spin", "untrusted", "web-sys", "winapi", @@ -2950,14 +2898,12 @@ dependencies = [ ] [[package]] -name = "rstar" -version = "0.9.3" +name = "rpds" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b40f1bfe5acdab44bc63e6699c28b74f75ec43afb59f3eda01e145aff86a25fa" +checksum = "66262ea963eff99163e6b741fbc3417a52cc13074728c1047e9911789df9b000" dependencies = [ - "heapless", - "num-traits", - "smallvec", + "archery", ] [[package]] @@ -3018,9 +2964,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.6" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" dependencies = [ "bitflags", "errno", @@ -3093,7 +3039,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "clap 4.0.32", + "clap 4.1.1", "const_format", "crc32c", "fs2", @@ -3479,21 +3425,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "spin" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" -dependencies = [ - "lock_api", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "static_assertions" version = "1.1.0" @@ -3507,7 +3438,7 @@ dependencies = [ "anyhow", "async-stream", "bytes", - "clap 4.0.32", + "clap 4.1.1", "const_format", "futures", "futures-core", @@ -3639,9 +3570,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -3749,9 +3680,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.24.1" +version = "1.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" dependencies = [ "autocfg", "bytes", @@ -4183,9 +4114,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "ureq" -version = "2.6.1" +version = "2.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733b5ad78377302af52c0dbcb2623d78fe50e4b3bf215948ff29e9ee031d8566" +checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" dependencies = [ "base64 0.13.1", "log", @@ -4287,7 +4218,7 @@ name = "wal_craft" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.0.32", + "clap 4.1.1", "env_logger", "log", "once_cell", @@ -4534,7 +4465,7 @@ dependencies = [ "anyhow", "bytes", "chrono", - "clap 4.0.32", + "clap 4.1.1", "crossbeam-utils", "either", "fail", diff --git a/Cargo.toml b/Cargo.toml index 74cc16d690..57f4b1d981 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ rand = "0.8" regex = "1.4" reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } routerify = "3" -rstar = "0.9.3" +rpds = "0.12.0" rustls = "0.20" rustls-pemfile = "1" rustls-split = "0.3" @@ -107,9 +107,6 @@ x509-parser = "0.14" env_logger = "0.10" log = "0.4" -## TODO switch when the new release is made -amplify_num = { git = "https://github.com/rust-amplify/rust-amplify.git", tag = "v4.0.0-beta.1" } - ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index cb9e4478bf..66c25e8576 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -11,7 +11,6 @@ default = [] testing = ["fail/failpoints"] [dependencies] -amplify_num.workspace = true anyhow.workspace = true async-stream.workspace = true async-trait.workspace = true @@ -41,7 +40,6 @@ postgres-protocol.workspace = true postgres-types.workspace = true rand.workspace = true regex.workspace = true -rstar.workspace = true scopeguard.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["raw_value"] } @@ -68,6 +66,7 @@ tenant_size_model.workspace = true utils.workspace = true workspace_hack.workspace = true reqwest.workspace = true +rpds.workspace = true [dev-dependencies] criterion.workspace = true diff --git a/pageserver/benches/bench_layer_map.rs b/pageserver/benches/bench_layer_map.rs index 6a01fdfc6f..e18c00da96 100644 --- a/pageserver/benches/bench_layer_map.rs +++ b/pageserver/benches/bench_layer_map.rs @@ -1,13 +1,12 @@ -use anyhow::Result; +use pageserver::keyspace::{KeyPartitioning, KeySpace}; use pageserver::repository::Key; use pageserver::tenant::layer_map::LayerMap; -use pageserver::tenant::storage_layer::{DeltaFileName, ImageFileName, ValueReconstructState}; -use pageserver::tenant::storage_layer::{Layer, ValueReconstructResult}; +use pageserver::tenant::storage_layer::Layer; +use pageserver::tenant::storage_layer::{DeltaFileName, ImageFileName, LayerDescriptor}; use rand::prelude::{SeedableRng, SliceRandom, StdRng}; use std::cmp::{max, min}; use std::fs::File; use std::io::{BufRead, BufReader}; -use std::ops::Range; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; @@ -17,102 +16,35 @@ use utils::lsn::Lsn; use criterion::{criterion_group, criterion_main, Criterion}; -struct DummyDelta { - key_range: Range, - lsn_range: Range, -} - -impl Layer for DummyDelta { - fn get_key_range(&self) -> Range { - self.key_range.clone() - } - - fn get_lsn_range(&self) -> Range { - self.lsn_range.clone() - } - fn get_value_reconstruct_data( - &self, - _key: Key, - _lsn_range: Range, - _reconstruct_data: &mut ValueReconstructState, - ) -> Result { - panic!() - } - - fn is_incremental(&self) -> bool { - true - } - - fn dump(&self, _verbose: bool) -> Result<()> { - unimplemented!() - } - - fn short_id(&self) -> String { - unimplemented!() - } -} - -struct DummyImage { - key_range: Range, - lsn: Lsn, -} - -impl Layer for DummyImage { - fn get_key_range(&self) -> Range { - self.key_range.clone() - } - - fn get_lsn_range(&self) -> Range { - // End-bound is exclusive - self.lsn..(self.lsn + 1) - } - - fn get_value_reconstruct_data( - &self, - _key: Key, - _lsn_range: Range, - _reconstruct_data: &mut ValueReconstructState, - ) -> Result { - panic!() - } - - fn is_incremental(&self) -> bool { - false - } - - fn dump(&self, _verbose: bool) -> Result<()> { - unimplemented!() - } - - fn short_id(&self) -> String { - unimplemented!() - } -} - -fn build_layer_map(filename_dump: PathBuf) -> LayerMap { - let mut layer_map = LayerMap::::default(); +fn build_layer_map(filename_dump: PathBuf) -> LayerMap { + let mut layer_map = LayerMap::::default(); let mut min_lsn = Lsn(u64::MAX); let mut max_lsn = Lsn(0); let filenames = BufReader::new(File::open(filename_dump).unwrap()).lines(); + let mut updates = layer_map.batch_update(); for fname in filenames { let fname = &fname.unwrap(); if let Some(imgfilename) = ImageFileName::parse_str(fname) { - let layer = DummyImage { - key_range: imgfilename.key_range, - lsn: imgfilename.lsn, + let layer = LayerDescriptor { + key: imgfilename.key_range, + lsn: imgfilename.lsn..(imgfilename.lsn + 1), + is_incremental: false, + short_id: fname.to_string(), }; - layer_map.insert_historic(Arc::new(layer)); + updates.insert_historic(Arc::new(layer)); min_lsn = min(min_lsn, imgfilename.lsn); max_lsn = max(max_lsn, imgfilename.lsn); } else if let Some(deltafilename) = DeltaFileName::parse_str(fname) { - let layer = DummyDelta { - key_range: deltafilename.key_range, - lsn_range: deltafilename.lsn_range.clone(), + let layer = LayerDescriptor { + key: deltafilename.key_range.clone(), + lsn: deltafilename.lsn_range.clone(), + is_incremental: true, + short_id: fname.to_string(), }; - layer_map.insert_historic(Arc::new(layer)); + updates.insert_historic(Arc::new(layer)); min_lsn = min(min_lsn, deltafilename.lsn_range.start); max_lsn = max(max_lsn, deltafilename.lsn_range.end); } else { @@ -122,11 +54,12 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap { println!("min: {min_lsn}, max: {max_lsn}"); + updates.flush(); layer_map } /// Construct a layer map query pattern for benchmarks -fn uniform_query_pattern(layer_map: &LayerMap) -> Vec<(Key, Lsn)> { +fn uniform_query_pattern(layer_map: &LayerMap) -> Vec<(Key, Lsn)> { // For each image layer we query one of the pages contained, at LSN right // before the image layer was created. This gives us a somewhat uniform // coverage of both the lsn and key space because image layers have @@ -150,6 +83,41 @@ fn uniform_query_pattern(layer_map: &LayerMap) -> Vec<(Key, Lsn)> { .collect() } +// Construct a partitioning for testing get_difficulty map when we +// don't have an exact result of `collect_keyspace` to work with. +fn uniform_key_partitioning(layer_map: &LayerMap, _lsn: Lsn) -> KeyPartitioning { + let mut parts = Vec::new(); + + // We add a partition boundary at the start of each image layer, + // no matter what lsn range it covers. This is just the easiest + // thing to do. A better thing to do would be to get a real + // partitioning from some database. Even better, remove the need + // for key partitions by deciding where to create image layers + // directly based on a coverage-based difficulty map. + let mut keys: Vec<_> = layer_map + .iter_historic_layers() + .filter_map(|l| { + if l.is_incremental() { + None + } else { + let kr = l.get_key_range(); + Some(kr.start.next()) + } + }) + .collect(); + keys.sort(); + + let mut current_key = Key::from_hex("000000000000000000000000000000000000").unwrap(); + for key in keys { + parts.push(KeySpace { + ranges: vec![current_key..key], + }); + current_key = key; + } + + KeyPartitioning { parts } +} + // Benchmark using metadata extracted from our performance test environment, from // a project where we have run pgbench many timmes. The pgbench database was initialized // between each test run. @@ -183,24 +151,68 @@ fn bench_from_captest_env(c: &mut Criterion) { // Benchmark using metadata extracted from a real project that was taknig // too long processing layer map queries. fn bench_from_real_project(c: &mut Criterion) { - // TODO consider compressing this file + // Init layer map + let now = Instant::now(); let layer_map = build_layer_map(PathBuf::from("benches/odd-brook-layernames.txt")); + println!("Finished layer map init in {:?}", now.elapsed()); + + // Choose uniformly distributed queries let queries: Vec<(Key, Lsn)> = uniform_query_pattern(&layer_map); - // Test with uniform query pattern - c.bench_function("real_map_uniform_queries", |b| { + // Choose inputs for get_difficulty_map + let latest_lsn = layer_map + .iter_historic_layers() + .map(|l| l.get_lsn_range().end) + .max() + .unwrap(); + let partitioning = uniform_key_partitioning(&layer_map, latest_lsn); + + // Check correctness of get_difficulty_map + // TODO put this in a dedicated test outside of this mod + { + println!("running correctness check"); + + let now = Instant::now(); + let result_bruteforce = layer_map.get_difficulty_map_bruteforce(latest_lsn, &partitioning); + assert!(result_bruteforce.len() == partitioning.parts.len()); + println!("Finished bruteforce in {:?}", now.elapsed()); + + let now = Instant::now(); + let result_fast = layer_map.get_difficulty_map(latest_lsn, &partitioning, None); + assert!(result_fast.len() == partitioning.parts.len()); + println!("Finished fast in {:?}", now.elapsed()); + + // Assert results are equal. Manually iterate for easier debugging. + let zip = std::iter::zip( + &partitioning.parts, + std::iter::zip(result_bruteforce, result_fast), + ); + for (_part, (bruteforce, fast)) in zip { + assert_eq!(bruteforce, fast); + } + + println!("No issues found"); + } + + // Define and name the benchmark function + let mut group = c.benchmark_group("real_map"); + group.bench_function("uniform_queries", |b| { b.iter(|| { for q in queries.clone().into_iter() { layer_map.search(q.0, q.1); } }); }); + group.bench_function("get_difficulty_map", |b| { + b.iter(|| { + layer_map.get_difficulty_map(latest_lsn, &partitioning, Some(3)); + }); + }); + group.finish(); } // Benchmark using synthetic data. Arrange image layers on stacked diagonal lines. fn bench_sequential(c: &mut Criterion) { - let mut layer_map: LayerMap = LayerMap::default(); - // Init layer map. Create 100_000 layers arranged in 1000 diagonal lines. // // TODO This code is pretty slow and runs even if we're only running other @@ -208,39 +220,39 @@ fn bench_sequential(c: &mut Criterion) { // Putting it inside the `bench_function` closure is not a solution // because then it runs multiple times during warmup. let now = Instant::now(); + let mut layer_map = LayerMap::default(); + let mut updates = layer_map.batch_update(); for i in 0..100_000 { - // TODO try inserting a super-wide layer in between every 10 to reflect - // what often happens with L1 layers that include non-rel changes. - // Maybe do that as a separate test. let i32 = (i as u32) % 100; let zero = Key::from_hex("000000000000000000000000000000000000").unwrap(); - let layer = DummyImage { - key_range: zero.add(10 * i32)..zero.add(10 * i32 + 1), - lsn: Lsn(10 * i), + let layer = LayerDescriptor { + key: zero.add(10 * i32)..zero.add(10 * i32 + 1), + lsn: Lsn(i)..Lsn(i + 1), + is_incremental: false, + short_id: format!("Layer {}", i), }; - layer_map.insert_historic(Arc::new(layer)); + updates.insert_historic(Arc::new(layer)); } - - // Manually measure runtime without criterion because criterion - // has a minimum sample size of 10 and I don't want to run it 10 times. - println!("Finished init in {:?}", now.elapsed()); + updates.flush(); + println!("Finished layer map init in {:?}", now.elapsed()); // Choose 100 uniformly random queries let rng = &mut StdRng::seed_from_u64(1); let queries: Vec<(Key, Lsn)> = uniform_query_pattern(&layer_map) - .choose_multiple(rng, 1) + .choose_multiple(rng, 100) .copied() .collect(); // Define and name the benchmark function - c.bench_function("sequential_uniform_queries", |b| { - // Run the search queries + let mut group = c.benchmark_group("sequential"); + group.bench_function("uniform_queries", |b| { b.iter(|| { for q in queries.clone().into_iter() { layer_map.search(q.0, q.1); } }); }); + group.finish(); } criterion_group!(group_1, bench_from_captest_env); diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 586fd20886..092503b7c5 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -37,6 +37,17 @@ impl Key { | self.field6 as i128 } + pub fn from_i128(x: i128) -> Self { + Key { + field1: ((x >> 120) & 0xf) as u8, + field2: ((x >> 104) & 0xFFFF) as u32, + field3: (x >> 72) as u32, + field4: (x >> 40) as u32, + field5: (x >> 32) as u8, + field6: x as u32, + } + } + pub fn next(&self) -> Key { self.add(1) } diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index 01c5359e88..ed1a32c8fd 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -9,24 +9,57 @@ //! are frozen, and it is split up into new image and delta layers and the //! corresponding files are written to disk. //! +//! Design overview: +//! +//! The `search` method of the layer map is on the read critical path, so we've +//! built an efficient data structure for fast reads, stored in `LayerMap::historic`. +//! Other read methods are less critical but still impact performance of background tasks. +//! +//! This data structure relies on a persistent/immutable binary search tree. See the +//! following lecture for an introduction https://www.youtube.com/watch?v=WqCWghETNDc&t=581s +//! Summary: A persistent/immutable BST (and persistent data structures in general) allows +//! you to modify the tree in such a way that each modification creates a new "version" +//! of the tree. When you modify it, you get a new version, but all previous versions are +//! still accessible too. So if someone is still holding a reference to an older version, +//! they continue to see the tree as it was then. The persistent BST stores all the +//! different versions in an efficient way. +//! +//! Our persistent BST maintains a map of which layer file "covers" each key. It has only +//! one dimension, the key. See `layer_coverage.rs`. We use the persistent/immutable property +//! to handle the LSN dimension. +//! +//! To build the layer map, we insert each layer to the persistent BST in LSN.start order, +//! starting from the oldest one. After each insertion, we grab a reference to that "version" +//! of the tree, and store it in another tree, a BtreeMap keyed by the LSN. See +//! `historic_layer_coverage.rs`. +//! +//! To search for a particular key-LSN pair, you first look up the right "version" in the +//! BTreeMap. Then you search that version of the BST with the key. +//! +//! The persistent BST keeps all the versions, but there is no way to change the old versions +//! afterwards. We can add layers as long as they have larger LSNs than any previous layer in +//! the map, but if we need to remove a layer, or insert anything with an older LSN, we need +//! to throw away most of the persistent BST and build a new one, starting from the oldest +//! LSN. See `LayerMap::flush_updates()`. +//! +mod historic_layer_coverage; +mod layer_coverage; + +use crate::keyspace::KeyPartitioning; use crate::metrics::NUM_ONDISK_LAYERS; use crate::repository::Key; -use crate::tenant::storage_layer::{range_eq, range_overlaps}; -use amplify_num::i256; +use crate::tenant::storage_layer::InMemoryLayer; +use crate::tenant::storage_layer::Layer; use anyhow::Result; -use num_traits::identities::{One, Zero}; -use num_traits::{Bounded, Num, Signed}; -use rstar::{RTree, RTreeObject, AABB}; -use std::cmp::Ordering; use std::collections::VecDeque; use std::ops::Range; -use std::ops::{Add, Div, Mul, Neg, Rem, Sub}; use std::sync::Arc; -use tracing::*; use utils::lsn::Lsn; -use super::storage_layer::{InMemoryLayer, Layer}; +use historic_layer_coverage::BufferedHistoricLayerCoverage; + +use super::storage_layer::range_eq; /// /// LayerMap tracks what layers exist on a timeline. @@ -51,8 +84,8 @@ pub struct LayerMap { /// pub frozen_layers: VecDeque>, - /// All the historic layers are kept here - historic_layers: RTree>, + /// Index of the historic layers optimized for search + historic: BufferedHistoricLayerCoverage>, /// L0 layers have key range Key::MIN..Key::MAX, and locating them using R-Tree search is very inefficient. /// So L0 layers are held in l0_delta_layers vector, in addition to the R-tree. @@ -65,177 +98,64 @@ impl Default for LayerMap { open_layer: None, next_open_layer_at: None, frozen_layers: VecDeque::default(), - historic_layers: RTree::default(), l0_delta_layers: Vec::default(), + historic: BufferedHistoricLayerCoverage::default(), } } } -struct LayerRTreeObject { - layer: Arc, - - envelope: AABB<[IntKey; 2]>, +/// The primary update API for the layer map. +/// +/// Batching historic layer insertions and removals is good for +/// performance and this struct helps us do that correctly. +#[must_use] +pub struct BatchedUpdates<'a, L: ?Sized + Layer> { + // While we hold this exclusive reference to the layer map the type checker + // will prevent us from accidentally reading any unflushed updates. + layer_map: &'a mut LayerMap, } -// Representation of Key as numeric type. -// We can not use native implementation of i128, because rstar::RTree -// doesn't handle properly integer overflow during area calculation: sum(Xi*Yi). -// Overflow will cause panic in debug mode and incorrect area calculation in release mode, -// which leads to non-optimally balanced R-Tree (but doesn't fit correctness of R-Tree work). -// By using i256 as the type, even though all the actual values would fit in i128, we can be -// sure that multiplication doesn't overflow. -// - -#[derive(Clone, PartialEq, Eq, PartialOrd, Debug)] -struct IntKey(i256); - -impl Copy for IntKey {} - -impl IntKey { - fn from(i: i128) -> Self { - IntKey(i256::from(i)) - } -} - -impl Bounded for IntKey { - fn min_value() -> Self { - IntKey(i256::MIN) - } - fn max_value() -> Self { - IntKey(i256::MAX) - } -} - -impl Signed for IntKey { - fn is_positive(&self) -> bool { - self.0 > i256::ZERO - } - fn is_negative(&self) -> bool { - self.0 < i256::ZERO - } - fn signum(&self) -> Self { - match self.0.cmp(&i256::ZERO) { - Ordering::Greater => IntKey(i256::ONE), - Ordering::Less => IntKey(-i256::ONE), - Ordering::Equal => IntKey(i256::ZERO), - } - } - fn abs(&self) -> Self { - IntKey(self.0.abs()) - } - fn abs_sub(&self, other: &Self) -> Self { - if self.0 <= other.0 { - IntKey(i256::ZERO) - } else { - IntKey(self.0 - other.0) - } - } -} - -impl Neg for IntKey { - type Output = Self; - fn neg(self) -> Self::Output { - IntKey(-self.0) - } -} - -impl Rem for IntKey { - type Output = Self; - fn rem(self, rhs: Self) -> Self::Output { - IntKey(self.0 % rhs.0) - } -} - -impl Div for IntKey { - type Output = Self; - fn div(self, rhs: Self) -> Self::Output { - IntKey(self.0 / rhs.0) - } -} - -impl Add for IntKey { - type Output = Self; - fn add(self, rhs: Self) -> Self::Output { - IntKey(self.0 + rhs.0) - } -} - -impl Sub for IntKey { - type Output = Self; - fn sub(self, rhs: Self) -> Self::Output { - IntKey(self.0 - rhs.0) - } -} - -impl Mul for IntKey { - type Output = Self; - fn mul(self, rhs: Self) -> Self::Output { - IntKey(self.0 * rhs.0) - } -} - -impl One for IntKey { - fn one() -> Self { - IntKey(i256::ONE) - } -} - -impl Zero for IntKey { - fn zero() -> Self { - IntKey(i256::ZERO) - } - fn is_zero(&self) -> bool { - self.0 == i256::ZERO - } -} - -impl Num for IntKey { - type FromStrRadixErr = ::FromStrRadixErr; - fn from_str_radix(str: &str, radix: u32) -> Result { - Ok(IntKey(i256::from(i128::from_str_radix(str, radix)?))) - } -} - -impl PartialEq for LayerRTreeObject { - fn eq(&self, other: &Self) -> bool { - // FIXME: ptr_eq might fail to return true for 'dyn' - // references. Clippy complains about this. In practice it - // seems to work, the assertion below would be triggered - // otherwise but this ought to be fixed. - #[allow(clippy::vtable_address_comparisons)] - Arc::ptr_eq(&self.layer, &other.layer) - } -} - -impl RTreeObject for LayerRTreeObject -where - L: ?Sized, -{ - type Envelope = AABB<[IntKey; 2]>; - fn envelope(&self) -> Self::Envelope { - self.envelope - } -} - -impl LayerRTreeObject +/// Provide ability to batch more updates while hiding the read +/// API so we don't accidentally read without flushing. +impl BatchedUpdates<'_, L> where L: ?Sized + Layer, { - fn new(layer: Arc) -> Self { - let key_range = layer.get_key_range(); - let lsn_range = layer.get_lsn_range(); + /// + /// Insert an on-disk layer. + /// + pub fn insert_historic(&mut self, layer: Arc) { + self.layer_map.insert_historic_noflush(layer) + } - let envelope = AABB::from_corners( - [ - IntKey::from(key_range.start.to_i128()), - IntKey::from(lsn_range.start.0 as i128), - ], - [ - IntKey::from(key_range.end.to_i128() - 1), - IntKey::from(lsn_range.end.0 as i128 - 1), - ], // AABB::upper is inclusive, while `key_range.end` and `lsn_range.end` are exclusive - ); - LayerRTreeObject { layer, envelope } + /// + /// Remove an on-disk layer from the map. + /// + /// This should be called when the corresponding file on disk has been deleted. + /// + pub fn remove_historic(&mut self, layer: Arc) { + self.layer_map.remove_historic_noflush(layer) + } + + // We will flush on drop anyway, but this method makes it + // more explicit that there is some work being done. + /// Apply all updates + pub fn flush(self) { + // Flush happens on drop + } +} + +// Ideally the flush() method should be called explicitly for more +// controlled execution. But if we forget we'd rather flush on drop +// than panic later or read without flushing. +// +// TODO maybe warn if flush hasn't explicitly been called +impl Drop for BatchedUpdates<'_, L> +where + L: ?Sized + Layer, +{ + fn drop(&mut self) { + self.layer_map.flush_updates(); } } @@ -281,125 +201,91 @@ where /// 'open' and 'frozen' layers! /// pub fn search(&self, key: Key, end_lsn: Lsn) -> Option> { - // Find the latest image layer that covers the given key - let mut latest_img: Option> = None; - let mut latest_img_lsn: Option = None; - let envelope = AABB::from_corners( - [IntKey::from(key.to_i128()), IntKey::from(0i128)], - [ - IntKey::from(key.to_i128()), - IntKey::from(end_lsn.0 as i128 - 1), - ], - ); - for e in self - .historic_layers - .locate_in_envelope_intersecting(&envelope) - { - let l = &e.layer; - if l.is_incremental() { - continue; - } - assert!(l.get_key_range().contains(&key)); - let img_lsn = l.get_lsn_range().start; - assert!(img_lsn < end_lsn); - if Lsn(img_lsn.0 + 1) == end_lsn { - // found exact match - return Some(SearchResult { - layer: Arc::clone(l), - lsn_floor: img_lsn, - }); - } - if img_lsn > latest_img_lsn.unwrap_or(Lsn(0)) { - latest_img = Some(Arc::clone(l)); - latest_img_lsn = Some(img_lsn); - } - } + let version = self.historic.get().unwrap().get_version(end_lsn.0 - 1)?; + let latest_delta = version.delta_coverage.query(key.to_i128()); + let latest_image = version.image_coverage.query(key.to_i128()); - // Search the delta layers - let mut latest_delta: Option> = None; - for e in self - .historic_layers - .locate_in_envelope_intersecting(&envelope) - { - let l = &e.layer; - if !l.is_incremental() { - continue; + match (latest_delta, latest_image) { + (None, None) => None, + (None, Some(image)) => { + let lsn_floor = image.get_lsn_range().start; + Some(SearchResult { + layer: image, + lsn_floor, + }) } - assert!(l.get_key_range().contains(&key)); - if l.get_lsn_range().start >= end_lsn { - info!( - "Candidate delta layer {}..{} is too new for lsn {}", - l.get_lsn_range().start, - l.get_lsn_range().end, - end_lsn - ); + (Some(delta), None) => { + let lsn_floor = delta.get_lsn_range().start; + Some(SearchResult { + layer: delta, + lsn_floor, + }) } - assert!(l.get_lsn_range().start < end_lsn); - if l.get_lsn_range().end >= end_lsn { - // this layer contains the requested point in the key/lsn space. - // No need to search any further - trace!( - "found layer {} for request on {key} at {end_lsn}", - l.short_id(), - ); - latest_delta.replace(Arc::clone(l)); - break; - } - if l.get_lsn_range().end > latest_img_lsn.unwrap_or(Lsn(0)) { - // this layer's end LSN is smaller than the requested point. If there's - // nothing newer, this is what we need to return. Remember this. - if let Some(old_candidate) = &latest_delta { - if l.get_lsn_range().end > old_candidate.get_lsn_range().end { - latest_delta.replace(Arc::clone(l)); - } + (Some(delta), Some(image)) => { + let img_lsn = image.get_lsn_range().start; + let image_is_newer = image.get_lsn_range().end >= delta.get_lsn_range().end; + let image_exact_match = img_lsn + 1 == end_lsn; + if image_is_newer || image_exact_match { + Some(SearchResult { + layer: image, + lsn_floor: img_lsn, + }) } else { - latest_delta.replace(Arc::clone(l)); + let lsn_floor = + std::cmp::max(delta.get_lsn_range().start, image.get_lsn_range().start + 1); + Some(SearchResult { + layer: delta, + lsn_floor, + }) } } } - if let Some(l) = latest_delta { - trace!( - "found (old) layer {} for request on {key} at {end_lsn}", - l.short_id(), - ); - let lsn_floor = std::cmp::max( - Lsn(latest_img_lsn.unwrap_or(Lsn(0)).0 + 1), - l.get_lsn_range().start, - ); - Some(SearchResult { - lsn_floor, - layer: l, - }) - } else if let Some(l) = latest_img { - trace!("found img layer and no deltas for request on {key} at {end_lsn}"); - Some(SearchResult { - lsn_floor: latest_img_lsn.unwrap(), - layer: l, - }) - } else { - trace!("no layer found for request on {key} at {end_lsn}"); - None - } + } + + /// Start a batch of updates, applied on drop + pub fn batch_update(&mut self) -> BatchedUpdates<'_, L> { + BatchedUpdates { layer_map: self } } /// /// Insert an on-disk layer /// - pub fn insert_historic(&mut self, layer: Arc) { - if layer.get_key_range() == (Key::MIN..Key::MAX) { - self.l0_delta_layers.push(layer.clone()); + /// Helper function for BatchedUpdates::insert_historic + /// + pub(self) fn insert_historic_noflush(&mut self, layer: Arc) { + let kr = layer.get_key_range(); + let lr = layer.get_lsn_range(); + self.historic.insert( + historic_layer_coverage::LayerKey { + key: kr.start.to_i128()..kr.end.to_i128(), + lsn: lr.start.0..lr.end.0, + is_image: !layer.is_incremental(), + }, + Arc::clone(&layer), + ); + + if Self::is_l0(&layer) { + self.l0_delta_layers.push(layer); } - self.historic_layers.insert(LayerRTreeObject::new(layer)); + NUM_ONDISK_LAYERS.inc(); } /// /// Remove an on-disk layer from the map. /// - /// This should be called when the corresponding file on disk has been deleted. + /// Helper function for BatchedUpdates::remove_historic /// - pub fn remove_historic(&mut self, layer: Arc) { - if layer.get_key_range() == (Key::MIN..Key::MAX) { + pub fn remove_historic_noflush(&mut self, layer: Arc) { + let kr = layer.get_key_range(); + let lr = layer.get_lsn_range(); + self.historic.remove(historic_layer_coverage::LayerKey { + key: kr.start.to_i128()..kr.end.to_i128(), + lsn: lr.start.0..lr.end.0, + is_image: !layer.is_incremental(), + }); + + if Self::is_l0(&layer) { let len_before = self.l0_delta_layers.len(); // FIXME: ptr_eq might fail to return true for 'dyn' @@ -411,98 +297,57 @@ where .retain(|other| !Arc::ptr_eq(other, &layer)); assert_eq!(self.l0_delta_layers.len(), len_before - 1); } - assert!(self - .historic_layers - .remove(&LayerRTreeObject::new(layer)) - .is_some()); + NUM_ONDISK_LAYERS.dec(); } + /// Helper function for BatchedUpdates::drop. + pub(self) fn flush_updates(&mut self) { + self.historic.rebuild(); + } + /// Is there a newer image layer for given key- and LSN-range? Or a set /// of image layers within the specified lsn range that cover the entire /// specified key range? /// /// This is used for garbage collection, to determine if an old layer can /// be deleted. - pub fn image_layer_exists( - &self, - key_range: &Range, - lsn_range: &Range, - ) -> Result { - let mut range_remain = key_range.clone(); + pub fn image_layer_exists(&self, key: &Range, lsn: &Range) -> Result { + if key.is_empty() { + // Vacuously true. There's a newer image for all 0 of the kerys in the range. + return Ok(true); + } - loop { - let mut made_progress = false; - let envelope = AABB::from_corners( - [ - IntKey::from(range_remain.start.to_i128()), - IntKey::from(lsn_range.start.0 as i128), - ], - [ - IntKey::from(range_remain.end.to_i128() - 1), - IntKey::from(lsn_range.end.0 as i128 - 1), - ], - ); - for e in self - .historic_layers - .locate_in_envelope_intersecting(&envelope) - { - let l = &e.layer; - if l.is_incremental() { - continue; - } - let img_lsn = l.get_lsn_range().start; - if l.get_key_range().contains(&range_remain.start) && lsn_range.contains(&img_lsn) { - made_progress = true; - let img_key_end = l.get_key_range().end; + let version = match self.historic.get().unwrap().get_version(lsn.end.0 - 1) { + Some(v) => v, + None => return Ok(false), + }; - if img_key_end >= range_remain.end { - return Ok(true); - } - range_remain.start = img_key_end; - } - } + let start = key.start.to_i128(); + let end = key.end.to_i128(); - if !made_progress { + let layer_covers = |layer: Option>| match layer { + Some(layer) => layer.get_lsn_range().start >= lsn.start, + None => false, + }; + + // Check the start is covered + if !layer_covers(version.image_coverage.query(start)) { + return Ok(false); + } + + // Check after all changes of coverage + for (_, change_val) in version.image_coverage.range(start..end) { + if !layer_covers(change_val) { return Ok(false); } } + + Ok(true) } pub fn iter_historic_layers(&self) -> impl '_ + Iterator> { - self.historic_layers.iter().map(|e| e.layer.clone()) - } - - /// Find the last image layer that covers 'key', ignoring any image layers - /// newer than 'lsn'. - fn find_latest_image(&self, key: Key, lsn: Lsn) -> Option> { - let mut candidate_lsn = Lsn(0); - let mut candidate = None; - let envelope = AABB::from_corners( - [IntKey::from(key.to_i128()), IntKey::from(0)], - [IntKey::from(key.to_i128()), IntKey::from(lsn.0 as i128)], - ); - for e in self - .historic_layers - .locate_in_envelope_intersecting(&envelope) - { - let l = &e.layer; - if l.is_incremental() { - continue; - } - - assert!(l.get_key_range().contains(&key)); - let this_lsn = l.get_lsn_range().start; - assert!(this_lsn <= lsn); - if this_lsn < candidate_lsn { - // our previous candidate was better - continue; - } - candidate_lsn = this_lsn; - candidate = Some(Arc::clone(l)); - } - - candidate + self.historic.iter() } /// @@ -518,94 +363,288 @@ where key_range: &Range, lsn: Lsn, ) -> Result, Option>)>> { - let mut points = vec![key_range.start]; - let envelope = AABB::from_corners( - [IntKey::from(key_range.start.to_i128()), IntKey::from(0)], - [ - IntKey::from(key_range.end.to_i128()), - IntKey::from(lsn.0 as i128), - ], - ); - for e in self - .historic_layers - .locate_in_envelope_intersecting(&envelope) - { - let l = &e.layer; - assert!(l.get_lsn_range().start <= lsn); - let range = l.get_key_range(); - if key_range.contains(&range.start) { - points.push(l.get_key_range().start); - } - if key_range.contains(&range.end) { - points.push(l.get_key_range().end); - } + let version = match self.historic.get().unwrap().get_version(lsn.0) { + Some(v) => v, + None => return Ok(vec![]), + }; + + let start = key_range.start.to_i128(); + let end = key_range.end.to_i128(); + + // Initialize loop variables + let mut coverage: Vec<(Range, Option>)> = vec![]; + let mut current_key = start; + let mut current_val = version.image_coverage.query(start); + + // Loop through the change events and push intervals + for (change_key, change_val) in version.image_coverage.range(start..end) { + let kr = Key::from_i128(current_key)..Key::from_i128(change_key); + coverage.push((kr, current_val.take())); + current_key = change_key; + current_val = change_val.clone(); } - points.push(key_range.end); - points.sort(); - points.dedup(); + // Add the final interval + let kr = Key::from_i128(current_key)..Key::from_i128(end); + coverage.push((kr, current_val.take())); - // Ok, we now have a list of "interesting" points in the key space - - // For each range between the points, find the latest image - let mut start = *points.first().unwrap(); - let mut ranges = Vec::new(); - for end in points[1..].iter() { - let img = self.find_latest_image(start, lsn); - - ranges.push((start..*end, img)); - - start = *end; - } - Ok(ranges) + Ok(coverage) } - /// Count the height of the tallest stack of deltas in this 2d region. + pub fn is_l0(layer: &L) -> bool { + range_eq(&layer.get_key_range(), &(Key::MIN..Key::MAX)) + } + + /// This function determines which layers are counted in `count_deltas`: + /// layers that should count towards deciding whether or not to reimage + /// a certain partition range. + /// + /// There are two kinds of layers we currently consider reimage-worthy: + /// + /// Case 1: Non-L0 layers are currently reimage-worthy by default. + /// TODO Some of these layers are very sparse and cover the entire key + /// range. Replacing 256MB of data (or less!) with terabytes of + /// images doesn't seem wise. We need a better heuristic, possibly + /// based on some of these factors: + /// a) whether this layer has any wal in this partition range + /// b) the size of the layer + /// c) the number of images needed to cover it + /// d) the estimated time until we'll have to reimage over it for GC + /// + /// Case 2: Since L0 layers by definition cover the entire key space, we consider + /// them reimage-worthy only when the entire key space can be covered by very few + /// images (currently 1). + /// TODO The optimal number should probably be slightly higher than 1, but to + /// implement that we need to plumb a lot more context into this function + /// than just the current partition_range. + pub fn is_reimage_worthy(layer: &L, partition_range: &Range) -> bool { + // Case 1 + if !Self::is_l0(layer) { + return true; + } + + // Case 2 + if range_eq(partition_range, &(Key::MIN..Key::MAX)) { + return true; + } + + false + } + + /// Count the height of the tallest stack of reimage-worthy deltas + /// in this 2d region. + /// + /// If `limit` is provided we don't try to count above that number. /// /// This number is used to compute the largest number of deltas that /// we'll need to visit for any page reconstruction in this region. /// We use this heuristic to decide whether to create an image layer. - /// - /// TODO currently we just return the total number of deltas in the - /// region, no matter if they're stacked on top of each other - /// or next to each other. - pub fn count_deltas(&self, key_range: &Range, lsn_range: &Range) -> Result { - let mut result = 0; - if lsn_range.start >= lsn_range.end { + pub fn count_deltas( + &self, + key: &Range, + lsn: &Range, + limit: Option, + ) -> Result { + // We get the delta coverage of the region, and for each part of the coverage + // we recurse right underneath the delta. The recursion depth is limited by + // the largest result this function could return, which is in practice between + // 3 and 10 (since we usually try to create an image when the number gets larger). + + if lsn.is_empty() || key.is_empty() || limit == Some(0) { return Ok(0); } - let envelope = AABB::from_corners( - [ - IntKey::from(key_range.start.to_i128()), - IntKey::from(lsn_range.start.0 as i128), - ], - [ - IntKey::from(key_range.end.to_i128() - 1), - IntKey::from(lsn_range.end.0 as i128 - 1), - ], - ); - for e in self - .historic_layers - .locate_in_envelope_intersecting(&envelope) - { - let l = &e.layer; - if !l.is_incremental() { - continue; - } - assert!(range_overlaps(&l.get_lsn_range(), lsn_range)); - assert!(range_overlaps(&l.get_key_range(), key_range)); - // We ignore level0 delta layers. Unless the whole keyspace fits - // into one partition - if !range_eq(key_range, &(Key::MIN..Key::MAX)) - && range_eq(&l.get_key_range(), &(Key::MIN..Key::MAX)) - { - continue; + let version = match self.historic.get().unwrap().get_version(lsn.end.0 - 1) { + Some(v) => v, + None => return Ok(0), + }; + + let start = key.start.to_i128(); + let end = key.end.to_i128(); + + // Initialize loop variables + let mut max_stacked_deltas = 0; + let mut current_key = start; + let mut current_val = version.delta_coverage.query(start); + + // Loop through the delta coverage and recurse on each part + for (change_key, change_val) in version.delta_coverage.range(start..end) { + // If there's a relevant delta in this part, add 1 and recurse down + if let Some(val) = current_val { + if val.get_lsn_range().end > lsn.start { + let kr = Key::from_i128(current_key)..Key::from_i128(change_key); + let lr = lsn.start..val.get_lsn_range().start; + if !kr.is_empty() { + let base_count = Self::is_reimage_worthy(&val, key) as usize; + let new_limit = limit.map(|l| l - base_count); + let max_stacked_deltas_underneath = + self.count_deltas(&kr, &lr, new_limit)?; + max_stacked_deltas = std::cmp::max( + max_stacked_deltas, + base_count + max_stacked_deltas_underneath, + ); + } + } } - result += 1; + current_key = change_key; + current_val = change_val.clone(); } - Ok(result) + + // Consider the last part + if let Some(val) = current_val { + if val.get_lsn_range().end > lsn.start { + let kr = Key::from_i128(current_key)..Key::from_i128(end); + let lr = lsn.start..val.get_lsn_range().start; + + if !kr.is_empty() { + let base_count = Self::is_reimage_worthy(&val, key) as usize; + let new_limit = limit.map(|l| l - base_count); + let max_stacked_deltas_underneath = self.count_deltas(&kr, &lr, new_limit)?; + max_stacked_deltas = std::cmp::max( + max_stacked_deltas, + base_count + max_stacked_deltas_underneath, + ); + } + } + } + + Ok(max_stacked_deltas) + } + + /// Count how many reimage-worthy layers we need to visit for given key-lsn pair. + /// + /// The `partition_range` argument is used as context for the reimage-worthiness decision. + /// + /// Used as a helper for correctness checks only. Performance not critical. + pub fn get_difficulty(&self, lsn: Lsn, key: Key, partition_range: &Range) -> usize { + match self.search(key, lsn) { + Some(search_result) => { + if search_result.layer.is_incremental() { + (Self::is_reimage_worthy(&search_result.layer, partition_range) as usize) + + self.get_difficulty(search_result.lsn_floor, key, partition_range) + } else { + 0 + } + } + None => 0, + } + } + + /// Used for correctness checking. Results are expected to be identical to + /// self.get_difficulty_map. Assumes self.search is correct. + pub fn get_difficulty_map_bruteforce( + &self, + lsn: Lsn, + partitioning: &KeyPartitioning, + ) -> Vec { + // Looking at the difficulty as a function of key, it could only increase + // when a delta layer starts or an image layer ends. Therefore it's sufficient + // to check the difficulties at: + // - the key.start for each non-empty part range + // - the key.start for each delta + // - the key.end for each image + let keys_iter: Box> = { + let mut keys: Vec = self + .iter_historic_layers() + .map(|layer| { + if layer.is_incremental() { + layer.get_key_range().start + } else { + layer.get_key_range().end + } + }) + .collect(); + keys.sort(); + Box::new(keys.into_iter()) + }; + let mut keys_iter = keys_iter.peekable(); + + // Iter the partition and keys together and query all the necessary + // keys, computing the max difficulty for each part. + partitioning + .parts + .iter() + .map(|part| { + let mut difficulty = 0; + // Partition ranges are assumed to be sorted and disjoint + // TODO assert it + for range in &part.ranges { + if !range.is_empty() { + difficulty = + std::cmp::max(difficulty, self.get_difficulty(lsn, range.start, range)); + } + while let Some(key) = keys_iter.peek() { + if key >= &range.end { + break; + } + let key = keys_iter.next().unwrap(); + if key < range.start { + continue; + } + difficulty = + std::cmp::max(difficulty, self.get_difficulty(lsn, key, range)); + } + } + difficulty + }) + .collect() + } + + /// For each part of a keyspace partitioning, return the maximum number of layers + /// that would be needed for page reconstruction in that part at the given LSN. + /// + /// If `limit` is provided we don't try to count above that number. + /// + /// This method is used to decide where to create new image layers. Computing the + /// result for the entire partitioning at once allows this function to be more + /// efficient, and further optimization is possible by using iterators instead, + /// to allow early return. + /// + /// TODO actually use this method instead of count_deltas. Currently we only use + /// it for benchmarks. + pub fn get_difficulty_map( + &self, + lsn: Lsn, + partitioning: &KeyPartitioning, + limit: Option, + ) -> Vec { + // TODO This is a naive implementation. Perf improvements to do: + // 1. Instead of calling self.image_coverage and self.count_deltas, + // iterate the image and delta coverage only once. + partitioning + .parts + .iter() + .map(|part| { + let mut difficulty = 0; + for range in &part.ranges { + if limit == Some(difficulty) { + break; + } + for (img_range, last_img) in self + .image_coverage(range, lsn) + .expect("why would this err?") + { + if limit == Some(difficulty) { + break; + } + let img_lsn = if let Some(last_img) = last_img { + last_img.get_lsn_range().end + } else { + Lsn(0) + }; + + if img_lsn < lsn { + let num_deltas = self + .count_deltas(&img_range, &(img_lsn..lsn), limit) + .expect("why would this err lol?"); + difficulty = std::cmp::max(difficulty, num_deltas); + } + } + } + difficulty + }) + .collect() } /// Return all L0 delta layers @@ -629,8 +668,8 @@ where } println!("historic_layers:"); - for e in self.historic_layers.iter() { - e.layer.dump(verbose)?; + for layer in self.iter_historic_layers() { + layer.dump(verbose)?; } println!("End dump LayerMap"); Ok(()) diff --git a/pageserver/src/tenant/layer_map/historic_layer_coverage.rs b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs new file mode 100644 index 0000000000..46821aef15 --- /dev/null +++ b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs @@ -0,0 +1,583 @@ +use std::collections::BTreeMap; +use std::ops::Range; + +use tracing::info; + +use super::layer_coverage::LayerCoverageTuple; + +/// Layers in this module are identified and indexed by this data. +/// +/// This is a helper struct to enable sorting layers by lsn.start. +/// +/// These three values are enough to uniquely identify a layer, since +/// a layer is obligated to contain all contents within range, so two +/// deltas (or images) with the same range have identical content. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct LayerKey { + // TODO I use i128 and u64 because it was easy for prototyping, + // testing, and benchmarking. If we can use the Lsn and Key + // types without overhead that would be preferable. + pub key: Range, + pub lsn: Range, + pub is_image: bool, +} + +impl PartialOrd for LayerKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for LayerKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // NOTE we really care about comparing by lsn.start first + self.lsn + .start + .cmp(&other.lsn.start) + .then(self.lsn.end.cmp(&other.lsn.end)) + .then(self.key.start.cmp(&other.key.start)) + .then(self.key.end.cmp(&other.key.end)) + .then(self.is_image.cmp(&other.is_image)) + } +} + +/// Efficiently queryable layer coverage for each LSN. +/// +/// Allows answering layer map queries very efficiently, +/// but doesn't allow retroactive insertion, which is +/// sometimes necessary. See BufferedHistoricLayerCoverage. +pub struct HistoricLayerCoverage { + /// The latest state + head: LayerCoverageTuple, + + /// All previous states + historic: BTreeMap>, +} + +impl Default for HistoricLayerCoverage { + fn default() -> Self { + Self::new() + } +} + +impl HistoricLayerCoverage { + pub fn new() -> Self { + Self { + head: LayerCoverageTuple::default(), + historic: BTreeMap::default(), + } + } + + /// Add a layer + /// + /// Panics if new layer has older lsn.start than an existing layer. + /// See BufferedHistoricLayerCoverage for a more general insertion method. + pub fn insert(&mut self, layer_key: LayerKey, value: Value) { + // It's only a persistent map, not a retroactive one + if let Some(last_entry) = self.historic.iter().next_back() { + let last_lsn = last_entry.0; + if layer_key.lsn.start < *last_lsn { + panic!("unexpected retroactive insert"); + } + } + + // Insert into data structure + if layer_key.is_image { + self.head + .image_coverage + .insert(layer_key.key, layer_key.lsn.clone(), value); + } else { + self.head + .delta_coverage + .insert(layer_key.key, layer_key.lsn.clone(), value); + } + + // Remember history. Clone is O(1) + self.historic.insert(layer_key.lsn.start, self.head.clone()); + } + + /// Query at a particular LSN, inclusive + pub fn get_version(&self, lsn: u64) -> Option<&LayerCoverageTuple> { + match self.historic.range(..=lsn).next_back() { + Some((_, v)) => Some(v), + None => None, + } + } + + /// Remove all entries after a certain LSN (inclusive) + pub fn trim(&mut self, begin: &u64) { + self.historic.split_off(begin); + self.head = self + .historic + .iter() + .rev() + .next() + .map(|(_, v)| v.clone()) + .unwrap_or_default(); + } +} + +/// This is the most basic test that demonstrates intended usage. +/// All layers in this test have height 1. +#[test] +fn test_persistent_simple() { + let mut map = HistoricLayerCoverage::::new(); + map.insert( + LayerKey { + key: 0..5, + lsn: 100..101, + is_image: true, + }, + "Layer 1".to_string(), + ); + map.insert( + LayerKey { + key: 3..9, + lsn: 110..111, + is_image: true, + }, + "Layer 2".to_string(), + ); + map.insert( + LayerKey { + key: 5..6, + lsn: 120..121, + is_image: true, + }, + "Layer 3".to_string(), + ); + + // After Layer 1 insertion + let version = map.get_version(105).unwrap(); + assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string())); + assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string())); + + // After Layer 2 insertion + let version = map.get_version(115).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string())); + assert_eq!(version.image_coverage.query(8), Some("Layer 2".to_string())); + assert_eq!(version.image_coverage.query(11), None); + + // After Layer 3 insertion + let version = map.get_version(125).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string())); + assert_eq!(version.image_coverage.query(5), Some("Layer 3".to_string())); + assert_eq!(version.image_coverage.query(7), Some("Layer 2".to_string())); +} + +/// Cover simple off-by-one edge cases +#[test] +fn test_off_by_one() { + let mut map = HistoricLayerCoverage::::new(); + map.insert( + LayerKey { + key: 3..5, + lsn: 100..110, + is_image: true, + }, + "Layer 1".to_string(), + ); + + // Check different LSNs + let version = map.get_version(99); + assert!(version.is_none()); + let version = map.get_version(100).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string())); + let version = map.get_version(110).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string())); + + // Check different keys + let version = map.get_version(105).unwrap(); + assert_eq!(version.image_coverage.query(2), None); + assert_eq!(version.image_coverage.query(3), Some("Layer 1".to_string())); + assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string())); + assert_eq!(version.image_coverage.query(5), None); +} + +/// Cover edge cases where layers begin or end on the same key +#[test] +fn test_key_collision() { + let mut map = HistoricLayerCoverage::::new(); + + map.insert( + LayerKey { + key: 3..5, + lsn: 100..110, + is_image: true, + }, + "Layer 10".to_string(), + ); + map.insert( + LayerKey { + key: 5..8, + lsn: 100..110, + is_image: true, + }, + "Layer 11".to_string(), + ); + map.insert( + LayerKey { + key: 3..4, + lsn: 200..210, + is_image: true, + }, + "Layer 20".to_string(), + ); + + // Check after layer 11 + let version = map.get_version(105).unwrap(); + assert_eq!(version.image_coverage.query(2), None); + assert_eq!( + version.image_coverage.query(3), + Some("Layer 10".to_string()) + ); + assert_eq!( + version.image_coverage.query(5), + Some("Layer 11".to_string()) + ); + assert_eq!( + version.image_coverage.query(7), + Some("Layer 11".to_string()) + ); + assert_eq!(version.image_coverage.query(8), None); + + // Check after layer 20 + let version = map.get_version(205).unwrap(); + assert_eq!(version.image_coverage.query(2), None); + assert_eq!( + version.image_coverage.query(3), + Some("Layer 20".to_string()) + ); + assert_eq!( + version.image_coverage.query(5), + Some("Layer 11".to_string()) + ); + assert_eq!( + version.image_coverage.query(7), + Some("Layer 11".to_string()) + ); + assert_eq!(version.image_coverage.query(8), None); +} + +/// Test when rectangles have nontrivial height and possibly overlap +#[test] +fn test_persistent_overlapping() { + let mut map = HistoricLayerCoverage::::new(); + + // Add 3 key-disjoint layers with varying LSN ranges + map.insert( + LayerKey { + key: 1..2, + lsn: 100..200, + is_image: true, + }, + "Layer 1".to_string(), + ); + map.insert( + LayerKey { + key: 4..5, + lsn: 110..200, + is_image: true, + }, + "Layer 2".to_string(), + ); + map.insert( + LayerKey { + key: 7..8, + lsn: 120..300, + is_image: true, + }, + "Layer 3".to_string(), + ); + + // Add wide and short layer + map.insert( + LayerKey { + key: 0..9, + lsn: 130..199, + is_image: true, + }, + "Layer 4".to_string(), + ); + + // Add wide layer taller than some + map.insert( + LayerKey { + key: 0..9, + lsn: 140..201, + is_image: true, + }, + "Layer 5".to_string(), + ); + + // Add wide layer taller than all + map.insert( + LayerKey { + key: 0..9, + lsn: 150..301, + is_image: true, + }, + "Layer 6".to_string(), + ); + + // After layer 4 insertion + let version = map.get_version(135).unwrap(); + assert_eq!(version.image_coverage.query(0), Some("Layer 4".to_string())); + assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string())); + assert_eq!(version.image_coverage.query(2), Some("Layer 4".to_string())); + assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string())); + assert_eq!(version.image_coverage.query(5), Some("Layer 4".to_string())); + assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string())); + assert_eq!(version.image_coverage.query(8), Some("Layer 4".to_string())); + + // After layer 5 insertion + let version = map.get_version(145).unwrap(); + assert_eq!(version.image_coverage.query(0), Some("Layer 5".to_string())); + assert_eq!(version.image_coverage.query(1), Some("Layer 5".to_string())); + assert_eq!(version.image_coverage.query(2), Some("Layer 5".to_string())); + assert_eq!(version.image_coverage.query(4), Some("Layer 5".to_string())); + assert_eq!(version.image_coverage.query(5), Some("Layer 5".to_string())); + assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string())); + assert_eq!(version.image_coverage.query(8), Some("Layer 5".to_string())); + + // After layer 6 insertion + let version = map.get_version(155).unwrap(); + assert_eq!(version.image_coverage.query(0), Some("Layer 6".to_string())); + assert_eq!(version.image_coverage.query(1), Some("Layer 6".to_string())); + assert_eq!(version.image_coverage.query(2), Some("Layer 6".to_string())); + assert_eq!(version.image_coverage.query(4), Some("Layer 6".to_string())); + assert_eq!(version.image_coverage.query(5), Some("Layer 6".to_string())); + assert_eq!(version.image_coverage.query(7), Some("Layer 6".to_string())); + assert_eq!(version.image_coverage.query(8), Some("Layer 6".to_string())); +} + +/// Wrapper for HistoricLayerCoverage that allows us to hack around the lack +/// of support for retroactive insertion by rebuilding the map since the +/// change. +/// +/// Why is this needed? We most often insert new layers with newer LSNs, +/// but during compaction we create layers with non-latest LSN, and during +/// GC we delete historic layers. +/// +/// Even though rebuilding is an expensive (N log N) solution to the problem, +/// it's not critical since we do something equally expensive just to decide +/// whether or not to create new image layers. +/// TODO It's not expensive but it's not great to hold a layer map write lock +/// for that long. +/// +/// If this becomes an actual bottleneck, one solution would be to build a +/// segment tree that holds PersistentLayerMaps. Though this would mean that +/// we take an additional log(N) performance hit for queries, which will probably +/// still be more critical. +/// +/// See this for more on persistent and retroactive techniques: +/// https://www.youtube.com/watch?v=WqCWghETNDc&t=581s +pub struct BufferedHistoricLayerCoverage { + /// A persistent layer map that we rebuild when we need to retroactively update + historic_coverage: HistoricLayerCoverage, + + /// We buffer insertion into the PersistentLayerMap to decrease the number of rebuilds. + buffer: BTreeMap>, + + /// All current layers. This is not used for search. Only to make rebuilds easier. + layers: BTreeMap, +} + +impl std::fmt::Debug for BufferedHistoricLayerCoverage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RetroactiveLayerMap") + .field("buffer", &self.buffer) + .field("layers", &self.layers) + .finish() + } +} + +impl Default for BufferedHistoricLayerCoverage { + fn default() -> Self { + Self::new() + } +} + +impl BufferedHistoricLayerCoverage { + pub fn new() -> Self { + Self { + historic_coverage: HistoricLayerCoverage::::new(), + buffer: BTreeMap::new(), + layers: BTreeMap::new(), + } + } + + pub fn insert(&mut self, layer_key: LayerKey, value: Value) { + self.buffer.insert(layer_key, Some(value)); + } + + pub fn remove(&mut self, layer_key: LayerKey) { + self.buffer.insert(layer_key, None); + } + + pub fn rebuild(&mut self) { + // Find the first LSN that needs to be rebuilt + let rebuild_since: u64 = match self.buffer.iter().next() { + Some((LayerKey { lsn, .. }, _)) => lsn.start, + None => return, // No need to rebuild if buffer is empty + }; + + // Apply buffered updates to self.layers + let num_updates = self.buffer.len(); + self.buffer.retain(|layer_key, layer| { + match layer { + Some(l) => { + self.layers.insert(layer_key.clone(), l.clone()); + } + None => { + self.layers.remove(layer_key); + } + }; + false + }); + + // Rebuild + let mut num_inserted = 0; + self.historic_coverage.trim(&rebuild_since); + for (layer_key, layer) in self.layers.range( + LayerKey { + lsn: rebuild_since..0, + key: 0..0, + is_image: false, + }.., + ) { + self.historic_coverage + .insert(layer_key.clone(), layer.clone()); + num_inserted += 1; + } + + // TODO maybe only warn if ratio is at least 10 + info!( + "Rebuilt layer map. Did {} insertions to process a batch of {} updates.", + num_inserted, num_updates, + ) + } + + /// Iterate all the layers + pub fn iter(&self) -> impl '_ + Iterator { + // NOTE we can actually perform this without rebuilding, + // but it's not necessary for now. + if !self.buffer.is_empty() { + panic!("rebuild pls") + } + + self.layers.values().cloned() + } + + /// Return a reference to a queryable map, assuming all updates + /// have already been processed using self.rebuild() + pub fn get(&self) -> anyhow::Result<&HistoricLayerCoverage> { + // NOTE we error here instead of implicitly rebuilding because + // rebuilding is somewhat expensive. + // TODO maybe implicitly rebuild and log/sentry an error? + if !self.buffer.is_empty() { + anyhow::bail!("rebuild required") + } + + Ok(&self.historic_coverage) + } +} + +#[test] +fn test_retroactive_regression_1() { + let mut map = BufferedHistoricLayerCoverage::new(); + + map.insert( + LayerKey { + key: 0..21267647932558653966460912964485513215, + lsn: 23761336..23761457, + is_image: false, + }, + "sdfsdfs".to_string(), + ); + + map.rebuild(); + + let version = map.get().unwrap().get_version(23761457).unwrap(); + assert_eq!( + version.delta_coverage.query(100), + Some("sdfsdfs".to_string()) + ); +} + +#[test] +fn test_retroactive_simple() { + let mut map = BufferedHistoricLayerCoverage::new(); + + // Append some images in increasing LSN order + map.insert( + LayerKey { + key: 0..5, + lsn: 100..101, + is_image: true, + }, + "Image 1".to_string(), + ); + map.insert( + LayerKey { + key: 3..9, + lsn: 110..111, + is_image: true, + }, + "Image 2".to_string(), + ); + map.insert( + LayerKey { + key: 4..6, + lsn: 120..121, + is_image: true, + }, + "Image 3".to_string(), + ); + map.insert( + LayerKey { + key: 8..9, + lsn: 120..121, + is_image: true, + }, + "Image 4".to_string(), + ); + + // Add a delta layer out of order + map.insert( + LayerKey { + key: 2..5, + lsn: 105..106, + is_image: true, + }, + "Delta 1".to_string(), + ); + + // Rebuild so we can start querying + map.rebuild(); + + // Query key 4 + let version = map.get().unwrap().get_version(90); + assert!(version.is_none()); + let version = map.get().unwrap().get_version(102).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string())); + let version = map.get().unwrap().get_version(107).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Delta 1".to_string())); + let version = map.get().unwrap().get_version(115).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string())); + let version = map.get().unwrap().get_version(125).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string())); + + // Remove Image 3 + map.remove(LayerKey { + key: 4..6, + lsn: 120..121, + is_image: true, + }); + map.rebuild(); + + // Check deletion worked + let version = map.get().unwrap().get_version(125).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string())); + assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string())); +} diff --git a/pageserver/src/tenant/layer_map/layer_coverage.rs b/pageserver/src/tenant/layer_map/layer_coverage.rs new file mode 100644 index 0000000000..4e3b4516dc --- /dev/null +++ b/pageserver/src/tenant/layer_map/layer_coverage.rs @@ -0,0 +1,154 @@ +use std::ops::Range; + +// TODO the `im` crate has 20x more downloads and also has +// persistent/immutable BTree. It also runs a bit faster but +// results are not the same on some tests. +use rpds::RedBlackTreeMapSync; + +/// Data structure that can efficiently: +/// - find the latest layer by lsn.end at a given key +/// - iterate the latest layers in a key range +/// - insert layers in non-decreasing lsn.start order +/// +/// The struct is parameterized over Value for easier +/// testing, but in practice it's some sort of layer. +pub struct LayerCoverage { + /// For every change in coverage (as we sweep the key space) + /// we store (lsn.end, value). + /// + /// We use an immutable/persistent tree so that we can keep historic + /// versions of this coverage without cloning the whole thing and + /// incurring quadratic memory cost. See HistoricLayerCoverage. + /// + /// We use the Sync version of the map because we want Self to + /// be Sync. Using nonsync might be faster, if we can work with + /// that. + nodes: RedBlackTreeMapSync>, +} + +impl Default for LayerCoverage { + fn default() -> Self { + Self::new() + } +} + +impl LayerCoverage { + pub fn new() -> Self { + Self { + nodes: RedBlackTreeMapSync::default(), + } + } + + /// Helper function to subdivide the key range without changing any values + /// + /// Complexity: O(log N) + fn add_node(&mut self, key: i128) { + let value = match self.nodes.range(..=key).last() { + Some((_, Some(v))) => Some(v.clone()), + Some((_, None)) => None, + None => None, + }; + self.nodes.insert_mut(key, value); + } + + /// Insert a layer. + /// + /// Complexity: worst case O(N), in practice O(log N). See NOTE in implementation. + pub fn insert(&mut self, key: Range, lsn: Range, value: Value) { + // Add nodes at endpoints + // + // NOTE The order of lines is important. We add nodes at the start + // and end of the key range **before updating any nodes** in order + // to pin down the current coverage outside of the relevant key range. + // Only the coverage inside the layer's key range should change. + self.add_node(key.start); + self.add_node(key.end); + + // Raise the height where necessary + // + // NOTE This loop is worst case O(N), but amortized O(log N) in the special + // case when rectangles have no height. In practice I don't think we'll see + // the kind of layer intersections needed to trigger O(N) behavior. The worst + // case is N/2 horizontal layers overlapped with N/2 vertical layers in a + // grid pattern. + let mut to_update = Vec::new(); + let mut to_remove = Vec::new(); + let mut prev_covered = false; + for (k, node) in self.nodes.range(key.clone()) { + let needs_cover = match node { + None => true, + Some((h, _)) => h < &lsn.end, + }; + if needs_cover { + match prev_covered { + true => to_remove.push(*k), + false => to_update.push(*k), + } + } + prev_covered = needs_cover; + } + if !prev_covered { + to_remove.push(key.end); + } + for k in to_update { + self.nodes.insert_mut(k, Some((lsn.end, value.clone()))); + } + for k in to_remove { + self.nodes.remove_mut(&k); + } + } + + /// Get the latest (by lsn.end) layer at a given key + /// + /// Complexity: O(log N) + pub fn query(&self, key: i128) -> Option { + self.nodes + .range(..=key) + .rev() + .next()? + .1 + .as_ref() + .map(|(_, v)| v.clone()) + } + + /// Iterate the changes in layer coverage in a given range. You will likely + /// want to start with self.query(key.start), and then follow up with self.range + /// + /// Complexity: O(log N + result_size) + pub fn range(&self, key: Range) -> impl '_ + Iterator)> { + self.nodes + .range(key) + .map(|(k, v)| (*k, v.as_ref().map(|x| x.1.clone()))) + } + + /// O(1) clone + pub fn clone(&self) -> Self { + Self { + nodes: self.nodes.clone(), + } + } +} + +/// Image and delta coverage at a specific LSN. +pub struct LayerCoverageTuple { + pub image_coverage: LayerCoverage, + pub delta_coverage: LayerCoverage, +} + +impl Default for LayerCoverageTuple { + fn default() -> Self { + Self { + image_coverage: LayerCoverage::default(), + delta_coverage: LayerCoverage::default(), + } + } +} + +impl LayerCoverageTuple { + pub fn clone(&self) -> Self { + Self { + image_coverage: self.image_coverage.clone(), + delta_coverage: self.delta_coverage.clone(), + } + } +} diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 6aee8ce23c..2149fc7eb7 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -196,3 +196,50 @@ pub fn downcast_remote_layer( None } } + +impl std::fmt::Debug for dyn Layer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Layer") + .field("short_id", &self.short_id()) + .finish() + } +} + +/// Holds metadata about a layer without any content. Used mostly for testing. +pub struct LayerDescriptor { + pub key: Range, + pub lsn: Range, + pub is_incremental: bool, + pub short_id: String, +} + +impl Layer for LayerDescriptor { + fn get_key_range(&self) -> Range { + self.key.clone() + } + + fn get_lsn_range(&self) -> Range { + self.lsn.clone() + } + + fn is_incremental(&self) -> bool { + self.is_incremental + } + + fn get_value_reconstruct_data( + &self, + _key: Key, + _lsn_range: Range, + _reconstruct_data: &mut ValueReconstructState, + ) -> Result { + todo!("This method shouldn't be part of the Layer trait") + } + + fn short_id(&self) -> String { + self.short_id.clone() + } + + fn dump(&self, _verbose: bool) -> Result<()> { + todo!() + } +} diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index d59858f582..5b84df74d4 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -970,6 +970,7 @@ impl Timeline { /// pub(super) fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> { let mut layers = self.layers.write().unwrap(); + let mut updates = layers.batch_update(); let mut num_layers = 0; let timer = self.metrics.load_layer_map_histo.start_timer(); @@ -1010,7 +1011,7 @@ impl Timeline { trace!("found layer {}", layer.path().display()); total_physical_size += file_size; - layers.insert_historic(Arc::new(layer)); + updates.insert_historic(Arc::new(layer)); num_layers += 1; } else if let Some(deltafilename) = DeltaFileName::parse_str(&fname) { // Create a DeltaLayer struct for each delta file. @@ -1041,7 +1042,7 @@ impl Timeline { trace!("found layer {}", layer.path().display()); total_physical_size += file_size; - layers.insert_historic(Arc::new(layer)); + updates.insert_historic(Arc::new(layer)); num_layers += 1; } else if fname == METADATA_FILE_NAME || fname.ends_with(".old") { // ignore these @@ -1067,6 +1068,7 @@ impl Timeline { } } + updates.flush(); layers.next_open_layer_at = Some(Lsn(disk_consistent_lsn.0) + 1); info!( @@ -1091,6 +1093,11 @@ impl Timeline { // Are we missing some files that are present in remote storage? // Create RemoteLayer instances for them. let mut local_only_layers = local_layers; + + // We're holding a layer map lock for a while but this + // method is only called during init so it's fine. + let mut layer_map = self.layers.write().unwrap(); + let mut updates = layer_map.batch_update(); for remote_layer_name in &index_part.timeline_layers { let local_layer = local_only_layers.remove(remote_layer_name); @@ -1129,7 +1136,7 @@ impl Timeline { anyhow::bail!("could not rename file {local_layer_path:?}: {err:?}"); } else { self.metrics.resident_physical_size_gauge.sub(local_size); - self.layers.write().unwrap().remove_historic(local_layer); + updates.remove_historic(local_layer); // fall-through to adding the remote layer } } else { @@ -1171,7 +1178,7 @@ impl Timeline { ); let remote_layer = Arc::new(remote_layer); - self.layers.write().unwrap().insert_historic(remote_layer); + updates.insert_historic(remote_layer); } LayerFileName::Delta(deltafilename) => { // Create a RemoteLayer for the delta file. @@ -1194,13 +1201,14 @@ impl Timeline { &remote_layer_metadata, ); let remote_layer = Arc::new(remote_layer); - self.layers.write().unwrap().insert_historic(remote_layer); + updates.insert_historic(remote_layer); } #[cfg(test)] LayerFileName::Test(_) => unreachable!(), } } + updates.flush(); Ok(local_only_layers) } @@ -2099,10 +2107,11 @@ impl Timeline { ])?; // Add it to the layer map - { - let mut layers = self.layers.write().unwrap(); - layers.insert_historic(Arc::new(new_delta)); - } + self.layers + .write() + .unwrap() + .batch_update() + .insert_historic(Arc::new(new_delta)); // update the timeline's physical size let sz = new_delta_path.metadata()?.len(); @@ -2166,13 +2175,15 @@ impl Timeline { // are some delta layers *later* than current 'lsn', if more WAL was processed and flushed // after we read last_record_lsn, which is passed here in the 'lsn' argument. if img_lsn < lsn { - let num_deltas = layers.count_deltas(&img_range, &(img_lsn..lsn))?; + let threshold = self.get_image_creation_threshold(); + let num_deltas = + layers.count_deltas(&img_range, &(img_lsn..lsn), Some(threshold))?; debug!( "key range {}-{}, has {} deltas on this timeline in LSN range {}..{}", img_range.start, img_range.end, num_deltas, img_lsn, lsn ); - if num_deltas >= self.get_image_creation_threshold() { + if num_deltas >= threshold { return Ok(true); } } @@ -2267,6 +2278,7 @@ impl Timeline { let mut layer_paths_to_upload = HashMap::with_capacity(image_layers.len()); let mut layers = self.layers.write().unwrap(); + let mut updates = layers.batch_update(); let timeline_path = self.conf.timeline_path(&self.timeline_id, &self.tenant_id); for l in image_layers { let path = l.filename(); @@ -2280,8 +2292,9 @@ impl Timeline { self.metrics .resident_physical_size_gauge .add(metadata.len()); - layers.insert_historic(Arc::new(l)); + updates.insert_historic(Arc::new(l)); } + updates.flush(); drop(layers); timer.stop_and_record(); @@ -2577,6 +2590,7 @@ impl Timeline { } let mut layers = self.layers.write().unwrap(); + let mut updates = layers.batch_update(); let mut new_layer_paths = HashMap::with_capacity(new_layers.len()); for l in new_layers { let new_delta_path = l.path(); @@ -2597,7 +2611,7 @@ impl Timeline { new_layer_paths.insert(new_delta_path, LayerFileMetadata::new(metadata.len())); let x: Arc = Arc::new(l); - layers.insert_historic(x); + updates.insert_historic(x); } // Now that we have reshuffled the data to set of new delta layers, we can @@ -2611,8 +2625,9 @@ impl Timeline { } layer_names_to_delete.push(l.filename()); l.delete()?; - layers.remove_historic(l); + updates.remove_historic(l); } + updates.flush(); drop(layers); // Also schedule the deletions in remote storage @@ -2812,6 +2827,7 @@ impl Timeline { // 3. it doesn't need to be retained for 'retain_lsns'; // 4. newer on-disk image layers cover the layer's whole key range // + // TODO holding a write lock is too agressive and avoidable let mut layers = self.layers.write().unwrap(); 'outer: for l in layers.iter_historic_layers() { result.layers_total += 1; @@ -2843,6 +2859,8 @@ impl Timeline { // might be referenced by child branches forever. // We can track this in child timeline GC and delete parent layers when // they are no longer needed. This might be complicated with long inheritance chains. + // + // TODO Vec is not a great choice for `retain_lsns` for retain_lsn in &retain_lsns { // start_lsn is inclusive if &l.get_lsn_range().start <= retain_lsn { @@ -2896,6 +2914,7 @@ impl Timeline { layers_to_remove.push(Arc::clone(&l)); } + let mut updates = layers.batch_update(); if !layers_to_remove.is_empty() { // Persist the new GC cutoff value in the metadata file, before // we actually remove anything. @@ -2913,7 +2932,13 @@ impl Timeline { } layer_names_to_delete.push(doomed_layer.filename()); doomed_layer.delete()?; // FIXME: schedule succeeded deletions before returning? - layers.remove_historic(doomed_layer); + + // TODO Removing from the bottom of the layer map is expensive. + // Maybe instead discard all layer map historic versions that + // won't be needed for page reconstruction for this timeline, + // and mark what we can't delete yet as deleted from the layer + // map index without actually rebuilding the index. + updates.remove_historic(doomed_layer); result.layers_removed += 1; } @@ -2925,6 +2950,7 @@ impl Timeline { remote_client.schedule_layer_file_deletion(&layer_names_to_delete)?; } } + updates.flush(); info!( "GC completed removing {} layers, cutoff {}", @@ -3081,11 +3107,13 @@ impl Timeline { // Delta- or ImageLayer in the layer map. let new_layer = remote_layer.create_downloaded_layer(self_clone.conf, *size); let mut layers = self_clone.layers.write().unwrap(); + let mut updates = layers.batch_update(); { let l: Arc = remote_layer.clone(); - layers.remove_historic(l); + updates.remove_historic(l); } - layers.insert_historic(new_layer); + updates.insert_historic(new_layer); + updates.flush(); drop(layers); // Now that we've inserted the download into the layer map, diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index f4b71ae9b7..acc38cfb9d 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -31,7 +31,7 @@ memchr = { version = "2" } nom = { version = "7" } num-bigint = { version = "0.4" } num-integer = { version = "0.1", features = ["i128"] } -num-traits = { version = "0.2", features = ["i128", "libm"] } +num-traits = { version = "0.2", features = ["i128"] } prost = { version = "0.11" } rand = { version = "0.8", features = ["small_rng"] } regex = { version = "1" } From 5c865f46bae28532ebe0ff6ed5317ba2b30e9892 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 23 Jan 2023 10:51:09 +0200 Subject: [PATCH 014/102] Fix slru_segment_key_range function: segno was assigned to incorrect Key field (#3354) --- pageserver/src/pgdatadir_mapping.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 6ae70e3a30..cc521c5e35 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -1405,15 +1405,15 @@ fn slru_segment_key_range(kind: SlruKind, segno: u32) -> Range { Key { field1: 0x01, field2, - field3: segno, - field4: 0, + field3: 1, + field4: segno, field5: 0, field6: 0, }..Key { field1: 0x01, field2, - field3: segno, - field4: 0, + field3: 1, + field4: segno, field5: 1, field6: 0, } From a44e5eda145ff3fad570b8008f370cf5ed069703 Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Mon, 23 Jan 2023 15:08:48 +0200 Subject: [PATCH 015/102] Adding pageserver3 to staging (#3403) --- .github/ansible/staging.us-east-2.hosts.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ansible/staging.us-east-2.hosts.yaml b/.github/ansible/staging.us-east-2.hosts.yaml index 4891875369..b46e729e32 100644 --- a/.github/ansible/staging.us-east-2.hosts.yaml +++ b/.github/ansible/staging.us-east-2.hosts.yaml @@ -29,6 +29,8 @@ storage: ansible_host: i-0565a8b4008aa3f40 pageserver-2.us-east-2.aws.neon.build: ansible_host: i-01e31cdf7e970586a + pageserver-3.us-east-2.aws.neon.build: + ansible_host: i-0602a0291365ef7cc safekeepers: hosts: From 7704caa3ac8f8df041afcae82d743298ffae7913 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 23 Jan 2023 17:12:51 +0200 Subject: [PATCH 016/102] More tenant size fixes (#3410) Small changes, but hopefully this will help with the panic detected in staging, for which we cannot get the debugging information right now (end-of-branch before branch-point). --- libs/tenant_size_model/src/lib.rs | 83 ++++++++++++---------- libs/tenant_size_model/src/main.rs | 107 +++++++++++++++-------------- pageserver/src/tenant/size.rs | 15 +++- 3 files changed, 114 insertions(+), 91 deletions(-) diff --git a/libs/tenant_size_model/src/lib.rs b/libs/tenant_size_model/src/lib.rs index 92bec8aebe..b156e1be9d 100644 --- a/libs/tenant_size_model/src/lib.rs +++ b/libs/tenant_size_model/src/lib.rs @@ -134,22 +134,25 @@ impl Storage { op: Cow<'static, str>, lsn: u64, size: Option, - ) where + ) -> anyhow::Result<()> + where K: std::borrow::Borrow, Q: std::hash::Hash + Eq + std::fmt::Debug, { - let lastseg_id = *self.branches.get(branch).unwrap(); + let Some(lastseg_id) = self.branches.get(branch).copied() else { anyhow::bail!("branch not found: {branch:?}") }; let newseg_id = self.segments.len(); let lastseg = &mut self.segments[lastseg_id]; assert!(lsn > lastseg.end_lsn); + let Some(start_size) = lastseg.end_size else { anyhow::bail!("no end_size on latest segment for {branch:?}") }; + let newseg = Segment { op, parent: Some(lastseg_id), start_lsn: lastseg.end_lsn, end_lsn: lsn, - start_size: lastseg.end_size.unwrap(), + start_size, end_size: size, children_after: Vec::new(), needed: false, @@ -158,6 +161,8 @@ impl Storage { self.segments.push(newseg); *self.branches.get_mut(branch).expect("read already") = newseg_id; + + Ok(()) } /// Advances the branch with the named operation, by the relative LSN and logical size bytes. @@ -167,21 +172,24 @@ impl Storage { op: Cow<'static, str>, lsn_bytes: u64, size_bytes: i64, - ) where + ) -> anyhow::Result<()> + where K: std::borrow::Borrow, - Q: std::hash::Hash + Eq, + Q: std::hash::Hash + Eq + std::fmt::Debug, { - let lastseg_id = *self.branches.get(branch).unwrap(); + let Some(lastseg_id) = self.branches.get(branch).copied() else { anyhow::bail!("branch not found: {branch:?}") }; let newseg_id = self.segments.len(); let lastseg = &mut self.segments[lastseg_id]; + let Some(last_end_size) = lastseg.end_size else { anyhow::bail!("no end_size on latest segment for {branch:?}") }; + let newseg = Segment { op, parent: Some(lastseg_id), start_lsn: lastseg.end_lsn, end_lsn: lastseg.end_lsn + lsn_bytes, - start_size: lastseg.end_size.unwrap(), - end_size: Some((lastseg.end_size.unwrap() as i64 + size_bytes) as u64), + start_size: last_end_size, + end_size: Some((last_end_size as i64 + size_bytes) as u64), children_after: Vec::new(), needed: false, }; @@ -189,33 +197,33 @@ impl Storage { self.segments.push(newseg); *self.branches.get_mut(branch).expect("read already") = newseg_id; + Ok(()) } - pub fn insert(&mut self, branch: &Q, bytes: u64) + pub fn insert(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()> where K: std::borrow::Borrow, - Q: std::hash::Hash + Eq, + Q: std::hash::Hash + Eq + std::fmt::Debug, { - self.modify_branch(branch, "insert".into(), bytes, bytes as i64); + self.modify_branch(branch, "insert".into(), bytes, bytes as i64) } - pub fn update(&mut self, branch: &Q, bytes: u64) + pub fn update(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()> where K: std::borrow::Borrow, - Q: std::hash::Hash + Eq, + Q: std::hash::Hash + Eq + std::fmt::Debug, { - self.modify_branch(branch, "update".into(), bytes, 0i64); + self.modify_branch(branch, "update".into(), bytes, 0i64) } - pub fn delete(&mut self, branch: &Q, bytes: u64) + pub fn delete(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()> where K: std::borrow::Borrow, - Q: std::hash::Hash + Eq, + Q: std::hash::Hash + Eq + std::fmt::Debug, { - self.modify_branch(branch, "delete".into(), bytes, -(bytes as i64)); + self.modify_branch(branch, "delete".into(), bytes, -(bytes as i64)) } - /// Panics if the parent branch cannot be found. pub fn branch(&mut self, parent: &Q, name: K) -> anyhow::Result<()> where K: std::borrow::Borrow + std::fmt::Debug, @@ -236,7 +244,7 @@ impl Storage { Ok(()) } - pub fn calculate(&mut self, retention_period: u64) -> SegmentSize { + pub fn calculate(&mut self, retention_period: u64) -> anyhow::Result { // Phase 1: Mark all the segments that need to be retained for (_branch, &last_seg_id) in self.branches.iter() { let last_seg = &self.segments[last_seg_id]; @@ -261,7 +269,7 @@ impl Storage { self.size_from_snapshot_later(0) } - fn size_from_wal(&self, seg_id: usize) -> SegmentSize { + fn size_from_wal(&self, seg_id: usize) -> anyhow::Result { let seg = &self.segments[seg_id]; let this_size = seg.end_lsn - seg.start_lsn; @@ -272,10 +280,10 @@ impl Storage { for &child_id in seg.children_after.iter() { // try each child both ways let child = &self.segments[child_id]; - let p1 = self.size_from_wal(child_id); + let p1 = self.size_from_wal(child_id)?; let p = if !child.needed { - let p2 = self.size_from_snapshot_later(child_id); + let p2 = self.size_from_snapshot_later(child_id)?; if p1.total() < p2.total() { p1 } else { @@ -286,15 +294,15 @@ impl Storage { }; children.push(p); } - SegmentSize { + Ok(SegmentSize { seg_id, method: if seg.needed { WalNeeded } else { Wal }, this_size, children, - } + }) } - fn size_from_snapshot_later(&self, seg_id: usize) -> SegmentSize { + fn size_from_snapshot_later(&self, seg_id: usize) -> anyhow::Result { // If this is needed, then it's time to do the snapshot and continue // with wal method. let seg = &self.segments[seg_id]; @@ -305,10 +313,10 @@ impl Storage { for &child_id in seg.children_after.iter() { // try each child both ways let child = &self.segments[child_id]; - let p1 = self.size_from_wal(child_id); + let p1 = self.size_from_wal(child_id)?; let p = if !child.needed { - let p2 = self.size_from_snapshot_later(child_id); + let p2 = self.size_from_snapshot_later(child_id)?; if p1.total() < p2.total() { p1 } else { @@ -319,12 +327,12 @@ impl Storage { }; children.push(p); } - SegmentSize { + Ok(SegmentSize { seg_id, method: WalNeeded, this_size: seg.start_size, children, - } + }) } else { // If any of the direct children are "needed", need to be able to reconstruct here let mut children_needed = false; @@ -339,7 +347,7 @@ impl Storage { let method1 = if !children_needed { let mut children = Vec::new(); for child in seg.children_after.iter() { - children.push(self.size_from_snapshot_later(*child)); + children.push(self.size_from_snapshot_later(*child)?); } Some(SegmentSize { seg_id, @@ -355,20 +363,25 @@ impl Storage { let method2 = if children_needed || seg.children_after.len() >= 2 { let mut children = Vec::new(); for child in seg.children_after.iter() { - children.push(self.size_from_wal(*child)); + children.push(self.size_from_wal(*child)?); } + let Some(this_size) = seg.end_size else { anyhow::bail!("no end_size at junction {seg_id}") }; Some(SegmentSize { seg_id, method: SnapshotAfter, - this_size: seg.end_size.unwrap(), + this_size, children, }) } else { None }; - match (method1, method2) { - (None, None) => panic!(), + Ok(match (method1, method2) { + (None, None) => anyhow::bail!( + "neither method was applicable: children_after={}, children_needed={}", + seg.children_after.len(), + children_needed + ), (Some(method), None) => method, (None, Some(method)) => method, (Some(method1), Some(method2)) => { @@ -378,7 +391,7 @@ impl Storage { method2 } } - } + }) } } diff --git a/libs/tenant_size_model/src/main.rs b/libs/tenant_size_model/src/main.rs index 9378a98a09..e32dd055f4 100644 --- a/libs/tenant_size_model/src/main.rs +++ b/libs/tenant_size_model/src/main.rs @@ -7,118 +7,118 @@ use tenant_size_model::{Segment, SegmentSize, Storage}; // Main branch only. Some updates on it. -fn scenario_1() -> (Vec, SegmentSize) { +fn scenario_1() -> anyhow::Result<(Vec, SegmentSize)> { // Create main branch let mut storage = Storage::new("main"); // Bulk load 5 GB of data to it - storage.insert("main", 5_000); + storage.insert("main", 5_000)?; // Stream of updates for _ in 0..5 { - storage.update("main", 1_000); + storage.update("main", 1_000)?; } - let size = storage.calculate(1000); + let size = storage.calculate(1000)?; - (storage.into_segments(), size) + Ok((storage.into_segments(), size)) } // Main branch only. Some updates on it. -fn scenario_2() -> (Vec, SegmentSize) { +fn scenario_2() -> anyhow::Result<(Vec, SegmentSize)> { // Create main branch let mut storage = Storage::new("main"); // Bulk load 5 GB of data to it - storage.insert("main", 5_000); + storage.insert("main", 5_000)?; // Stream of updates for _ in 0..5 { - storage.update("main", 1_000); + storage.update("main", 1_000)?; } // Branch - storage.branch("main", "child").unwrap(); - storage.update("child", 1_000); + storage.branch("main", "child")?; + storage.update("child", 1_000)?; // More updates on parent - storage.update("main", 1_000); + storage.update("main", 1_000)?; - let size = storage.calculate(1000); + let size = storage.calculate(1000)?; - (storage.into_segments(), size) + Ok((storage.into_segments(), size)) } // Like 2, but more updates on main -fn scenario_3() -> (Vec, SegmentSize) { +fn scenario_3() -> anyhow::Result<(Vec, SegmentSize)> { // Create main branch let mut storage = Storage::new("main"); // Bulk load 5 GB of data to it - storage.insert("main", 5_000); + storage.insert("main", 5_000)?; // Stream of updates for _ in 0..5 { - storage.update("main", 1_000); + storage.update("main", 1_000)?; } // Branch - storage.branch("main", "child").unwrap(); - storage.update("child", 1_000); + storage.branch("main", "child")?; + storage.update("child", 1_000)?; // More updates on parent for _ in 0..5 { - storage.update("main", 1_000); + storage.update("main", 1_000)?; } - let size = storage.calculate(1000); + let size = storage.calculate(1000)?; - (storage.into_segments(), size) + Ok((storage.into_segments(), size)) } // Diverged branches -fn scenario_4() -> (Vec, SegmentSize) { +fn scenario_4() -> anyhow::Result<(Vec, SegmentSize)> { // Create main branch let mut storage = Storage::new("main"); // Bulk load 5 GB of data to it - storage.insert("main", 5_000); + storage.insert("main", 5_000)?; // Stream of updates for _ in 0..5 { - storage.update("main", 1_000); + storage.update("main", 1_000)?; } // Branch - storage.branch("main", "child").unwrap(); - storage.update("child", 1_000); + storage.branch("main", "child")?; + storage.update("child", 1_000)?; // More updates on parent for _ in 0..8 { - storage.update("main", 1_000); + storage.update("main", 1_000)?; } - let size = storage.calculate(1000); + let size = storage.calculate(1000)?; - (storage.into_segments(), size) + Ok((storage.into_segments(), size)) } -fn scenario_5() -> (Vec, SegmentSize) { +fn scenario_5() -> anyhow::Result<(Vec, SegmentSize)> { let mut storage = Storage::new("a"); - storage.insert("a", 5000); - storage.branch("a", "b").unwrap(); - storage.update("b", 4000); - storage.update("a", 2000); - storage.branch("a", "c").unwrap(); - storage.insert("c", 4000); - storage.insert("a", 2000); + storage.insert("a", 5000)?; + storage.branch("a", "b")?; + storage.update("b", 4000)?; + storage.update("a", 2000)?; + storage.branch("a", "c")?; + storage.insert("c", 4000)?; + storage.insert("a", 2000)?; - let size = storage.calculate(5000); + let size = storage.calculate(5000)?; - (storage.into_segments(), size) + Ok((storage.into_segments(), size)) } -fn scenario_6() -> (Vec, SegmentSize) { +fn scenario_6() -> anyhow::Result<(Vec, SegmentSize)> { use std::borrow::Cow; const NO_OP: Cow<'static, str> = Cow::Borrowed(""); @@ -133,18 +133,18 @@ fn scenario_6() -> (Vec, SegmentSize) { let mut storage = Storage::new(None); - storage.branch(&None, branches[0]).unwrap(); // at 0 - storage.modify_branch(&branches[0], NO_OP, 108951064, 43696128); // at 108951064 - storage.branch(&branches[0], branches[1]).unwrap(); // at 108951064 - storage.modify_branch(&branches[1], NO_OP, 15560408, -1851392); // at 124511472 - storage.modify_branch(&branches[0], NO_OP, 174464360, -1531904); // at 283415424 - storage.branch(&branches[0], branches[2]).unwrap(); // at 283415424 - storage.modify_branch(&branches[2], NO_OP, 15906192, 8192); // at 299321616 - storage.modify_branch(&branches[0], NO_OP, 18909976, 32768); // at 302325400 + storage.branch(&None, branches[0])?; // at 0 + storage.modify_branch(&branches[0], NO_OP, 108951064, 43696128)?; // at 108951064 + storage.branch(&branches[0], branches[1])?; // at 108951064 + storage.modify_branch(&branches[1], NO_OP, 15560408, -1851392)?; // at 124511472 + storage.modify_branch(&branches[0], NO_OP, 174464360, -1531904)?; // at 283415424 + storage.branch(&branches[0], branches[2])?; // at 283415424 + storage.modify_branch(&branches[2], NO_OP, 15906192, 8192)?; // at 299321616 + storage.modify_branch(&branches[0], NO_OP, 18909976, 32768)?; // at 302325400 - let size = storage.calculate(100_000); + let size = storage.calculate(100_000)?; - (storage.into_segments(), size) + Ok((storage.into_segments(), size)) } fn main() { @@ -163,7 +163,8 @@ fn main() { eprintln!("invalid scenario {}", other); std::process::exit(1); } - }; + } + .unwrap(); graphviz_tree(&segments, &size); } @@ -251,7 +252,7 @@ fn graphviz_tree(segments: &[Segment], tree: &SegmentSize) { #[test] fn scenarios_return_same_size() { - type ScenarioFn = fn() -> (Vec, SegmentSize); + type ScenarioFn = fn() -> anyhow::Result<(Vec, SegmentSize)>; let truths: &[(u32, ScenarioFn, _)] = &[ (line!(), scenario_1, 8000), (line!(), scenario_2, 9000), @@ -262,7 +263,7 @@ fn scenarios_return_same_size() { ]; for (line, scenario, expected) in truths { - let (_, size) = scenario(); + let (_, size) = scenario().unwrap(); assert_eq!(*expected, size.total_children(), "scenario on line {line}"); } } diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index 2181d6d4dc..61cb32fc76 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -194,6 +194,15 @@ pub(super) async fn gather_inputs( let timelines = tenant.list_timelines(); + if timelines.is_empty() { + // perhaps the tenant has just been created, and as such doesn't have any data yet + return Ok(ModelInputs { + updates: vec![], + retention_period: 0, + timeline_inputs: HashMap::default(), + }); + } + // record the used/inserted cache keys here, to remove extras not to start leaking // after initial run the cache should be quite stable, but live timelines will eventually // require new lsns to be inspected. @@ -505,10 +514,10 @@ impl ModelInputs { let Lsn(now) = *lsn; match op { Command::Update(sz) => { - storage.insert_point(&Some(*timeline_id), "".into(), now, Some(*sz)); + storage.insert_point(&Some(*timeline_id), "".into(), now, Some(*sz))?; } Command::EndOfBranch => { - storage.insert_point(&Some(*timeline_id), "".into(), now, None); + storage.insert_point(&Some(*timeline_id), "".into(), now, None)?; } Command::BranchFrom(parent) => { // This branch command may fail if it cannot find a parent to branch from. @@ -517,7 +526,7 @@ impl ModelInputs { } } - Ok(storage.calculate(self.retention_period).total_children()) + Ok(storage.calculate(self.retention_period)?.total_children()) } } From 6b6570b580d3807846ddb85635ac2354a1e462d5 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 18 Jan 2023 17:25:51 +0100 Subject: [PATCH 017/102] remove TimelineState::Suspended, introduce TimelineState::Loading The TimelineState::Suspsended was dubious to begin with. I suppose that the intention was that timelines could transition back and forth between Active and Suspended states. But practically, the code before this patch never did that. The transitions were: () ==Timeline::new==> Suspended ==*==> {Active,Broken,Stopping} One exception: Tenant::set_stopping() could transition timelines like so: !Broken ==Tenant::set_stopping()==> Suspended But Tenant itself cannot transition from stopping state to any other state. Thus, this patch removes TimelineState::Suspended and introduces a new state Loading. The aforementioned transitions change as follows: - () ==Timeline::new==> Suspended ==*==> {Active,Broken,Stopping} + () ==Timeline::new==> Loading ==*==> {Active,Broken,Stopping} - !Broken ==Tenant::set_stopping()==> Suspended + !Broken ==Tenant::set_stopping()==> Stopping Walreceiver's connection manager loop watches TimelineState to decide whether it should retry connecting, or exit. This patch changes the loop to exit when it observes the transition into Stopping state. Walreceiver isn't supposed to be started until the timeline transitions into Active state. So, this patch also adds some warn!() messages in case this happens anyways. --- libs/pageserver_api/src/models.rs | 19 +++++++++---------- pageserver/src/tenant.rs | 2 +- pageserver/src/tenant/timeline.rs | 4 ++-- .../src/walreceiver/connection_manager.rs | 14 ++++++++++++-- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index b5027cb331..d81e05aaa2 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -44,18 +44,17 @@ impl TenantState { /// A state of a timeline in pageserver's memory. #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum TimelineState { - /// Timeline is fully operational. If the containing Tenant is Active, the timeline's - /// background jobs are running otherwise they will be launched when the tenant is activated. + /// The timeline is recognized by the pageserver but is not yet operational. + /// In particular, the walreceiver connection loop is not running for this timeline. + /// It will eventually transition to state Active or Broken. + Loading, + /// The timeline is fully operational. + /// It can be queried, and the walreceiver connection loop is running. Active, - /// A timeline is recognized by pageserver, but not yet ready to operate. - /// The status indicates, that the timeline could eventually go back to Active automatically: - /// for example, if the owning tenant goes back to Active again. - Suspended, - /// A timeline is recognized by pageserver, but not yet ready to operate and not allowed to - /// automatically become Active after certain events: only a management call can change this status. + /// The timeline was previously Loading or Active but is shutting down. + /// It cannot transition back into any other state. Stopping, - /// A timeline is recognized by the pageserver, but can no longer be used for - /// any operations, because it failed to be activated. + /// The timeline is broken and not operational (previous states: Loading or Active). Broken, } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index c18c645e5b..6c49393969 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1487,7 +1487,7 @@ impl Tenant { .values() .filter(|timeline| timeline.current_state() != TimelineState::Broken); for timeline in not_broken_timelines { - timeline.set_state(TimelineState::Suspended); + timeline.set_state(TimelineState::Stopping); } } TenantState::Broken => { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 5b84df74d4..860ee45397 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -812,7 +812,7 @@ impl Timeline { pg_version: u32, ) -> Arc { let disk_consistent_lsn = metadata.disk_consistent_lsn(); - let (state, _) = watch::channel(TimelineState::Suspended); + let (state, _) = watch::channel(TimelineState::Loading); let (layer_flush_start_tx, _) = tokio::sync::watch::channel(0); let (layer_flush_done_tx, _) = tokio::sync::watch::channel((0, Ok(()))); @@ -1400,7 +1400,7 @@ impl Timeline { TimelineState::Active => continue, TimelineState::Broken | TimelineState::Stopping - | TimelineState::Suspended => { + | TimelineState::Loading => { break format!("aborted because timeline became inactive (new state: {new_state:?})") } } diff --git a/pageserver/src/walreceiver/connection_manager.rs b/pageserver/src/walreceiver/connection_manager.rs index 8b60e59305..39668ef910 100644 --- a/pageserver/src/walreceiver/connection_manager.rs +++ b/pageserver/src/walreceiver/connection_manager.rs @@ -183,13 +183,23 @@ async fn connection_manager_loop_step( new_event = async { loop { + if walreceiver_state.timeline.current_state() == TimelineState::Loading { + warn!("wal connection manager should only be launched after timeline has become active"); + } match timeline_state_updates.changed().await { Ok(()) => { let new_state = walreceiver_state.timeline.current_state(); match new_state { // we're already active as walreceiver, no need to reactivate TimelineState::Active => continue, - TimelineState::Broken | TimelineState::Stopping | TimelineState::Suspended => return ControlFlow::Continue(new_state), + TimelineState::Broken | TimelineState::Stopping => { + info!("timeline entered terminal state {new_state:?}, stopping wal connection manager loop"); + return ControlFlow::Break(()); + } + TimelineState::Loading => { + warn!("timeline transitioned back to Loading state, that should not happen"); + return ControlFlow::Continue(new_state); + } } } Err(_sender_dropped_error) => return ControlFlow::Break(()), @@ -197,7 +207,7 @@ async fn connection_manager_loop_step( } } => match new_event { ControlFlow::Continue(new_state) => { - info!("Timeline became inactive (new state: {new_state:?}), dropping current connections until it reactivates"); + info!("observed timeline state change, new state is {new_state:?}"); return ControlFlow::Continue(()); } ControlFlow::Break(()) => { From a4be54d21fe0cc923c1545e66fb5dd92b95664e2 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 23 Jan 2023 17:46:22 +0100 Subject: [PATCH 018/102] [compute_ctl] Stop updating roles on each compute start (#3391) I noticed that `compute_ctl` updates all roles on each start, search for rows like > - web_access:[FILTERED] -> update in the compute startup log. It happens since we had an adhoc hack for md5 hashes comparison, which doesn't work with scram hashes stored in the `pg_authid`. It doesn't really hurt, as nothing changes, but we just run >= 2 extra queries on each start, so fix it. --- compute_tools/src/params.rs | 8 +++++++- compute_tools/src/spec.rs | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/compute_tools/src/params.rs b/compute_tools/src/params.rs index 925a2f8ef3..0ce01ff478 100644 --- a/compute_tools/src/params.rs +++ b/compute_tools/src/params.rs @@ -1,3 +1,9 @@ pub const DEFAULT_LOG_LEVEL: &str = "info"; -pub const DEFAULT_CONNSTRING: &str = "host=localhost user=postgres"; +// From Postgres docs: +// To ease transition from the md5 method to the newer SCRAM method, if md5 is specified +// as a method in pg_hba.conf but the user's password on the server is encrypted for SCRAM +// (see below), then SCRAM-based authentication will automatically be chosen instead. +// https://www.postgresql.org/docs/15/auth-password.html +// +// So it's safe to set md5 here, as `control-plane` anyway uses SCRAM for all roles. pub const PG_HBA_ALL_MD5: &str = "host\tall\t\tall\t\t0.0.0.0/0\t\tmd5"; diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 97cd623052..4c7cca545c 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -152,8 +152,20 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { { RoleAction::Update } else if let Some(pg_pwd) = &r.encrypted_password { - // Check whether password changed or not (trim 'md5:' prefix first) - if pg_pwd[3..] != *role.encrypted_password.as_ref().unwrap() { + // Check whether password changed or not (trim 'md5' prefix first if any) + // + // This is a backward compatibility hack, which comes from the times when we were using + // md5 for everyone and hashes were stored in the console db without md5 prefix. So when + // role comes from the control-plane (json spec) `Role.encrypted_password` doesn't have md5 prefix, + // but when role comes from Postgres (`get_existing_roles` / `existing_roles`) it has this prefix. + // Here is the only place so far where we compare hashes, so it seems to be the best candidate + // to place this compatibility layer. + let pg_pwd = if let Some(stripped) = pg_pwd.strip_prefix("md5") { + stripped + } else { + pg_pwd + }; + if pg_pwd != *role.encrypted_password.as_ref().unwrap() { RoleAction::Update } else { RoleAction::None From fd18692dfbb3c0e4b16dd521d07a9077aff80752 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Mon, 23 Jan 2023 14:41:09 +0200 Subject: [PATCH 019/102] Output coloured pageserver logs for tty stdout --- Cargo.lock | 1 + libs/utils/Cargo.toml | 1 + libs/utils/src/logging.rs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index edb739f017..c76861267f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4157,6 +4157,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "atty", "bincode", "byteorder", "bytes", diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 020e4d9dd7..1f6c96bdbe 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +atty.workspace = true sentry.workspace = true async-trait.workspace = true anyhow.workspace = true diff --git a/libs/utils/src/logging.rs b/libs/utils/src/logging.rs index 82c9267f4a..02684d3d16 100644 --- a/libs/utils/src/logging.rs +++ b/libs/utils/src/logging.rs @@ -34,7 +34,7 @@ pub fn init(log_format: LogFormat) -> anyhow::Result<()> { let base_logger = tracing_subscriber::fmt() .with_env_filter(env_filter) .with_target(false) - .with_ansi(false) + .with_ansi(atty::is(atty::Stream::Stdout)) .with_writer(std::io::stdout); match log_format { From 55c184fcd7848ba06da9445c19f8ce781a2ccf45 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 24 Jan 2023 10:54:33 +0100 Subject: [PATCH 020/102] fix some anyhow::Context::context calls that should use with_context(format!(...)) Noticed this while combing through some production logs. --- pageserver/src/http/routes.rs | 2 +- pageserver/src/pgdatadir_mapping.rs | 2 +- pageserver/src/tenant/timeline.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 1eb24c1507..ebb454b4de 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -282,7 +282,7 @@ async fn timeline_detail_handler(request: Request) -> Result(timeline_info) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index cc521c5e35..31ae74ded5 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -488,7 +488,7 @@ impl Timeline { let mut buf = self .get(relsize_key, lsn) .await - .context("read relation size of {rel:?}")?; + .with_context(|| format!("read relation size of {rel:?}"))?; let relsize = buf.get_u32_le(); total_size += relsize as u64; diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 860ee45397..0f75e035c7 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -2285,7 +2285,7 @@ impl Timeline { let metadata = timeline_path .join(path.file_name()) .metadata() - .context("reading metadata of layer file {path}")?; + .with_context(|| format!("reading metadata of layer file {}", path.file_name()))?; layer_paths_to_upload.insert(path, LayerFileMetadata::new(metadata.len())); From f74080cbad4062058520ae69b7539a45dbe29d74 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 24 Jan 2023 13:57:13 +0200 Subject: [PATCH 021/102] feat(http): support ?inputs_only=true for tenant_size (#3419) this makes debugging problematic cases in the future easier, as we can just request the model inputs, use them locally to reproduce the issue with the model. --- pageserver/src/http/openapi_spec.yml | 10 +++++++++- pageserver/src/http/routes.rs | 26 +++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index f9b8a81dad..23faff7ace 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -430,6 +430,13 @@ paths: schema: type: string format: hex + - name: inputs_only + in: query + required: false + schema: + type: boolean + description: | + When true, skip calculation and only provide the model inputs (for debugging). Defaults to false. get: description: | Calculate tenant's size, which is a mixture of WAL (bytes) and logical_size (bytes). @@ -449,8 +456,9 @@ paths: format: hex size: type: integer + nullable: true description: | - Size metric in bytes. + Size metric in bytes or null if inputs_only=true was given. "401": description: Unauthorized Error content: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index ebb454b4de..3ce239e090 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -453,21 +453,39 @@ async fn tenant_status(request: Request) -> Result, ApiErro json_response(StatusCode::OK, tenant_info) } +/// HTTP endpoint to query the current tenant_size of a tenant. +/// +/// This is not used by consumption metrics under [`crate::consumption_metrics`], but can be used +/// to debug any of the calculations. Requires `tenant_id` request parameter, supports +/// `inputs_only=true|false` (default false) which supports debugging failure to calculate model +/// values. async fn tenant_size_handler(request: Request) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; + let inputs_only = if query_param_present(&request, "inputs_only") { + get_query_param(&request, "inputs_only")? + .parse() + .map_err(|_| ApiError::BadRequest(anyhow!("failed to parse inputs_only")))? + } else { + false + }; + let tenant = mgr::get_tenant(tenant_id, true) .await .map_err(ApiError::InternalServerError)?; - // this can be long operation, it currently is not backed by any request coalescing or similar + // this can be long operation let inputs = tenant .gather_size_inputs() .await .map_err(ApiError::InternalServerError)?; - let size = inputs.calculate().map_err(ApiError::InternalServerError)?; + let size = if !inputs_only { + Some(inputs.calculate().map_err(ApiError::InternalServerError)?) + } else { + None + }; /// Private response type with the additional "unstable" `inputs` field. /// @@ -479,7 +497,9 @@ async fn tenant_size_handler(request: Request) -> Result, A #[serde_as(as = "serde_with::DisplayFromStr")] id: TenantId, /// Size is a mixture of WAL and logical size, so the unit is bytes. - size: u64, + /// + /// Will be none if `?inputs_only=true` was given. + size: Option, inputs: crate::tenant::size::ModelInputs, } From 98d0a0d2422b99db7274e706670ec1b6c175d6cc Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 24 Jan 2023 14:53:39 +0200 Subject: [PATCH 022/102] fix(http): omit needless string allocs (#3421) Drive-by fix noticed while #3419. --- pageserver/src/http/routes.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 3ce239e090..c433b695a7 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -239,11 +239,7 @@ fn query_param_present(request: &Request, param: &str) -> bool { request .uri() .query() - .map(|v| { - url::form_urlencoded::parse(v.as_bytes()) - .into_owned() - .any(|(p, _)| p == param) - }) + .map(|v| url::form_urlencoded::parse(v.as_bytes()).any(|(p, _)| p == param)) .unwrap_or(false) } @@ -252,13 +248,12 @@ fn get_query_param(request: &Request, param_name: &str) -> Result Date: Mon, 23 Jan 2023 14:08:26 +0100 Subject: [PATCH 023/102] add doc comment that outlines which tokio tasks walreceiver creates --- pageserver/src/task_mgr.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 02e2e2ee14..331d8e8ac8 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -183,12 +183,29 @@ pub enum TaskKind { // associated with one later, after receiving a command from the client. PageRequestHandler, - // Manages the WAL receiver connection for one timeline. It subscribes to - // events from storage_broker, decides which safekeeper to connect to. It spawns a - // separate WalReceiverConnection task to handle each connection. + /// Manages the WAL receiver connection for one timeline. + /// It subscribes to events from storage_broker and decides which safekeeper to connect to. + /// Once the decision has been made, it establishes the connection using the `tokio-postgres` library. + /// There is at most one connection at any given time. + /// + /// That `tokio-postgres` library represents a connection as two objects: a `Client` and a `Connection`. + /// The `Client` object is what library users use to make requests & get responses. + /// Internally, `Client` hands over requests to the `Connection` object. + /// The `Connection` object is responsible for speaking the wire protocol. + /// + /// Walreceiver uses its own abstraction called `TaskHandle` to represent the activity of establishing and handling a connection. + /// That abstraction doesn't use `task_mgr` and hence, has no `TaskKind`. + /// The [`WalReceiverManager`] task ensures that this `TaskHandle` task does not outlive the [`WalReceiverManager`] task. + /// + /// Once the connection is established, the `TaskHandle` task creates a + /// [`WalReceiverConnection`] task_mgr task that is responsible for polling + /// the `Connection` object. + /// A `CancellationToken` created by the `TaskHandle` task ensures + /// that the [`WalReceiverConnection`] task will cancel soon after as the `TaskHandle` is dropped. WalReceiverManager, - // Handles a connection to a safekeeper, to stream WAL to a timeline. + /// The task that polls the `tokio-postgres::Connection` object. + /// See the comment on [`WalReceiverManager`]. WalReceiverConnection, // Garbage collection worker. One per tenant From 7a333cfb1275e9baeb60c5fd45a956a0c730d8a7 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 18 Jan 2023 18:05:41 +0100 Subject: [PATCH 024/102] be noisy about unexpected Timeline state transitions --- pageserver/src/tenant/timeline.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 0f75e035c7..1242d3e3c8 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -732,13 +732,16 @@ impl Timeline { pub fn set_state(&self, new_state: TimelineState) { match (self.current_state(), new_state) { (equal_state_1, equal_state_2) if equal_state_1 == equal_state_2 => { - debug!("Ignoring new state, equal to the existing one: {equal_state_2:?}"); + warn!("Ignoring new state, equal to the existing one: {equal_state_2:?}"); + } + (st, TimelineState::Loading) => { + error!("ignoring transition from {st:?} into Loading state"); } (TimelineState::Broken, _) => { error!("Ignoring state update {new_state:?} for broken tenant"); } (TimelineState::Stopping, TimelineState::Active) => { - debug!("Not activating a Stopping timeline"); + error!("Not activating a Stopping timeline"); } (_, new_state) => { self.state.send_replace(new_state); From 0b673c12d7fb32b21d39bd293b21204d5d5f9172 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 18 Jan 2023 18:05:41 +0100 Subject: [PATCH 025/102] timeline: don't transition Active=>Active during pageserver startup Before this patch, when `initialize_with_lock` was called via `timeline_init_and_sync`, we would transition the timeline like so: load_local_timeline/load_remote_timeline: timeline_init_and_sync Timeline::new () => Loading initialize_with_lock: set_state(Active) Loading => Active timeline.activate() Active => Active --- pageserver/src/tenant.rs | 10 ++++------ pageserver/src/tenant/timeline.rs | 5 +++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 6c49393969..a503604ae8 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -188,7 +188,7 @@ impl UninitializedTimeline<'_> { mut self, timelines: &mut HashMap>, load_layer_map: bool, - launch_wal_receiver: bool, + activate: bool, ) -> anyhow::Result> { let timeline_id = self.timeline_id; let tenant_id = self.owning_tenant.tenant_id; @@ -221,13 +221,12 @@ impl UninitializedTimeline<'_> { "Failed to remove uninit mark file for timeline {tenant_id}/{timeline_id}" ) })?; - new_timeline.set_state(TimelineState::Active); v.insert(Arc::clone(&new_timeline)); new_timeline.maybe_spawn_flush_loop(); - if launch_wal_receiver { - new_timeline.launch_wal_receiver(); + if activate { + new_timeline.activate(); } } } @@ -1462,8 +1461,7 @@ impl Tenant { tasks::start_background_loops(self.tenant_id); for timeline in not_broken_timelines { - timeline.set_state(TimelineState::Active); - timeline.launch_wal_receiver(); + timeline.activate(); } } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 1242d3e3c8..b22a339839 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -729,6 +729,11 @@ impl Timeline { Ok(()) } + pub fn activate(self: &Arc) { + self.set_state(TimelineState::Active); + self.launch_wal_receiver(); + } + pub fn set_state(&self, new_state: TimelineState) { match (self.current_state(), new_state) { (equal_state_1, equal_state_2) if equal_state_1 == equal_state_2 => { From 0c16ad859151953ed86434990e102f0cd50820d0 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Tue, 24 Jan 2023 14:11:48 +0200 Subject: [PATCH 026/102] Tone down broker subscription errors --- pageserver/src/walreceiver/connection_manager.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pageserver/src/walreceiver/connection_manager.rs b/pageserver/src/walreceiver/connection_manager.rs index 39668ef910..2214cd79dd 100644 --- a/pageserver/src/walreceiver/connection_manager.rs +++ b/pageserver/src/walreceiver/connection_manager.rs @@ -299,7 +299,9 @@ async fn subscribe_for_timeline_updates( return resp.into_inner(); } Err(e) => { - warn!("Attempt #{attempt}, failed to subscribe for timeline {id} updates in broker: {e:#}"); + // Safekeeper nodes can stop pushing timeline updates to the broker, when no new writes happen and + // entire WAL is streamed. Keep this noticeable with logging, but do not warn/error. + info!("Attempt #{attempt}, failed to subscribe for timeline {id} updates in broker: {e:#}"); continue; } } From 1c3636d8485405164fa4b7bb2cfe0422561fb1b6 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Tue, 24 Jan 2023 18:03:33 +0200 Subject: [PATCH 027/102] Tone down walreceiver connection timeout errors (#3425) Closes https://github.com/neondatabase/neon/issues/3342 --- pageserver/src/walreceiver/walreceiver_connection.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pageserver/src/walreceiver/walreceiver_connection.rs b/pageserver/src/walreceiver/walreceiver_connection.rs index 1b9e4923fb..246bfb837a 100644 --- a/pageserver/src/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/walreceiver/walreceiver_connection.rs @@ -77,9 +77,13 @@ pub async fn handle_walreceiver_connection( info!("DB connection stream finished: {expected_error}"); return Ok(()); } - Err(elapsed) => anyhow::bail!( - "Timed out while waiting {elapsed} for walreceiver connection to open" - ), + Err(_) => { + // Timing out to connect to a safekeeper node could happen long time, due to + // many reasons that pageserver cannot control. + // Do not produce an error, but make it visible, that timeouts happen by logging the `event. + info!("Timed out while waiting {connect_timeout:?} for walreceiver connection to open"); + return Ok(()); + } } }; From aabca55d7e7e020a30f6afd286c850af88063f28 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Tue, 24 Jan 2023 17:18:16 +0100 Subject: [PATCH 028/102] Migrate update version to management APIv2 (#3430) --- .github/ansible/deploy.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/ansible/deploy.yaml b/.github/ansible/deploy.yaml index 4adc685684..a17dc9c78f 100644 --- a/.github/ansible/deploy.yaml +++ b/.github/ansible/deploy.yaml @@ -117,7 +117,8 @@ shell: cmd: | INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -d '{"version": {{ current_version }} }' -X PATCH {{ console_mgmt_base_url }}/api/v1/pageservers/$INSTANCE_ID + curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version + curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/pageservers tags: - pageserver @@ -186,6 +187,7 @@ shell: cmd: | INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) - curl -sfS -d '{"version": {{ current_version }} }' -X PATCH {{ console_mgmt_base_url }}/api/v1/safekeepers/$INSTANCE_ID + curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version + curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/safekeepers tags: - safekeeper From 9fbef1159f75d62f81fcacc273eb260641fdc020 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 25 Jan 2023 12:36:30 +0200 Subject: [PATCH 029/102] Tone down http error printing (#3434) Only print backtraces for internal server error variants of the API error. --- libs/utils/src/http/error.rs | 17 +++++++++++++---- test_runner/regress/test_tenant_detach.py | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/libs/utils/src/http/error.rs b/libs/utils/src/http/error.rs index b0ecb746d9..1ba0422993 100644 --- a/libs/utils/src/http/error.rs +++ b/libs/utils/src/http/error.rs @@ -1,6 +1,7 @@ use hyper::{header, Body, Response, StatusCode}; use serde::{Deserialize, Serialize}; use thiserror::Error; +use tracing::error; #[derive(Debug, Error)] pub enum ApiError { @@ -76,8 +77,16 @@ impl HttpErrorBody { } pub async fn handler(err: routerify::RouteError) -> Response { - tracing::error!("Error processing HTTP request: {:?}", err); - err.downcast::() - .expect("handler should always return api error") - .into_response() + let api_error = err + .downcast::() + .expect("handler should always return api error"); + + // Print a stack trace for Internal Server errors + if let ApiError::InternalServerError(_) = api_error.as_ref() { + error!("Error processing HTTP request: {api_error:?}"); + } else { + error!("Error processing HTTP request: {api_error:#}"); + } + + api_error.into_response() } diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index db5bb679f2..ef6ac1527e 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -237,7 +237,7 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() - env.pageserver.allowed_errors.append(".*NotFound\\(Tenant .* not found") + env.pageserver.allowed_errors.append(".*NotFound: Tenant .* not found") # first check for non existing tenant tenant_id = TenantId.generate() From 3d5faa0295088eeb2ebfeab599cc7d2818582062 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 25 Jan 2023 12:39:53 +0200 Subject: [PATCH 030/102] Unify common image args in compute-node Dockerfiles (#3437) Part of https://github.com/neondatabase/neon/issues/3436 --- Dockerfile.compute-node-v14 | 4 +++- Dockerfile.compute-node-v15 | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile.compute-node-v14 b/Dockerfile.compute-node-v14 index 2deb95a93f..71195eff47 100644 --- a/Dockerfile.compute-node-v14 +++ b/Dockerfile.compute-node-v14 @@ -3,6 +3,8 @@ # except for the version of Postgres that is built. # +ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG IMAGE=rust ARG TAG=pinned ######################################################################################### @@ -158,7 +160,7 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \ # Compile and run the Neon-specific `compute_ctl` binary # ######################################################################################### -FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools +FROM $REPOSITORY/$IMAGE:$TAG AS compute-tools USER nonroot # Copy entire project to get Cargo.* files with proper dependencies for the whole project COPY --chown=nonroot . . diff --git a/Dockerfile.compute-node-v15 b/Dockerfile.compute-node-v15 index 8647ce2bf4..8d023631ee 100644 --- a/Dockerfile.compute-node-v15 +++ b/Dockerfile.compute-node-v15 @@ -3,6 +3,8 @@ # except for the version of Postgres that is built. # +ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG IMAGE=rust ARG TAG=pinned ######################################################################################### @@ -158,7 +160,7 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \ # Compile and run the Neon-specific `compute_ctl` binary # ######################################################################################### -FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:$TAG AS compute-tools +FROM $REPOSITORY/$IMAGE:$TAG AS compute-tools USER nonroot # Copy entire project to get Cargo.* files with proper dependencies for the whole project COPY --chown=nonroot . . From ea6f41324ab48e3abb0d7371b49e874517ccf98c Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 25 Jan 2023 12:50:33 +0200 Subject: [PATCH 031/102] Tone down postgres client io errors (#3435) Closes https://github.com/neondatabase/neon/issues/3343 --- pageserver/src/page_service.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 344a8d1c00..d52faaa872 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -30,13 +30,12 @@ use std::sync::Arc; use std::time::Duration; use tracing::*; use utils::id::ConnectionId; -use utils::postgres_backend_async::QueryError; use utils::{ auth::{Claims, JwtAuth, Scope}, id::{TenantId, TimelineId}, lsn::Lsn, postgres_backend::AuthType, - postgres_backend_async::{self, PostgresBackend}, + postgres_backend_async::{self, is_expected_io_error, PostgresBackend, QueryError}, simple_rcu::RcuReadGuard, }; @@ -194,21 +193,17 @@ async fn page_service_conn_main( let mut conn_handler = PageServerHandler::new(conf, auth); let pgbackend = PostgresBackend::new(socket, auth_type, None)?; - let result = pgbackend + match pgbackend .run(&mut conn_handler, task_mgr::shutdown_watcher) - .await; - match result { + .await + { Ok(()) => { // we've been requested to shut down Ok(()) } Err(QueryError::Disconnected(ConnectionError::Socket(io_error))) => { - // `ConnectionReset` error happens when the Postgres client closes the connection. - // As this disconnection happens quite often and is expected, - // we decided to downgrade the logging level to `INFO`. - // See: https://github.com/neondatabase/neon/issues/1683. - if io_error.kind() == io::ErrorKind::ConnectionReset { - info!("Postgres client disconnected"); + if is_expected_io_error(&io_error) { + info!("Postgres client disconnected ({io_error})"); Ok(()) } else { Err(io_error).context("Postgres connection error") From bc4f594ed65ace3632fa5c7a57d07e82638c3cc8 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Wed, 25 Jan 2023 10:36:29 +0100 Subject: [PATCH 032/102] Fix Sentry Version --- pageserver/src/bin/pageserver.rs | 7 +++++-- proxy/src/main.rs | 4 ++-- safekeeper/src/bin/safekeeper.rs | 7 +++++-- storage_broker/src/bin/storage_broker.rs | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 5de6e4def5..c5b0ec8935 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -26,7 +26,7 @@ use utils::{ logging, postgres_backend::AuthType, project_git_version, - sentry_init::{init_sentry, release_name}, + sentry_init::init_sentry, signals::{self, Signal}, tcp_listener, }; @@ -85,7 +85,10 @@ fn main() -> anyhow::Result<()> { }; // initialize sentry if SENTRY_DSN is provided - let _sentry_guard = init_sentry(release_name!(), &[("node_id", &conf.id.to_string())]); + let _sentry_guard = init_sentry( + Some(GIT_VERSION.into()), + &[("node_id", &conf.id.to_string())], + ); let tenants_path = conf.tenants_path(); if !tenants_path.exists() { diff --git a/proxy/src/main.rs b/proxy/src/main.rs index 5d44774df9..1b61ab108f 100644 --- a/proxy/src/main.rs +++ b/proxy/src/main.rs @@ -30,7 +30,7 @@ use std::{borrow::Cow, future::Future, net::SocketAddr}; use tokio::{net::TcpListener, task::JoinError}; use tracing::{info, info_span, Instrument}; use utils::project_git_version; -use utils::sentry_init::{init_sentry, release_name}; +use utils::sentry_init::init_sentry; project_git_version!(GIT_VERSION); @@ -49,7 +49,7 @@ async fn main() -> anyhow::Result<()> { .init(); // initialize sentry if SENTRY_DSN is provided - let _sentry_guard = init_sentry(release_name!(), &[]); + let _sentry_guard = init_sentry(Some(GIT_VERSION.into()), &[]); let arg_matches = cli().get_matches(); diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index b130ea86bd..1a068412c8 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -38,7 +38,7 @@ use utils::{ id::NodeId, logging::{self, LogFormat}, project_git_version, - sentry_init::{init_sentry, release_name}, + sentry_init::init_sentry, signals, tcp_listener, }; @@ -173,7 +173,10 @@ fn main() -> anyhow::Result<()> { }; // initialize sentry if SENTRY_DSN is provided - let _sentry_guard = init_sentry(release_name!(), &[("node_id", &conf.my_id.to_string())]); + let _sentry_guard = init_sentry( + Some(GIT_VERSION.into()), + &[("node_id", &conf.my_id.to_string())], + ); start_safekeeper(conf) } diff --git a/storage_broker/src/bin/storage_broker.rs b/storage_broker/src/bin/storage_broker.rs index 6d80e96bf1..e33369bbb1 100644 --- a/storage_broker/src/bin/storage_broker.rs +++ b/storage_broker/src/bin/storage_broker.rs @@ -45,7 +45,7 @@ use storage_broker::{ use utils::id::TenantTimelineId; use utils::logging::{self, LogFormat}; use utils::project_git_version; -use utils::sentry_init::{init_sentry, release_name}; +use utils::sentry_init::init_sentry; project_git_version!(GIT_VERSION); @@ -425,7 +425,7 @@ async fn http1_handler( #[tokio::main] async fn main() -> Result<(), Box> { // initialize sentry if SENTRY_DSN is provided - let _sentry_guard = init_sentry(release_name!(), &[]); + let _sentry_guard = init_sentry(Some(GIT_VERSION.into()), &[]); let args = Args::parse(); From 5223b62a194269bc5d19d325eacc357f48ed9302 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Mon, 16 Jan 2023 10:38:59 +0100 Subject: [PATCH 033/102] Compile unit extension --- Dockerfile.compute-node-v14 | 17 +++++++++++++++++ Dockerfile.compute-node-v15 | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Dockerfile.compute-node-v14 b/Dockerfile.compute-node-v14 index 71195eff47..fb115ca641 100644 --- a/Dockerfile.compute-node-v14 +++ b/Dockerfile.compute-node-v14 @@ -137,6 +137,22 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3 echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control +######################################################################################### +# +# Layer "unit-pg-build" +# compile unit extension +# +######################################################################################### +FROM build-deps AS unit-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz && \ + tar xvzf 7.7.tar.gz && \ + cd postgresql-unit-7.7 && \ + make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/unit.control + ######################################################################################### # # Layer "neon-pg-ext-build" @@ -148,6 +164,7 @@ COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=h3-pg-build /h3/usr / +COPY --from=unit-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ diff --git a/Dockerfile.compute-node-v15 b/Dockerfile.compute-node-v15 index 8d023631ee..efbe0c6872 100644 --- a/Dockerfile.compute-node-v15 +++ b/Dockerfile.compute-node-v15 @@ -137,6 +137,22 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3 echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control +######################################################################################### +# +# Layer "unit-pg-build" +# compile unit extension +# +######################################################################################### +FROM build-deps AS unit-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz && \ + tar xvzf 7.7.tar.gz && \ + cd postgresql-unit-7.7 && \ + make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/unit.control + ######################################################################################### # # Layer "neon-pg-ext-build" @@ -148,6 +164,7 @@ COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=h3-pg-build /h3/usr / +COPY --from=unit-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ From 572332ab50b2d87fa15b11b2c8c1f9e1313c915b Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 25 Jan 2023 13:40:08 +0200 Subject: [PATCH 034/102] Tone down page_service timeouts (#3426) Closes https://github.com/neondatabase/neon/issues/3341 --- pageserver/src/page_service.rs | 66 ++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index d52faaa872..0154867dae 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -13,6 +13,7 @@ use anyhow::Context; use bytes::Buf; use bytes::Bytes; use futures::{Stream, StreamExt}; +use pageserver_api::models::TenantState; use pageserver_api::models::{ PagestreamBeMessage, PagestreamDbSizeRequest, PagestreamDbSizeResponse, PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse, @@ -424,7 +425,7 @@ impl PageServerHandler { ) -> Result<(), QueryError> { task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); - let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?; + let timeline = get_active_tenant_timeline(tenant_id, timeline_id).await?; let last_record_lsn = timeline.get_last_record_lsn(); if last_record_lsn != start_lsn { return Err(QueryError::Other( @@ -622,7 +623,7 @@ impl PageServerHandler { full_backup: bool, ) -> anyhow::Result<()> { // check that the timeline exists - let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?; + let timeline = get_active_tenant_timeline(tenant_id, timeline_id).await?; let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); if let Some(lsn) = lsn { // Backup was requested at a particular LSN. Wait for it to arrive. @@ -779,7 +780,7 @@ impl postgres_backend_async::Handler for PageServerHandler { .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?; self.check_permission(Some(tenant_id))?; - let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?; + let timeline = get_active_tenant_timeline(tenant_id, timeline_id).await?; let end_of_timeline = timeline.get_last_record_rlsn(); @@ -985,27 +986,64 @@ impl postgres_backend_async::Handler for PageServerHandler { } } +#[derive(thiserror::Error, Debug)] +enum GetActiveTenantError { + #[error( + "Timed out waiting {wait_time:?} for tenant active state. Latest state: {latest_state:?}" + )] + WaitForActiveTimeout { + latest_state: TenantState, + wait_time: Duration, + }, + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +impl From for QueryError { + fn from(e: GetActiveTenantError) -> Self { + match e { + GetActiveTenantError::WaitForActiveTimeout { .. } => QueryError::Disconnected( + ConnectionError::Socket(io::Error::new(io::ErrorKind::TimedOut, e.to_string())), + ), + GetActiveTenantError::Other(e) => QueryError::Other(e), + } + } +} + /// Get active tenant. /// /// If the tenant is Loading, waits for it to become Active, for up to 30 s. That /// ensures that queries don't fail immediately after pageserver startup, because /// all tenants are still loading. -async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> anyhow::Result> { +async fn get_active_tenant_with_timeout( + tenant_id: TenantId, +) -> Result, GetActiveTenantError> { let tenant = mgr::get_tenant(tenant_id, false).await?; - match tokio::time::timeout(Duration::from_secs(30), tenant.wait_to_become_active()).await { - Ok(wait_result) => wait_result - // no .context(), the error message is good enough and some tests depend on it - .map(move |()| tenant), - Err(_) => anyhow::bail!("Timeout waiting for tenant {tenant_id} to become Active"), + let wait_time = Duration::from_secs(30); + match tokio::time::timeout(wait_time, tenant.wait_to_become_active()).await { + Ok(Ok(())) => Ok(tenant), + // no .context(), the error message is good enough and some tests depend on it + Ok(Err(wait_error)) => Err(GetActiveTenantError::Other(wait_error)), + Err(_) => { + let latest_state = tenant.current_state(); + if latest_state == TenantState::Active { + Ok(tenant) + } else { + Err(GetActiveTenantError::WaitForActiveTimeout { + latest_state, + wait_time, + }) + } + } } } /// Shorthand for getting a reference to a Timeline of an Active tenant. -async fn get_active_timeline_with_timeout( +async fn get_active_tenant_timeline( tenant_id: TenantId, timeline_id: TimelineId, -) -> anyhow::Result> { - get_active_tenant_with_timeout(tenant_id) - .await - .and_then(|tenant| tenant.get_timeline(timeline_id, true)) +) -> Result, GetActiveTenantError> { + let tenant = get_active_tenant_with_timeout(tenant_id).await?; + let timeline = tenant.get_timeline(timeline_id, true)?; + Ok(timeline) } From dee71404a21a3541f8d2df44fda94ffeb49a4a0c Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Wed, 25 Jan 2023 14:39:55 +0100 Subject: [PATCH 035/102] Use TLS for staging link proxy (#3443) Fixes #3416 on staging Adding domain parameter result in: * Issuing TLS cert for that domain * Passing that cert to proxy with `--tls-key`/`--tls-cert` --- .github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml b/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml index cb062f705d..157ae66ed1 100644 --- a/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml +++ b/.github/helm-values/dev-us-east-2-beta.neon-proxy-link.yaml @@ -8,6 +8,7 @@ settings: authBackend: "link" authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/" uri: "https://console.stage.neon.tech/psql_session/" + domain: "pg.neon.build" sentryEnvironment: "staging" metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events" metricCollectionInterval: "1min" From 01b4b0c2f3731f16f4b9b1cfcb5e7937c76df989 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 13 Jan 2023 17:02:22 +0100 Subject: [PATCH 036/102] Introduce RequestContext Motivation ========== Layer Eviction Needs Context ---------------------------- Before we start implementing layer eviction, we need to collect some access statistics per layer file or maybe even page. Part of these statistics should be the initiator of a page read request to answer the question of whether it was page_service vs. one of the background loops, and if the latter, which of them? Further, it would be nice to learn more about what activity in the pageserver initiated an on-demand download of a layer file. We will use this information to test out layer eviction policies. Read more about the current plan for layer eviction here: https://github.com/neondatabase/neon/issues/2476#issuecomment-1370822104 task_mgr problems + cancellation + tenant/timeline lifecycle ------------------------------------------------------------ Apart from layer eviction, we have long-standing problems with task_mgr, task cancellation, and various races around tenant / timeline lifecycle transitions. One approach to solve these is to abandon task_mgr in favor of a mechanism similar to Golang's context.Context, albeit extended to support waiting for completion, and specialized to the needs in the pageserver. Heikki solves all of the above at once in PR https://github.com/neondatabase/neon/pull/3228 , which is not yet merged at the time of writing. What Is This Patch About ======================== This patch addresses the immediate needs of layer eviction by introducing a `RequestContext` structure that is plumbed through the pageserver - all the way from the various entrypoints (page_service, management API, tenant background loops) down to Timeline::{get,get_reconstruct_data}. The struct carries a description of the kind of activity that initiated the call. We re-use task_mgr::TaskKind for this. Also, it carries the desired on-demand download behavior of the entrypoint. Timeline::get_reconstruct_data can then log the TaskKind that initiated the on-demand download. I developed this patch by git-checking-out Heikki's big RequestContext PR https://github.com/neondatabase/neon/pull/3228 , then deleting all the functionality that we do not need to address the needs for layer eviction. After that, I added a few things on top: 1. The concept of attached_child and detached_child in preparation for cancellation signalling through RequestContext, which will be added in a future patch. 2. A kill switch to turn DownloadBehavior::Error into a warning. 3. Renamed WalReceiverConnection to WalReceiverConnectionPoller and added an additional TaskKind WalReceiverConnectionHandler.These were necessary to create proper detached_child-type RequestContexts for the various tasks that walreceiver starts. How To Review This Patch ======================== Start your review with the module-level comment in context.rs. It explains the idea of RequestContext, what parts of it are implemented in this patch, and the future plans for RequestContext. Then review the various `task_mgr::spawn` call sites. At each of them, we should be creating a new detached_child RequestContext. Then review the (few) RequestContext::attached_child call sites and ensure that the spawned tasks do not outlive the task that spawns them. If they do, these call sites should use detached_child() instead. Then review the todo_child() call sites and judge whether it's worth the trouble of plumbing through a parent context from the caller(s). Lastly, go through the bulk of mechanical changes that simply forwards the &ctx. --- pageserver/src/basebackup.rs | 50 ++- pageserver/src/bin/pageserver.rs | 48 +- pageserver/src/config.rs | 28 ++ pageserver/src/consumption_metrics.rs | 14 +- pageserver/src/context.rs | 199 +++++++++ pageserver/src/http/routes.rs | 76 +++- pageserver/src/import_datadir.rs | 52 ++- pageserver/src/lib.rs | 1 + pageserver/src/page_service.rs | 148 +++++-- pageserver/src/pgdatadir_mapping.rs | 165 ++++--- pageserver/src/task_mgr.rs | 28 +- pageserver/src/tenant.rs | 319 +++++++------ pageserver/src/tenant/mgr.rs | 39 +- .../src/tenant/remote_timeline_client.rs | 25 +- pageserver/src/tenant/size.rs | 16 +- pageserver/src/tenant/tasks.rs | 13 +- pageserver/src/tenant/timeline.rs | 210 +++++++-- pageserver/src/walingest.rs | 418 ++++++++++++------ .../src/walreceiver/connection_manager.rs | 25 +- .../src/walreceiver/walreceiver_connection.rs | 16 +- 20 files changed, 1351 insertions(+), 539 deletions(-) create mode 100644 pageserver/src/context.rs diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index f1d92ac36b..06d4853274 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -27,6 +27,7 @@ use tracing::*; /// use tokio_tar::{Builder, EntryType, Header}; +use crate::context::RequestContext; use crate::tenant::Timeline; use pageserver_api::reltag::{RelTag, SlruKind}; @@ -52,6 +53,7 @@ pub async fn send_basebackup_tarball<'a, W>( req_lsn: Option, prev_lsn: Option, full_backup: bool, + ctx: &'a RequestContext, ) -> anyhow::Result<()> where W: AsyncWrite + Send + Sync + Unpin, @@ -110,6 +112,7 @@ where lsn: backup_lsn, prev_record_lsn: prev_lsn, full_backup, + ctx, }; basebackup .send_tarball() @@ -129,6 +132,7 @@ where lsn: Lsn, prev_record_lsn: Lsn, full_backup: bool, + ctx: &'a RequestContext, } impl<'a, W> Basebackup<'a, W> @@ -171,23 +175,37 @@ where SlruKind::MultiXactOffsets, SlruKind::MultiXactMembers, ] { - for segno in self.timeline.list_slru_segments(kind, self.lsn).await? { + for segno in self + .timeline + .list_slru_segments(kind, self.lsn, self.ctx) + .await? + { self.add_slru_segment(kind, segno).await?; } } // Create tablespace directories - for ((spcnode, dbnode), has_relmap_file) in self.timeline.list_dbdirs(self.lsn).await? { + for ((spcnode, dbnode), has_relmap_file) in + self.timeline.list_dbdirs(self.lsn, self.ctx).await? + { self.add_dbdir(spcnode, dbnode, has_relmap_file).await?; // Gather and send relational files in each database if full backup is requested. if self.full_backup { - for rel in self.timeline.list_rels(spcnode, dbnode, self.lsn).await? { + for rel in self + .timeline + .list_rels(spcnode, dbnode, self.lsn, self.ctx) + .await? + { self.add_rel(rel).await?; } } } - for xid in self.timeline.list_twophase_files(self.lsn).await? { + for xid in self + .timeline + .list_twophase_files(self.lsn, self.ctx) + .await? + { self.add_twophase_file(xid).await?; } @@ -203,7 +221,10 @@ where } async fn add_rel(&mut self, tag: RelTag) -> anyhow::Result<()> { - let nblocks = self.timeline.get_rel_size(tag, self.lsn, false).await?; + let nblocks = self + .timeline + .get_rel_size(tag, self.lsn, false, self.ctx) + .await?; // If the relation is empty, create an empty file if nblocks == 0 { @@ -223,7 +244,7 @@ where for blknum in startblk..endblk { let img = self .timeline - .get_rel_page_at_lsn(tag, blknum, self.lsn, false) + .get_rel_page_at_lsn(tag, blknum, self.lsn, false, self.ctx) .await?; segment_data.extend_from_slice(&img[..]); } @@ -245,14 +266,14 @@ where async fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> { let nblocks = self .timeline - .get_slru_segment_size(slru, segno, self.lsn) + .get_slru_segment_size(slru, segno, self.lsn, self.ctx) .await?; let mut slru_buf: Vec = Vec::with_capacity(nblocks as usize * BLCKSZ as usize); for blknum in 0..nblocks { let img = self .timeline - .get_slru_page_at_lsn(slru, segno, blknum, self.lsn) + .get_slru_page_at_lsn(slru, segno, blknum, self.lsn, self.ctx) .await?; if slru == SlruKind::Clog { @@ -287,7 +308,7 @@ where let relmap_img = if has_relmap_file { let img = self .timeline - .get_relmap_file(spcnode, dbnode, self.lsn) + .get_relmap_file(spcnode, dbnode, self.lsn, self.ctx) .await?; ensure!(img.len() == 512); Some(img) @@ -323,7 +344,7 @@ where if !has_relmap_file && self .timeline - .list_rels(spcnode, dbnode, self.lsn) + .list_rels(spcnode, dbnode, self.lsn, self.ctx) .await? .is_empty() { @@ -356,7 +377,10 @@ where // Extract twophase state files // async fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> { - let img = self.timeline.get_twophase_file(xid, self.lsn).await?; + let img = self + .timeline + .get_twophase_file(xid, self.lsn, self.ctx) + .await?; let mut buf = BytesMut::new(); buf.extend_from_slice(&img[..]); @@ -394,12 +418,12 @@ where let checkpoint_bytes = self .timeline - .get_checkpoint(self.lsn) + .get_checkpoint(self.lsn, self.ctx) .await .context("failed to get checkpoint bytes")?; let pg_control_bytes = self .timeline - .get_control_file(self.lsn) + .get_control_file(self.lsn, self.ctx) .await .context("failed get control bytes")?; diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index c5b0ec8935..52b0c7f2be 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -13,6 +13,7 @@ use tracing::*; use metrics::set_build_info_metric; use pageserver::{ config::{defaults::*, PageServerConf}, + context::{DownloadBehavior, RequestContext}, http, page_cache, page_service, task_mgr, task_mgr::TaskKind, task_mgr::{ @@ -328,6 +329,13 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { ); if let Some(metric_collection_endpoint) = &conf.metric_collection_endpoint { + let metrics_ctx = RequestContext::todo_child( + TaskKind::MetricsCollection, + // This task itself shouldn't download anything. + // The actual size calculation does need downloads, and + // creates a child context with the right DownloadBehavior. + DownloadBehavior::Error, + ); task_mgr::spawn( MGMT_REQUEST_RUNTIME.handle(), TaskKind::MetricsCollection, @@ -341,6 +349,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { conf.metric_collection_interval, conf.synthetic_size_calculation_interval, conf.id, + metrics_ctx, ) .instrument(info_span!("metrics_collection")) .await?; @@ -352,17 +361,34 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { // Spawn a task to listen for libpq connections. It will spawn further tasks // for each connection. We created the listener earlier already. - task_mgr::spawn( - COMPUTE_REQUEST_RUNTIME.handle(), - TaskKind::LibpqEndpointListener, - None, - None, - "libpq endpoint listener", - true, - async move { - page_service::libpq_listener_main(conf, auth, pageserver_listener, conf.auth_type).await - }, - ); + { + let libpq_ctx = RequestContext::todo_child( + TaskKind::LibpqEndpointListener, + // listener task shouldn't need to download anything. (We will + // create a separate sub-contexts for each connection, with their + // own download behavior. This context is used only to listen and + // accept connections.) + DownloadBehavior::Error, + ); + task_mgr::spawn( + COMPUTE_REQUEST_RUNTIME.handle(), + TaskKind::LibpqEndpointListener, + None, + None, + "libpq endpoint listener", + true, + async move { + page_service::libpq_listener_main( + conf, + auth, + pageserver_listener, + conf.auth_type, + libpq_ctx, + ) + .await + }, + ); + } // All started up! Now just sit and wait for shutdown signal. signals.handle(|signal| match signal { diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 51d1664e52..a3b051279d 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -158,6 +158,8 @@ pub struct PageServerConf { pub synthetic_size_calculation_interval: Duration, pub test_remote_failures: u64, + + pub ondemand_download_behavior_treat_error_as_warn: bool, } /// We do not want to store this in a PageServerConf because the latter may be logged @@ -222,6 +224,8 @@ struct PageServerConfigBuilder { synthetic_size_calculation_interval: BuilderValue, test_remote_failures: BuilderValue, + + ondemand_download_behavior_treat_error_as_warn: BuilderValue, } impl Default for PageServerConfigBuilder { @@ -267,6 +271,8 @@ impl Default for PageServerConfigBuilder { metric_collection_endpoint: Set(DEFAULT_METRIC_COLLECTION_ENDPOINT), test_remote_failures: Set(0), + + ondemand_download_behavior_treat_error_as_warn: Set(false), } } } @@ -363,6 +369,14 @@ impl PageServerConfigBuilder { self.test_remote_failures = BuilderValue::Set(fail_first); } + pub fn ondemand_download_behavior_treat_error_as_warn( + &mut self, + ondemand_download_behavior_treat_error_as_warn: bool, + ) { + self.ondemand_download_behavior_treat_error_as_warn = + BuilderValue::Set(ondemand_download_behavior_treat_error_as_warn); + } + pub fn build(self) -> anyhow::Result { Ok(PageServerConf { listen_pg_addr: self @@ -422,6 +436,11 @@ impl PageServerConfigBuilder { test_remote_failures: self .test_remote_failures .ok_or(anyhow!("missing test_remote_failuers"))?, + ondemand_download_behavior_treat_error_as_warn: self + .ondemand_download_behavior_treat_error_as_warn + .ok_or(anyhow!( + "missing ondemand_download_behavior_treat_error_as_warn" + ))?, }) } } @@ -600,6 +619,7 @@ impl PageServerConf { "synthetic_size_calculation_interval" => builder.synthetic_size_calculation_interval(parse_toml_duration(key, item)?), "test_remote_failures" => builder.test_remote_failures(parse_toml_u64(key, item)?), + "ondemand_download_behavior_treat_error_as_warn" => builder.ondemand_download_behavior_treat_error_as_warn(parse_toml_bool(key, item)?), _ => bail!("unrecognized pageserver option '{key}'"), } } @@ -724,6 +744,7 @@ impl PageServerConf { metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, synthetic_size_calculation_interval: Duration::from_secs(60), test_remote_failures: 0, + ondemand_download_behavior_treat_error_as_warn: false, } } } @@ -749,6 +770,11 @@ fn parse_toml_u64(name: &str, item: &Item) -> Result { Ok(i as u64) } +fn parse_toml_bool(name: &str, item: &Item) -> Result { + item.as_bool() + .with_context(|| format!("configure option {name} is not a bool")) +} + fn parse_toml_duration(name: &str, item: &Item) -> Result { let s = item .as_str() @@ -907,6 +933,7 @@ log_format = 'json' defaults::DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL )?, test_remote_failures: 0, + ondemand_download_behavior_treat_error_as_warn: false, }, "Correct defaults should be used when no config values are provided" ); @@ -954,6 +981,7 @@ log_format = 'json' metric_collection_endpoint: Some(Url::parse("http://localhost:80/metrics")?), synthetic_size_calculation_interval: Duration::from_secs(333), test_remote_failures: 0, + ondemand_download_behavior_treat_error_as_warn: false, }, "Should be able to parse all basic config values correctly" ); diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index c07026261d..17432234ed 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -3,6 +3,7 @@ //! and push them to a HTTP endpoint. //! Cache metrics to send only the updated ones. //! +use crate::context::{DownloadBehavior, RequestContext}; use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}; use crate::tenant::mgr; use anyhow; @@ -47,12 +48,15 @@ pub async fn collect_metrics( metric_collection_interval: Duration, synthetic_size_calculation_interval: Duration, node_id: NodeId, + ctx: RequestContext, ) -> anyhow::Result<()> { let mut ticker = tokio::time::interval(metric_collection_interval); info!("starting collect_metrics"); // spin up background worker that caclulates tenant sizes + let worker_ctx = + ctx.detached_child(TaskKind::CalculateSyntheticSize, DownloadBehavior::Download); task_mgr::spawn( BACKGROUND_RUNTIME.handle(), TaskKind::CalculateSyntheticSize, @@ -61,7 +65,7 @@ pub async fn collect_metrics( "synthetic size calculation", false, async move { - calculate_synthetic_size_worker(synthetic_size_calculation_interval) + calculate_synthetic_size_worker(synthetic_size_calculation_interval, &worker_ctx) .instrument(info_span!("synthetic_size_worker")) .await?; Ok(()) @@ -79,7 +83,7 @@ pub async fn collect_metrics( return Ok(()); }, _ = ticker.tick() => { - if let Err(err) = collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id).await + if let Err(err) = collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id, &ctx).await { error!("metrics collection failed: {err:?}"); } @@ -102,6 +106,7 @@ pub async fn collect_metrics_iteration( cached_metrics: &mut HashMap, metric_collection_endpoint: &reqwest::Url, node_id: NodeId, + ctx: &RequestContext, ) -> anyhow::Result<()> { let mut current_metrics: Vec<(PageserverConsumptionMetricsKey, u64)> = Vec::new(); trace!( @@ -137,7 +142,7 @@ pub async fn collect_metrics_iteration( timeline_written_size, )); - let (timeline_logical_size, is_exact) = timeline.get_current_logical_size()?; + let (timeline_logical_size, is_exact) = timeline.get_current_logical_size(ctx)?; // Only send timeline logical size when it is fully calculated. if is_exact { current_metrics.push(( @@ -258,6 +263,7 @@ pub async fn collect_metrics_iteration( /// Caclculate synthetic size for each active tenant pub async fn calculate_synthetic_size_worker( synthetic_size_calculation_interval: Duration, + ctx: &RequestContext, ) -> anyhow::Result<()> { info!("starting calculate_synthetic_size_worker"); @@ -280,7 +286,7 @@ pub async fn calculate_synthetic_size_worker( if let Ok(tenant) = mgr::get_tenant(tenant_id, true).await { - if let Err(e) = tenant.calculate_synthetic_size().await { + if let Err(e) = tenant.calculate_synthetic_size(ctx).await { error!("failed to calculate synthetic size for tenant {}: {}", tenant_id, e); } } diff --git a/pageserver/src/context.rs b/pageserver/src/context.rs new file mode 100644 index 0000000000..e826d28e6d --- /dev/null +++ b/pageserver/src/context.rs @@ -0,0 +1,199 @@ +//! This module defines `RequestContext`, a structure that we use throughout +//! the pageserver to propagate high-level context from places +//! that _originate_ activity down to the shared code paths at the +//! heart of the pageserver. It's inspired by Golang's `context.Context`. +//! +//! For example, in `Timeline::get(page_nr, lsn)` we need to answer the following questions: +//! 1. What high-level activity ([`TaskKind`]) needs this page? +//! We need that information as a categorical dimension for page access +//! statistics, which we, in turn, need to guide layer eviction policy design. +//! 2. How should we behave if, to produce the page image, we need to +//! on-demand download a layer file ([`DownloadBehavior`]). +//! +//! [`RequestContext`] satisfies those needs. +//! The current implementation is a small `struct` that is passed through +//! the call chain by reference. +//! +//! ### Future Work +//! +//! However, we do not intend to stop here, since there are other needs that +//! require carrying information from high to low levels of the app. +//! +//! Most importantly, **cancellation signaling** in response to +//! 1. timeouts (page_service max response time) and +//! 2. lifecycle requests (detach tenant, delete timeline). +//! +//! Related to that, there is sometimes a need to ensure that all tokio tasks spawned +//! by the transitive callees of a request have finished. The keyword here +//! is **Structured Concurrency**, and right now, we use `task_mgr` in most places, +//! `TaskHandle` in some places, and careful code review around `FuturesUnordered` +//! or `JoinSet` in other places. +//! +//! We do not yet have a systematic cancellation story in pageserver, and it is +//! pretty clear that [`RequestContext`] will be responsible for that. +//! So, the API already prepares for this role through the +//! [`RequestContext::detached_child`] and [`RequestContext::attached_child`] methods. +//! See their doc comments for details on how we will use them in the future. +//! +//! It is not clear whether or how we will enforce Structured Concurrency, and +//! what role [`RequestContext`] will play there. +//! So, the API doesn't prepare us for this topic. +//! +//! Other future uses of `RequestContext`: +//! - Communicate compute & IO priorities (user-initiated request vs. background-loop) +//! - Request IDs for distributed tracing +//! - Request/Timeline/Tenant-scoped log levels +//! +//! RequestContext might look quite different once it supports those features. +//! Likely, it will have a shape similar to Golang's `context.Context`. +//! +//! ### Why A Struct Instead Of Method Parameters +//! +//! What's typical about such information is that it needs to be passed down +//! along the call chain from high level to low level, but few of the functions +//! in the middle need to understand it. +//! Further, it is to be expected that we will need to propagate more data +//! in the future (see the earlier section on future work). +//! Hence, for functions in the middle of the call chain, we have the following +//! requirements: +//! 1. It should be easy to forward the context to callees. +//! 2. To propagate more data from high-level to low-level code, the functions in +//! the middle should not need to be modified. +//! The solution is to have a container structure ([`RequestContext`]) that +//! carries the information. Functions that don't care about what's in it +//! pass it along to callees. +//! +//! ### Why Not Task-Local Variables +//! +//! One could use task-local variables (the equivalent of thread-local variables) +//! to address the immediate needs outlined above. +//! However, we reject task-local variables because: +//! 1. they are implicit, thereby making it harder to trace the data flow in code +//! reviews and during debugging, +//! 2. they can be mutable, which enables implicit return data flow, +//! 3. they are restrictive in that code which fans out into multiple tasks, +//! or even threads, needs to carefully propagate the state. +//! +//! In contrast, information flow with [`RequestContext`] is +//! 1. always explicit, +//! 2. strictly uni-directional because RequestContext is immutable, +//! 3. tangible because a [`RequestContext`] is just a value. +//! When creating child activities, regardless of whether it's a task, +//! thread, or even an RPC to another service, the value can +//! be used like any other argument. +//! +//! The solution is that all code paths are infected with precisely one +//! [`RequestContext`] argument. Functions in the middle of the call chain +//! only need to pass it on. +use crate::task_mgr::TaskKind; + +// The main structure of this module, see module-level comment. +pub struct RequestContext { + task_kind: TaskKind, + download_behavior: DownloadBehavior, +} + +/// Desired behavior if the operation requires an on-demand download +/// to proceed. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum DownloadBehavior { + /// Download the layer file. It can take a while. + Download, + + /// Download the layer file, but print a warning to the log. This should be used + /// in code where the layer file is expected to already exist locally. + Warn, + + /// Return a PageReconstructError::NeedsDownload error + Error, +} + +impl RequestContext { + /// Create a new RequestContext that has no parent. + /// + /// The function is called `new` because, once we add children + /// to it using `detached_child` or `attached_child`, the context + /// form a tree (not implemented yet since cancellation will be + /// the first feature that requires a tree). + /// + /// # Future: Cancellation + /// + /// The only reason why a context like this one can be canceled is + /// because someone explicitly canceled it. + /// It has no parent, so it cannot inherit cancellation from there. + pub fn new(task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self { + RequestContext { + task_kind, + download_behavior, + } + } + + /// Create a detached child context for a task that may outlive `self`. + /// + /// Use this when spawning new background activity that should complete + /// even if the current request is canceled. + /// + /// # Future: Cancellation + /// + /// Cancellation of `self` will not propagate to the child context returned + /// by this method. + /// + /// # Future: Structured Concurrency + /// + /// We could add the Future as a parameter to this function, spawn it as a task, + /// and pass to the new task the child context as an argument. + /// That would be an ergonomic improvement. + /// + /// We could make new calls to this function fail if `self` is already canceled. + pub fn detached_child(&self, task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self { + self.child_impl(task_kind, download_behavior) + } + + /// Create a child of context `self` for a task that shall not outlive `self`. + /// + /// Use this when fanning-out work to other async tasks. + /// + /// # Future: Cancellation + /// + /// Cancelling a context will propagate to its attached children. + /// + /// # Future: Structured Concurrency + /// + /// We could add the Future as a parameter to this function, spawn it as a task, + /// and track its `JoinHandle` inside the `RequestContext`. + /// + /// We could then provide another method to allow waiting for all child tasks + /// to finish. + /// + /// We could make new calls to this function fail if `self` is already canceled. + /// Alternatively, we could allow the creation but not spawn the task. + /// The method to wait for child tasks would return an error, indicating + /// that the child task was not started because the context was canceled. + pub fn attached_child(&self) -> Self { + self.child_impl(self.task_kind(), self.download_behavior()) + } + + /// Use this function when you should be creating a child context using + /// [`attached_child`] or [`detached_child`], but your caller doesn't provide + /// a context and you are unwilling to change all callers to provide one. + /// + /// Before we add cancellation, we should get rid of this method. + pub fn todo_child(task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self { + Self::new(task_kind, download_behavior) + } + + fn child_impl(&self, task_kind: TaskKind, download_behavior: DownloadBehavior) -> Self { + RequestContext { + task_kind, + download_behavior, + } + } + + pub fn task_kind(&self) -> TaskKind { + self.task_kind + } + + pub fn download_behavior(&self) -> DownloadBehavior { + self.download_behavior + } +} diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index c433b695a7..541c779efd 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -12,7 +12,9 @@ use super::models::{ StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo, TimelineCreateRequest, TimelineInfo, }; +use crate::context::{DownloadBehavior, RequestContext}; use crate::pgdatadir_mapping::LsnForTimestamp; +use crate::task_mgr::TaskKind; use crate::tenant::config::TenantConfOpt; use crate::tenant::{PageReconstructError, Timeline}; use crate::{config::PageServerConf, tenant::mgr}; @@ -81,6 +83,16 @@ fn check_permission(request: &Request, tenant_id: Option) -> Res fn apierror_from_prerror(err: PageReconstructError) -> ApiError { match err { PageReconstructError::Other(err) => ApiError::InternalServerError(err), + PageReconstructError::NeedsDownload(_, _) => { + // This shouldn't happen, because we use a RequestContext that requests to + // download any missing layer files on-demand. + ApiError::InternalServerError(anyhow::anyhow!( + "would need to download remote layer file" + )) + } + PageReconstructError::Cancelled => { + ApiError::InternalServerError(anyhow::anyhow!("request was cancelled")) + } PageReconstructError::WalRedo(err) => { ApiError::InternalServerError(anyhow::Error::new(err)) } @@ -91,8 +103,9 @@ fn apierror_from_prerror(err: PageReconstructError) -> ApiError { async fn build_timeline_info( timeline: &Arc, include_non_incremental_logical_size: bool, + ctx: &RequestContext, ) -> anyhow::Result { - let mut info = build_timeline_info_common(timeline)?; + let mut info = build_timeline_info_common(timeline, ctx)?; if include_non_incremental_logical_size { // XXX we should be using spawn_ondemand_logical_size_calculation here. // Otherwise, if someone deletes the timeline / detaches the tenant while @@ -102,6 +115,7 @@ async fn build_timeline_info( .get_current_logical_size_non_incremental( info.last_record_lsn, CancellationToken::new(), + ctx, ) .await?, ); @@ -109,7 +123,10 @@ async fn build_timeline_info( Ok(info) } -fn build_timeline_info_common(timeline: &Arc) -> anyhow::Result { +fn build_timeline_info_common( + timeline: &Arc, + ctx: &RequestContext, +) -> anyhow::Result { let last_record_lsn = timeline.get_last_record_lsn(); let (wal_source_connstr, last_received_msg_lsn, last_received_msg_ts) = { let guard = timeline.last_received_wal.lock().unwrap(); @@ -129,7 +146,7 @@ fn build_timeline_info_common(timeline: &Arc) -> anyhow::Result None, lsn @ Lsn(_) => Some(lsn), }; - let current_logical_size = match timeline.get_current_logical_size() { + let current_logical_size = match timeline.get_current_logical_size(ctx) { Ok((size, _)) => Some(size), Err(err) => { error!("Timeline info creation failed to get current logical size: {err:?}"); @@ -180,6 +197,8 @@ async fn timeline_create_handler(mut request: Request) -> Result) -> Result { // Created. Construct a TimelineInfo for it. - let timeline_info = build_timeline_info_common(&new_timeline) + let timeline_info = build_timeline_info_common(&new_timeline, &ctx) .map_err(ApiError::InternalServerError)?; json_response(StatusCode::CREATED, timeline_info) } @@ -208,6 +228,8 @@ async fn timeline_list_handler(request: Request) -> Result, query_param_present(&request, "include-non-incremental-logical-size"); check_permission(&request, Some(tenant_id))?; + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); + let response_data = async { let tenant = mgr::get_tenant(tenant_id, true) .await @@ -217,7 +239,7 @@ async fn timeline_list_handler(request: Request) -> Result, let mut response_data = Vec::with_capacity(timelines.len()); for timeline in timelines { let timeline_info = - build_timeline_info(&timeline, include_non_incremental_logical_size) + build_timeline_info(&timeline, include_non_incremental_logical_size, &ctx) .await .context( "Failed to convert tenant timeline {timeline_id} into the local one: {e:?}", @@ -266,6 +288,9 @@ async fn timeline_detail_handler(request: Request) -> Result) -> Result(timeline_info) } @@ -299,12 +325,13 @@ async fn get_lsn_by_timestamp_handler(request: Request) -> Result) -> Result, let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); + info!("Handling tenant attach {tenant_id}"); let state = get_state(&request); if let Some(remote_storage) = &state.remote_storage { // FIXME: distinguish between "Tenant already exists" and other errors - mgr::attach_tenant(state.conf, tenant_id, remote_storage.clone()) + mgr::attach_tenant(state.conf, tenant_id, remote_storage.clone(), &ctx) .instrument(info_span!("tenant_attach", tenant = %tenant_id)) .await .map_err(ApiError::InternalServerError)?; @@ -346,7 +375,9 @@ async fn timeline_delete_handler(request: Request) -> Result) -> Result, A let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); + let state = get_state(&request); - mgr::load_tenant(state.conf, tenant_id, state.remote_storage.clone()) + mgr::load_tenant(state.conf, tenant_id, state.remote_storage.clone(), &ctx) .instrument(info_span!("load", tenant = %tenant_id)) .await .map_err(ApiError::InternalServerError)?; @@ -466,13 +499,14 @@ async fn tenant_size_handler(request: Request) -> Result, A false }; + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let tenant = mgr::get_tenant(tenant_id, true) .await .map_err(ApiError::InternalServerError)?; // this can be long operation let inputs = tenant - .gather_size_inputs() + .gather_size_inputs(&ctx) .await .map_err(ApiError::InternalServerError)?; @@ -521,6 +555,8 @@ fn bad_duration<'a>(field_name: &'static str, value: &'a str) -> impl 'a + Fn() async fn tenant_create_handler(mut request: Request) -> Result, ApiError> { check_permission(&request, None)?; + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); + let request_data: TenantCreateRequest = json_request(&mut request).await?; let mut tenant_conf = TenantConfOpt::default(); @@ -598,6 +634,7 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result) -> Result) -> Result) -> Result Result<()> { let mut pg_control: Option = None; @@ -69,7 +71,7 @@ pub async fn import_timeline_from_postgres_datadir( let mut file = tokio::fs::File::open(absolute_path).await?; let len = metadata.len() as usize; if let Some(control_file) = - import_file(&mut modification, relative_path, &mut file, len).await? + import_file(&mut modification, relative_path, &mut file, len, ctx).await? { pg_control = Some(control_file); } @@ -99,6 +101,7 @@ pub async fn import_timeline_from_postgres_datadir( tline, Lsn(pg_control.checkPointCopy.redo), pgdata_lsn, + ctx, ) .await?; @@ -113,6 +116,7 @@ async fn import_rel( dboid: Oid, reader: &mut (impl AsyncRead + Send + Sync + Unpin), len: usize, + ctx: &RequestContext, ) -> anyhow::Result<()> { // Does it look like a relation file? trace!("importing rel file {}", path.display()); @@ -147,7 +151,10 @@ async fn import_rel( // FIXME: use proper error type for this, instead of parsing the error message. // Or better yet, keep track of which relations we've already created // https://github.com/neondatabase/neon/issues/3309 - if let Err(e) = modification.put_rel_creation(rel, nblocks as u32).await { + if let Err(e) = modification + .put_rel_creation(rel, nblocks as u32, ctx) + .await + { if e.to_string().contains("already exists") { debug!("relation {} already exists. we must be extending it", rel); } else { @@ -182,7 +189,7 @@ async fn import_rel( // // If we process rel segments out of order, // put_rel_extend will skip the update. - modification.put_rel_extend(rel, blknum).await?; + modification.put_rel_extend(rel, blknum, ctx).await?; Ok(()) } @@ -195,6 +202,7 @@ async fn import_slru( path: &Path, reader: &mut (impl AsyncRead + Send + Sync + Unpin), len: usize, + ctx: &RequestContext, ) -> anyhow::Result<()> { info!("importing slru file {path:?}"); @@ -211,7 +219,7 @@ async fn import_slru( ensure!(nblocks <= pg_constants::SLRU_PAGES_PER_SEGMENT as usize); modification - .put_slru_segment_creation(slru, segno, nblocks as u32) + .put_slru_segment_creation(slru, segno, nblocks as u32, ctx) .await?; let mut rpageno = 0; @@ -252,15 +260,15 @@ async fn import_wal( tline: &Timeline, startpoint: Lsn, endpoint: Lsn, + ctx: &RequestContext, ) -> anyhow::Result<()> { - use std::io::Read; let mut waldecoder = WalStreamDecoder::new(startpoint, tline.pg_version); let mut segno = startpoint.segment_number(WAL_SEGMENT_SIZE); let mut offset = startpoint.segment_offset(WAL_SEGMENT_SIZE); let mut last_lsn = startpoint; - let mut walingest = WalIngest::new(tline, startpoint).await?; + let mut walingest = WalIngest::new(tline, startpoint, ctx).await?; while last_lsn <= endpoint { // FIXME: assume postgresql tli 1 for now @@ -283,6 +291,7 @@ async fn import_wal( file.seek(std::io::SeekFrom::Start(offset as u64))?; } + use std::io::Read; let nread = file.read_to_end(&mut buf)?; if nread != WAL_SEGMENT_SIZE - offset { // Maybe allow this for .partial files? @@ -297,7 +306,7 @@ async fn import_wal( while last_lsn <= endpoint { if let Some((lsn, recdata)) = waldecoder.poll_decode()? { walingest - .ingest_record(recdata, lsn, &mut modification, &mut decoded) + .ingest_record(recdata, lsn, &mut modification, &mut decoded, ctx) .await?; last_lsn = lsn; @@ -326,6 +335,7 @@ pub async fn import_basebackup_from_tar( tline: &Timeline, reader: &mut (impl AsyncRead + Send + Sync + Unpin), base_lsn: Lsn, + ctx: &RequestContext, ) -> Result<()> { info!("importing base at {base_lsn}"); let mut modification = tline.begin_modification(base_lsn); @@ -344,7 +354,7 @@ pub async fn import_basebackup_from_tar( match header.entry_type() { tokio_tar::EntryType::Regular => { if let Some(res) = - import_file(&mut modification, file_path.as_ref(), &mut entry, len).await? + import_file(&mut modification, file_path.as_ref(), &mut entry, len, ctx).await? { // We found the pg_control file. pg_control = Some(res); @@ -376,13 +386,14 @@ pub async fn import_wal_from_tar( reader: &mut (impl AsyncRead + Send + Sync + Unpin), start_lsn: Lsn, end_lsn: Lsn, + ctx: &RequestContext, ) -> Result<()> { // Set up walingest mutable state let mut waldecoder = WalStreamDecoder::new(start_lsn, tline.pg_version); let mut segno = start_lsn.segment_number(WAL_SEGMENT_SIZE); let mut offset = start_lsn.segment_offset(WAL_SEGMENT_SIZE); let mut last_lsn = start_lsn; - let mut walingest = WalIngest::new(tline, start_lsn).await?; + let mut walingest = WalIngest::new(tline, start_lsn, ctx).await?; // Ingest wal until end_lsn info!("importing wal until {}", end_lsn); @@ -431,7 +442,7 @@ pub async fn import_wal_from_tar( while last_lsn <= end_lsn { if let Some((lsn, recdata)) = waldecoder.poll_decode()? { walingest - .ingest_record(recdata, lsn, &mut modification, &mut decoded) + .ingest_record(recdata, lsn, &mut modification, &mut decoded, ctx) .await?; last_lsn = lsn; @@ -466,6 +477,7 @@ async fn import_file( file_path: &Path, reader: &mut (impl AsyncRead + Send + Sync + Unpin), len: usize, + ctx: &RequestContext, ) -> Result> { let file_name = match file_path.file_name() { Some(name) => name.to_string_lossy(), @@ -498,14 +510,16 @@ async fn import_file( } "pg_filenode.map" => { let bytes = read_all_bytes(reader).await?; - modification.put_relmap_file(spcnode, dbnode, bytes).await?; + modification + .put_relmap_file(spcnode, dbnode, bytes, ctx) + .await?; debug!("imported relmap file") } "PG_VERSION" => { debug!("ignored PG_VERSION file"); } _ => { - import_rel(modification, file_path, spcnode, dbnode, reader, len).await?; + import_rel(modification, file_path, spcnode, dbnode, reader, len, ctx).await?; debug!("imported rel creation"); } } @@ -521,38 +535,40 @@ async fn import_file( match file_name.as_ref() { "pg_filenode.map" => { let bytes = read_all_bytes(reader).await?; - modification.put_relmap_file(spcnode, dbnode, bytes).await?; + modification + .put_relmap_file(spcnode, dbnode, bytes, ctx) + .await?; debug!("imported relmap file") } "PG_VERSION" => { debug!("ignored PG_VERSION file"); } _ => { - import_rel(modification, file_path, spcnode, dbnode, reader, len).await?; + import_rel(modification, file_path, spcnode, dbnode, reader, len, ctx).await?; debug!("imported rel creation"); } } } else if file_path.starts_with("pg_xact") { let slru = SlruKind::Clog; - import_slru(modification, slru, file_path, reader, len).await?; + import_slru(modification, slru, file_path, reader, len, ctx).await?; debug!("imported clog slru"); } else if file_path.starts_with("pg_multixact/offsets") { let slru = SlruKind::MultiXactOffsets; - import_slru(modification, slru, file_path, reader, len).await?; + import_slru(modification, slru, file_path, reader, len, ctx).await?; debug!("imported multixact offsets slru"); } else if file_path.starts_with("pg_multixact/members") { let slru = SlruKind::MultiXactMembers; - import_slru(modification, slru, file_path, reader, len).await?; + import_slru(modification, slru, file_path, reader, len, ctx).await?; debug!("imported multixact members slru"); } else if file_path.starts_with("pg_twophase") { let xid = u32::from_str_radix(file_name.as_ref(), 16)?; let bytes = read_all_bytes(reader).await?; modification - .put_twophase_file(xid, Bytes::copy_from_slice(&bytes[..])) + .put_twophase_file(xid, Bytes::copy_from_slice(&bytes[..]), ctx) .await?; debug!("imported twophase file"); } else if file_path.starts_with("pg_wal") { diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 91cde477ad..3ea2165da9 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -2,6 +2,7 @@ mod auth; pub mod basebackup; pub mod config; pub mod consumption_metrics; +pub mod context; pub mod http; pub mod import_datadir; pub mod keyspace; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 0154867dae..878928ae06 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -43,6 +43,7 @@ use utils::{ use crate::auth::check_permission; use crate::basebackup; use crate::config::PageServerConf; +use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir::import_wal_from_tar; use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME}; use crate::task_mgr; @@ -123,6 +124,7 @@ pub async fn libpq_listener_main( auth: Option>, listener: TcpListener, auth_type: AuthType, + listener_ctx: RequestContext, ) -> anyhow::Result<()> { listener.set_nonblocking(true)?; let tokio_listener = tokio::net::TcpListener::from_std(listener)?; @@ -146,6 +148,9 @@ pub async fn libpq_listener_main( debug!("accepted connection from {}", peer_addr); let local_auth = auth.clone(); + let connection_ctx = listener_ctx + .detached_child(TaskKind::PageRequestHandler, DownloadBehavior::Download); + // PageRequestHandler tasks are not associated with any particular // timeline in the task manager. In practice most connections will // only deal with a particular timeline, but we don't know which one @@ -157,7 +162,7 @@ pub async fn libpq_listener_main( None, "serving compute connection task", false, - page_service_conn_main(conf, local_auth, socket, auth_type), + page_service_conn_main(conf, local_auth, socket, auth_type, connection_ctx), ); } Err(err) => { @@ -177,6 +182,7 @@ async fn page_service_conn_main( auth: Option>, socket: tokio::net::TcpStream, auth_type: AuthType, + connection_ctx: RequestContext, ) -> anyhow::Result<()> { // Immediately increment the gauge, then create a job to decrement it on task exit. // One of the pros of `defer!` is that this will *most probably* @@ -191,7 +197,11 @@ async fn page_service_conn_main( .set_nodelay(true) .context("could not set TCP_NODELAY")?; - let mut conn_handler = PageServerHandler::new(conf, auth); + // XXX: pgbackend.run() should take the connection_ctx, + // and create a child per-query context when it invokes process_query. + // But it's in a shared crate, so, we store connection_ctx inside PageServerHandler + // and create the per-query context in process_query ourselves. + let mut conn_handler = PageServerHandler::new(conf, auth, connection_ctx); let pgbackend = PostgresBackend::new(socket, auth_type, None)?; match pgbackend @@ -251,30 +261,42 @@ struct PageServerHandler { _conf: &'static PageServerConf, auth: Option>, claims: Option, + + /// The context created for the lifetime of the connection + /// services by this PageServerHandler. + /// For each query received over the connection, + /// `process_query` creates a child context from this one. + connection_ctx: RequestContext, } impl PageServerHandler { - pub fn new(conf: &'static PageServerConf, auth: Option>) -> Self { + pub fn new( + conf: &'static PageServerConf, + auth: Option>, + connection_ctx: RequestContext, + ) -> Self { PageServerHandler { _conf: conf, auth, claims: None, + connection_ctx, } } - #[instrument(skip(self, pgb))] + #[instrument(skip(self, pgb, ctx))] async fn handle_pagerequests( &self, pgb: &mut PostgresBackend, tenant_id: TenantId, timeline_id: TimelineId, + ctx: RequestContext, ) -> anyhow::Result<()> { // NOTE: pagerequests handler exits when connection is closed, // so there is no need to reset the association task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); // Make request tracer if needed - let tenant = get_active_tenant_with_timeout(tenant_id).await?; + let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; let mut tracer = if tenant.get_trace_read_requests() { let connection_id = ConnectionId::generate(); let path = tenant @@ -325,22 +347,27 @@ impl PageServerHandler { let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader())?; + // TODO: We could create a new per-request context here, with unique ID. + // Currently we use the same per-timeline context for all requests + let response = match neon_fe_msg { PagestreamFeMessage::Exists(req) => { let _timer = metrics.get_rel_exists.start_timer(); - self.handle_get_rel_exists_request(&timeline, &req).await + self.handle_get_rel_exists_request(&timeline, &req, &ctx) + .await } PagestreamFeMessage::Nblocks(req) => { let _timer = metrics.get_rel_size.start_timer(); - self.handle_get_nblocks_request(&timeline, &req).await + self.handle_get_nblocks_request(&timeline, &req, &ctx).await } PagestreamFeMessage::GetPage(req) => { let _timer = metrics.get_page_at_lsn.start_timer(); - self.handle_get_page_at_lsn_request(&timeline, &req).await + self.handle_get_page_at_lsn_request(&timeline, &req, &ctx) + .await } PagestreamFeMessage::DbSize(req) => { let _timer = metrics.get_db_size.start_timer(); - self.handle_db_size_request(&timeline, &req).await + self.handle_db_size_request(&timeline, &req, &ctx).await } }; @@ -359,7 +386,8 @@ impl PageServerHandler { Ok(()) } - #[instrument(skip(self, pgb))] + #[allow(clippy::too_many_arguments)] + #[instrument(skip(self, pgb, ctx))] async fn handle_import_basebackup( &self, pgb: &mut PostgresBackend, @@ -368,12 +396,13 @@ impl PageServerHandler { base_lsn: Lsn, _end_lsn: Lsn, pg_version: u32, + ctx: RequestContext, ) -> Result<(), QueryError> { task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); // Create empty timeline info!("creating new timeline"); - let tenant = get_active_tenant_with_timeout(tenant_id).await?; - let timeline = tenant.create_empty_timeline(timeline_id, base_lsn, pg_version)?; + let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let timeline = tenant.create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx)?; // TODO mark timeline as not ready until it reaches end_lsn. // We might have some wal to import as well, and we should prevent compute @@ -392,7 +421,7 @@ impl PageServerHandler { let mut copyin_stream = Box::pin(copyin_stream(pgb)); timeline - .import_basebackup_from_tar(&mut copyin_stream, base_lsn) + .import_basebackup_from_tar(&mut copyin_stream, base_lsn, &ctx) .await?; // Drain the rest of the Copy data @@ -414,7 +443,7 @@ impl PageServerHandler { Ok(()) } - #[instrument(skip(self, pgb))] + #[instrument(skip(self, pgb, ctx))] async fn handle_import_wal( &self, pgb: &mut PostgresBackend, @@ -422,10 +451,11 @@ impl PageServerHandler { timeline_id: TimelineId, start_lsn: Lsn, end_lsn: Lsn, + ctx: RequestContext, ) -> Result<(), QueryError> { task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); - let timeline = get_active_tenant_timeline(tenant_id, timeline_id).await?; + let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; let last_record_lsn = timeline.get_last_record_lsn(); if last_record_lsn != start_lsn { return Err(QueryError::Other( @@ -442,7 +472,7 @@ impl PageServerHandler { pgb.flush().await?; let mut copyin_stream = Box::pin(copyin_stream(pgb)); let mut reader = tokio_util::io::StreamReader::new(&mut copyin_stream); - import_wal_from_tar(&timeline, &mut reader, start_lsn, end_lsn).await?; + import_wal_from_tar(&timeline, &mut reader, start_lsn, end_lsn, &ctx).await?; info!("wal import complete"); // Drain the rest of the Copy data @@ -488,6 +518,7 @@ impl PageServerHandler { mut lsn: Lsn, latest: bool, latest_gc_cutoff_lsn: &RcuReadGuard, + ctx: &RequestContext, ) -> anyhow::Result { if latest { // Latest page version was requested. If LSN is given, it is a hint @@ -511,7 +542,7 @@ impl PageServerHandler { if lsn <= last_record_lsn { lsn = last_record_lsn; } else { - timeline.wait_lsn(lsn).await?; + timeline.wait_lsn(lsn, ctx).await?; // Since we waited for 'lsn' to arrive, that is now the last // record LSN. (Or close enough for our purposes; the // last-record LSN can advance immediately after we return @@ -521,7 +552,7 @@ impl PageServerHandler { if lsn == Lsn(0) { anyhow::bail!("invalid LSN(0) in request"); } - timeline.wait_lsn(lsn).await?; + timeline.wait_lsn(lsn, ctx).await?; } anyhow::ensure!( lsn >= **latest_gc_cutoff_lsn, @@ -531,52 +562,60 @@ impl PageServerHandler { Ok(lsn) } - #[instrument(skip(self, timeline, req), fields(rel = %req.rel, req_lsn = %req.lsn))] + #[instrument(skip(self, timeline, req, ctx), fields(rel = %req.rel, req_lsn = %req.lsn))] async fn handle_get_rel_exists_request( &self, timeline: &Timeline, req: &PagestreamExistsRequest, + ctx: &RequestContext, ) -> anyhow::Result { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); - let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn) - .await?; + let lsn = + Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn, ctx) + .await?; - let exists = timeline.get_rel_exists(req.rel, lsn, req.latest).await?; + let exists = timeline + .get_rel_exists(req.rel, lsn, req.latest, ctx) + .await?; Ok(PagestreamBeMessage::Exists(PagestreamExistsResponse { exists, })) } - #[instrument(skip(self, timeline, req), fields(rel = %req.rel, req_lsn = %req.lsn))] + #[instrument(skip(self, timeline, req, ctx), fields(rel = %req.rel, req_lsn = %req.lsn))] async fn handle_get_nblocks_request( &self, timeline: &Timeline, req: &PagestreamNblocksRequest, + ctx: &RequestContext, ) -> anyhow::Result { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); - let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn) - .await?; + let lsn = + Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn, ctx) + .await?; - let n_blocks = timeline.get_rel_size(req.rel, lsn, req.latest).await?; + let n_blocks = timeline.get_rel_size(req.rel, lsn, req.latest, ctx).await?; Ok(PagestreamBeMessage::Nblocks(PagestreamNblocksResponse { n_blocks, })) } - #[instrument(skip(self, timeline, req), fields(dbnode = %req.dbnode, req_lsn = %req.lsn))] + #[instrument(skip(self, timeline, req, ctx), fields(dbnode = %req.dbnode, req_lsn = %req.lsn))] async fn handle_db_size_request( &self, timeline: &Timeline, req: &PagestreamDbSizeRequest, + ctx: &RequestContext, ) -> anyhow::Result { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); - let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn) - .await?; + let lsn = + Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn, ctx) + .await?; let total_blocks = timeline - .get_db_size(DEFAULTTABLESPACE_OID, req.dbnode, lsn, req.latest) + .get_db_size(DEFAULTTABLESPACE_OID, req.dbnode, lsn, req.latest, ctx) .await?; let db_size = total_blocks as i64 * BLCKSZ as i64; @@ -585,15 +624,17 @@ impl PageServerHandler { })) } - #[instrument(skip(self, timeline, req), fields(rel = %req.rel, blkno = %req.blkno, req_lsn = %req.lsn))] + #[instrument(skip(self, timeline, req, ctx), fields(rel = %req.rel, blkno = %req.blkno, req_lsn = %req.lsn))] async fn handle_get_page_at_lsn_request( &self, timeline: &Timeline, req: &PagestreamGetPageRequest, + ctx: &RequestContext, ) -> anyhow::Result { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); - let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn) - .await?; + let lsn = + Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn, ctx) + .await?; /* // Add a 1s delay to some requests. The delay helps the requests to // hit the race condition from github issue #1047 more easily. @@ -604,7 +645,7 @@ impl PageServerHandler { */ let page = timeline - .get_rel_page_at_lsn(req.rel, req.blkno, lsn, req.latest) + .get_rel_page_at_lsn(req.rel, req.blkno, lsn, req.latest, ctx) .await?; Ok(PagestreamBeMessage::GetPage(PagestreamGetPageResponse { @@ -612,23 +653,25 @@ impl PageServerHandler { })) } - #[instrument(skip(self, pgb))] + #[allow(clippy::too_many_arguments)] + #[instrument(skip(self, pgb, ctx))] async fn handle_basebackup_request( - &self, + &mut self, pgb: &mut PostgresBackend, tenant_id: TenantId, timeline_id: TimelineId, lsn: Option, prev_lsn: Option, full_backup: bool, + ctx: RequestContext, ) -> anyhow::Result<()> { // check that the timeline exists - let timeline = get_active_tenant_timeline(tenant_id, timeline_id).await?; + let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); if let Some(lsn) = lsn { // Backup was requested at a particular LSN. Wait for it to arrive. info!("waiting for {}", lsn); - timeline.wait_lsn(lsn).await?; + timeline.wait_lsn(lsn, &ctx).await?; timeline .check_lsn_is_in_scope(lsn, &latest_gc_cutoff_lsn) .context("invalid basebackup lsn")?; @@ -641,8 +684,15 @@ impl PageServerHandler { // Send a tarball of the latest layer on the timeline { let mut writer = pgb.copyout_writer(); - basebackup::send_basebackup_tarball(&mut writer, &timeline, lsn, prev_lsn, full_backup) - .await?; + basebackup::send_basebackup_tarball( + &mut writer, + &timeline, + lsn, + prev_lsn, + full_backup, + &ctx, + ) + .await?; } pgb.write_message(&BeMessage::CopyDone)?; @@ -713,6 +763,7 @@ impl postgres_backend_async::Handler for PageServerHandler { pgb: &mut PostgresBackend, query_string: &str, ) -> Result<(), QueryError> { + let ctx = self.connection_ctx.attached_child(); debug!("process query {query_string:?}"); if query_string.starts_with("pagestream ") { @@ -730,7 +781,7 @@ impl postgres_backend_async::Handler for PageServerHandler { self.check_permission(Some(tenant_id))?; - self.handle_pagerequests(pgb, tenant_id, timeline_id) + self.handle_pagerequests(pgb, tenant_id, timeline_id, ctx) .await?; } else if query_string.starts_with("basebackup ") { let (_, params_raw) = query_string.split_at("basebackup ".len()); @@ -759,7 +810,7 @@ impl postgres_backend_async::Handler for PageServerHandler { }; // Check that the timeline exists - self.handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, None, false) + self.handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, None, false, ctx) .await?; pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?; } @@ -780,7 +831,7 @@ impl postgres_backend_async::Handler for PageServerHandler { .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?; self.check_permission(Some(tenant_id))?; - let timeline = get_active_tenant_timeline(tenant_id, timeline_id).await?; + let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; let end_of_timeline = timeline.get_last_record_rlsn(); @@ -831,7 +882,7 @@ impl postgres_backend_async::Handler for PageServerHandler { self.check_permission(Some(tenant_id))?; // Check that the timeline exists - self.handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, prev_lsn, true) + self.handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, prev_lsn, true, ctx) .await?; pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?; } else if query_string.starts_with("import basebackup ") { @@ -874,6 +925,7 @@ impl postgres_backend_async::Handler for PageServerHandler { base_lsn, end_lsn, pg_version, + ctx, ) .await { @@ -910,7 +962,7 @@ impl postgres_backend_async::Handler for PageServerHandler { self.check_permission(Some(tenant_id))?; match self - .handle_import_wal(pgb, tenant_id, timeline_id, start_lsn, end_lsn) + .handle_import_wal(pgb, tenant_id, timeline_id, start_lsn, end_lsn, ctx) .await { Ok(()) => pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?, @@ -940,7 +992,7 @@ impl postgres_backend_async::Handler for PageServerHandler { self.check_permission(Some(tenant_id))?; - let tenant = get_active_tenant_with_timeout(tenant_id).await?; + let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; pgb.write_message(&BeMessage::RowDescription(&[ RowDescriptor::int8_col(b"checkpoint_distance"), RowDescriptor::int8_col(b"checkpoint_timeout"), @@ -1017,6 +1069,7 @@ impl From for QueryError { /// all tenants are still loading. async fn get_active_tenant_with_timeout( tenant_id: TenantId, + _ctx: &RequestContext, /* require get a context to support cancellation in the future */ ) -> Result, GetActiveTenantError> { let tenant = mgr::get_tenant(tenant_id, false).await?; let wait_time = Duration::from_secs(30); @@ -1042,8 +1095,9 @@ async fn get_active_tenant_with_timeout( async fn get_active_tenant_timeline( tenant_id: TenantId, timeline_id: TimelineId, + ctx: &RequestContext, ) -> Result, GetActiveTenantError> { - let tenant = get_active_tenant_with_timeout(tenant_id).await?; + let tenant = get_active_tenant_with_timeout(tenant_id, ctx).await?; let timeline = tenant.get_timeline(timeline_id, true)?; Ok(timeline) } diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 31ae74ded5..6eea023af1 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -7,6 +7,7 @@ //! Clarify that) //! use super::tenant::{PageReconstructError, Timeline}; +use crate::context::RequestContext; use crate::keyspace::{KeySpace, KeySpaceAccum}; use crate::repository::*; use crate::walrecord::NeonWalRecord; @@ -97,6 +98,7 @@ impl Timeline { blknum: BlockNumber, lsn: Lsn, latest: bool, + ctx: &RequestContext, ) -> Result { if tag.relnode == 0 { return Err(PageReconstructError::Other(anyhow::anyhow!( @@ -104,7 +106,7 @@ impl Timeline { ))); } - let nblocks = self.get_rel_size(tag, lsn, latest).await?; + let nblocks = self.get_rel_size(tag, lsn, latest, ctx).await?; if blknum >= nblocks { debug!( "read beyond EOF at {} blk {} at {}, size is {}: returning all-zeros page", @@ -114,7 +116,7 @@ impl Timeline { } let key = rel_block_to_key(tag, blknum); - self.get(key, lsn).await + self.get(key, lsn, ctx).await } // Get size of a database in blocks @@ -124,13 +126,14 @@ impl Timeline { dbnode: Oid, lsn: Lsn, latest: bool, + ctx: &RequestContext, ) -> Result { let mut total_blocks = 0; - let rels = self.list_rels(spcnode, dbnode, lsn).await?; + let rels = self.list_rels(spcnode, dbnode, lsn, ctx).await?; for rel in rels { - let n_blocks = self.get_rel_size(rel, lsn, latest).await?; + let n_blocks = self.get_rel_size(rel, lsn, latest, ctx).await?; total_blocks += n_blocks as usize; } Ok(total_blocks) @@ -142,6 +145,7 @@ impl Timeline { tag: RelTag, lsn: Lsn, latest: bool, + ctx: &RequestContext, ) -> Result { if tag.relnode == 0 { return Err(PageReconstructError::Other(anyhow::anyhow!( @@ -154,7 +158,7 @@ impl Timeline { } if (tag.forknum == FSM_FORKNUM || tag.forknum == VISIBILITYMAP_FORKNUM) - && !self.get_rel_exists(tag, lsn, latest).await? + && !self.get_rel_exists(tag, lsn, latest, ctx).await? { // FIXME: Postgres sometimes calls smgrcreate() to create // FSM, and smgrnblocks() on it immediately afterwards, @@ -164,7 +168,7 @@ impl Timeline { } let key = rel_size_to_key(tag); - let mut buf = self.get(key, lsn).await?; + let mut buf = self.get(key, lsn, ctx).await?; let nblocks = buf.get_u32_le(); if latest { @@ -186,6 +190,7 @@ impl Timeline { tag: RelTag, lsn: Lsn, _latest: bool, + ctx: &RequestContext, ) -> Result { if tag.relnode == 0 { return Err(PageReconstructError::Other(anyhow::anyhow!( @@ -199,7 +204,7 @@ impl Timeline { } // fetch directory listing let key = rel_dir_to_key(tag.spcnode, tag.dbnode); - let buf = self.get(key, lsn).await?; + let buf = self.get(key, lsn, ctx).await?; match RelDirectory::des(&buf).context("deserialization failure") { Ok(dir) => { @@ -216,10 +221,11 @@ impl Timeline { spcnode: Oid, dbnode: Oid, lsn: Lsn, + ctx: &RequestContext, ) -> Result, PageReconstructError> { // fetch directory listing let key = rel_dir_to_key(spcnode, dbnode); - let buf = self.get(key, lsn).await?; + let buf = self.get(key, lsn, ctx).await?; match RelDirectory::des(&buf).context("deserialization failure") { Ok(dir) => { @@ -244,9 +250,10 @@ impl Timeline { segno: u32, blknum: BlockNumber, lsn: Lsn, + ctx: &RequestContext, ) -> Result { let key = slru_block_to_key(kind, segno, blknum); - self.get(key, lsn).await + self.get(key, lsn, ctx).await } /// Get size of an SLRU segment @@ -255,9 +262,10 @@ impl Timeline { kind: SlruKind, segno: u32, lsn: Lsn, + ctx: &RequestContext, ) -> Result { let key = slru_segment_size_to_key(kind, segno); - let mut buf = self.get(key, lsn).await?; + let mut buf = self.get(key, lsn, ctx).await?; Ok(buf.get_u32_le()) } @@ -267,10 +275,11 @@ impl Timeline { kind: SlruKind, segno: u32, lsn: Lsn, + ctx: &RequestContext, ) -> Result { // fetch directory listing let key = slru_dir_to_key(kind); - let buf = self.get(key, lsn).await?; + let buf = self.get(key, lsn, ctx).await?; match SlruSegmentDirectory::des(&buf).context("deserialization failure") { Ok(dir) => { @@ -291,6 +300,7 @@ impl Timeline { pub async fn find_lsn_for_timestamp( &self, search_timestamp: TimestampTz, + ctx: &RequestContext, ) -> Result { let gc_cutoff_lsn_guard = self.get_latest_gc_cutoff_lsn(); let min_lsn = *gc_cutoff_lsn_guard; @@ -313,6 +323,7 @@ impl Timeline { Lsn(mid * 8), &mut found_smaller, &mut found_larger, + ctx, ) .await?; @@ -362,14 +373,18 @@ impl Timeline { probe_lsn: Lsn, found_smaller: &mut bool, found_larger: &mut bool, + ctx: &RequestContext, ) -> Result { - for segno in self.list_slru_segments(SlruKind::Clog, probe_lsn).await? { + for segno in self + .list_slru_segments(SlruKind::Clog, probe_lsn, ctx) + .await? + { let nblocks = self - .get_slru_segment_size(SlruKind::Clog, segno, probe_lsn) + .get_slru_segment_size(SlruKind::Clog, segno, probe_lsn, ctx) .await?; for blknum in (0..nblocks).rev() { let clog_page = self - .get_slru_page_at_lsn(SlruKind::Clog, segno, blknum, probe_lsn) + .get_slru_page_at_lsn(SlruKind::Clog, segno, blknum, probe_lsn, ctx) .await?; if clog_page.len() == BLCKSZ as usize + 8 { @@ -394,11 +409,12 @@ impl Timeline { &self, kind: SlruKind, lsn: Lsn, + ctx: &RequestContext, ) -> Result, PageReconstructError> { // fetch directory entry let key = slru_dir_to_key(kind); - let buf = self.get(key, lsn).await?; + let buf = self.get(key, lsn, ctx).await?; match SlruSegmentDirectory::des(&buf).context("deserialization failure") { Ok(dir) => Ok(dir.segments), Err(e) => Err(PageReconstructError::from(e)), @@ -410,18 +426,21 @@ impl Timeline { spcnode: Oid, dbnode: Oid, lsn: Lsn, + ctx: &RequestContext, ) -> Result { let key = relmap_file_key(spcnode, dbnode); - self.get(key, lsn).await + let buf = self.get(key, lsn, ctx).await?; + Ok(buf) } pub async fn list_dbdirs( &self, lsn: Lsn, + ctx: &RequestContext, ) -> Result, PageReconstructError> { // fetch directory entry - let buf = self.get(DBDIR_KEY, lsn).await?; + let buf = self.get(DBDIR_KEY, lsn, ctx).await?; match DbDirectory::des(&buf).context("deserialization failure") { Ok(dir) => Ok(dir.dbdirs), @@ -433,18 +452,20 @@ impl Timeline { &self, xid: TransactionId, lsn: Lsn, + ctx: &RequestContext, ) -> Result { let key = twophase_file_key(xid); - let buf = self.get(key, lsn).await?; + let buf = self.get(key, lsn, ctx).await?; Ok(buf) } pub async fn list_twophase_files( &self, lsn: Lsn, + ctx: &RequestContext, ) -> Result, PageReconstructError> { // fetch directory entry - let buf = self.get(TWOPHASEDIR_KEY, lsn).await?; + let buf = self.get(TWOPHASEDIR_KEY, lsn, ctx).await?; match TwoPhaseDirectory::des(&buf).context("deserialization failure") { Ok(dir) => Ok(dir.xids), @@ -452,12 +473,20 @@ impl Timeline { } } - pub async fn get_control_file(&self, lsn: Lsn) -> Result { - self.get(CONTROLFILE_KEY, lsn).await + pub async fn get_control_file( + &self, + lsn: Lsn, + ctx: &RequestContext, + ) -> Result { + self.get(CONTROLFILE_KEY, lsn, ctx).await } - pub async fn get_checkpoint(&self, lsn: Lsn) -> Result { - self.get(CHECKPOINT_KEY, lsn).await + pub async fn get_checkpoint( + &self, + lsn: Lsn, + ctx: &RequestContext, + ) -> Result { + self.get(CHECKPOINT_KEY, lsn, ctx).await } /// Does the same as get_current_logical_size but counted on demand. @@ -469,15 +498,16 @@ impl Timeline { &self, lsn: Lsn, cancel: CancellationToken, + ctx: &RequestContext, ) -> Result { // Fetch list of database dirs and iterate them - let buf = self.get(DBDIR_KEY, lsn).await.context("read dbdir")?; + let buf = self.get(DBDIR_KEY, lsn, ctx).await.context("read dbdir")?; let dbdir = DbDirectory::des(&buf).context("deserialize db directory")?; let mut total_size: u64 = 0; for (spcnode, dbnode) in dbdir.dbdirs.keys() { for rel in self - .list_rels(*spcnode, *dbnode, lsn) + .list_rels(*spcnode, *dbnode, lsn, ctx) .await .context("list rels")? { @@ -486,7 +516,7 @@ impl Timeline { } let relsize_key = rel_size_to_key(rel); let mut buf = self - .get(relsize_key, lsn) + .get(relsize_key, lsn, ctx) .await .with_context(|| format!("read relation size of {rel:?}"))?; let relsize = buf.get_u32_le(); @@ -501,7 +531,11 @@ impl Timeline { /// Get a KeySpace that covers all the Keys that are in use at the given LSN. /// Anything that's not listed maybe removed from the underlying storage (from /// that LSN forwards). - pub async fn collect_keyspace(&self, lsn: Lsn) -> anyhow::Result { + pub async fn collect_keyspace( + &self, + lsn: Lsn, + ctx: &RequestContext, + ) -> anyhow::Result { // Iterate through key ranges, greedily packing them into partitions let mut result = KeySpaceAccum::new(); @@ -509,7 +543,7 @@ impl Timeline { result.add_key(DBDIR_KEY); // Fetch list of database dirs and iterate them - let buf = self.get(DBDIR_KEY, lsn).await?; + let buf = self.get(DBDIR_KEY, lsn, ctx).await?; let dbdir = DbDirectory::des(&buf).context("deserialization failure")?; let mut dbs: Vec<(Oid, Oid)> = dbdir.dbdirs.keys().cloned().collect(); @@ -519,14 +553,14 @@ impl Timeline { result.add_key(rel_dir_to_key(spcnode, dbnode)); let mut rels: Vec = self - .list_rels(spcnode, dbnode, lsn) + .list_rels(spcnode, dbnode, lsn, ctx) .await? .into_iter() .collect(); rels.sort_unstable(); for rel in rels { let relsize_key = rel_size_to_key(rel); - let mut buf = self.get(relsize_key, lsn).await?; + let mut buf = self.get(relsize_key, lsn, ctx).await?; let relsize = buf.get_u32_le(); result.add_range(rel_block_to_key(rel, 0)..rel_block_to_key(rel, relsize)); @@ -542,13 +576,13 @@ impl Timeline { ] { let slrudir_key = slru_dir_to_key(kind); result.add_key(slrudir_key); - let buf = self.get(slrudir_key, lsn).await?; + let buf = self.get(slrudir_key, lsn, ctx).await?; let dir = SlruSegmentDirectory::des(&buf).context("deserialization failure")?; let mut segments: Vec = dir.segments.iter().cloned().collect(); segments.sort_unstable(); for segno in segments { let segsize_key = slru_segment_size_to_key(kind, segno); - let mut buf = self.get(segsize_key, lsn).await?; + let mut buf = self.get(segsize_key, lsn, ctx).await?; let segsize = buf.get_u32_le(); result.add_range( @@ -560,7 +594,7 @@ impl Timeline { // Then pg_twophase result.add_key(TWOPHASEDIR_KEY); - let buf = self.get(TWOPHASEDIR_KEY, lsn).await?; + let buf = self.get(TWOPHASEDIR_KEY, lsn, ctx).await?; let twophase_dir = TwoPhaseDirectory::des(&buf).context("deserialization failure")?; let mut xids: Vec = twophase_dir.xids.iter().cloned().collect(); xids.sort_unstable(); @@ -723,9 +757,10 @@ impl<'a> DatadirModification<'a> { spcnode: Oid, dbnode: Oid, img: Bytes, + ctx: &RequestContext, ) -> anyhow::Result<()> { // Add it to the directory (if it doesn't exist already) - let buf = self.get(DBDIR_KEY).await?; + let buf = self.get(DBDIR_KEY, ctx).await?; let mut dbdir = DbDirectory::des(&buf)?; let r = dbdir.dbdirs.insert((spcnode, dbnode), true); @@ -755,9 +790,10 @@ impl<'a> DatadirModification<'a> { &mut self, xid: TransactionId, img: Bytes, + ctx: &RequestContext, ) -> anyhow::Result<()> { // Add it to the directory entry - let buf = self.get(TWOPHASEDIR_KEY).await?; + let buf = self.get(TWOPHASEDIR_KEY, ctx).await?; let mut dir = TwoPhaseDirectory::des(&buf)?; if !dir.xids.insert(xid) { anyhow::bail!("twophase file for xid {} already exists", xid); @@ -781,16 +817,21 @@ impl<'a> DatadirModification<'a> { Ok(()) } - pub async fn drop_dbdir(&mut self, spcnode: Oid, dbnode: Oid) -> anyhow::Result<()> { + pub async fn drop_dbdir( + &mut self, + spcnode: Oid, + dbnode: Oid, + ctx: &RequestContext, + ) -> anyhow::Result<()> { let req_lsn = self.tline.get_last_record_lsn(); let total_blocks = self .tline - .get_db_size(spcnode, dbnode, req_lsn, true) + .get_db_size(spcnode, dbnode, req_lsn, true, ctx) .await?; // Remove entry from dbdir - let buf = self.get(DBDIR_KEY).await?; + let buf = self.get(DBDIR_KEY, ctx).await?; let mut dir = DbDirectory::des(&buf)?; if dir.dbdirs.remove(&(spcnode, dbnode)).is_some() { let buf = DbDirectory::ser(&dir)?; @@ -817,11 +858,12 @@ impl<'a> DatadirModification<'a> { &mut self, rel: RelTag, nblocks: BlockNumber, + ctx: &RequestContext, ) -> anyhow::Result<()> { anyhow::ensure!(rel.relnode != 0, "invalid relnode"); // It's possible that this is the first rel for this db in this // tablespace. Create the reldir entry for it if so. - let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY).await?)?; + let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY, ctx).await?)?; let rel_dir_key = rel_dir_to_key(rel.spcnode, rel.dbnode); let mut rel_dir = if dbdir.dbdirs.get(&(rel.spcnode, rel.dbnode)).is_none() { // Didn't exist. Update dbdir @@ -833,7 +875,7 @@ impl<'a> DatadirModification<'a> { RelDirectory::default() } else { // reldir already exists, fetch it - RelDirectory::des(&self.get(rel_dir_key).await?)? + RelDirectory::des(&self.get(rel_dir_key, ctx).await?)? }; // Add the new relation to the rel directory entry, and write it back @@ -865,13 +907,14 @@ impl<'a> DatadirModification<'a> { &mut self, rel: RelTag, nblocks: BlockNumber, + ctx: &RequestContext, ) -> anyhow::Result<()> { anyhow::ensure!(rel.relnode != 0, "invalid relnode"); let last_lsn = self.tline.get_last_record_lsn(); - if self.tline.get_rel_exists(rel, last_lsn, true).await? { + if self.tline.get_rel_exists(rel, last_lsn, true, ctx).await? { let size_key = rel_size_to_key(rel); // Fetch the old size first - let old_size = self.get(size_key).await?.get_u32_le(); + let old_size = self.get(size_key, ctx).await?.get_u32_le(); // Update the entry with the new size. let buf = nblocks.to_le_bytes(); @@ -895,12 +938,13 @@ impl<'a> DatadirModification<'a> { &mut self, rel: RelTag, nblocks: BlockNumber, + ctx: &RequestContext, ) -> anyhow::Result<()> { anyhow::ensure!(rel.relnode != 0, "invalid relnode"); // Put size let size_key = rel_size_to_key(rel); - let old_size = self.get(size_key).await?.get_u32_le(); + let old_size = self.get(size_key, ctx).await?.get_u32_le(); // only extend relation here. never decrease the size if nblocks > old_size { @@ -916,12 +960,12 @@ impl<'a> DatadirModification<'a> { } /// Drop a relation. - pub async fn put_rel_drop(&mut self, rel: RelTag) -> anyhow::Result<()> { + pub async fn put_rel_drop(&mut self, rel: RelTag, ctx: &RequestContext) -> anyhow::Result<()> { anyhow::ensure!(rel.relnode != 0, "invalid relnode"); // Remove it from the directory entry let dir_key = rel_dir_to_key(rel.spcnode, rel.dbnode); - let buf = self.get(dir_key).await?; + let buf = self.get(dir_key, ctx).await?; let mut dir = RelDirectory::des(&buf)?; if dir.rels.remove(&(rel.relnode, rel.forknum)) { @@ -932,7 +976,7 @@ impl<'a> DatadirModification<'a> { // update logical size let size_key = rel_size_to_key(rel); - let old_size = self.get(size_key).await?.get_u32_le(); + let old_size = self.get(size_key, ctx).await?.get_u32_le(); self.pending_nblocks -= old_size as i64; // Remove enty from relation size cache @@ -949,10 +993,11 @@ impl<'a> DatadirModification<'a> { kind: SlruKind, segno: u32, nblocks: BlockNumber, + ctx: &RequestContext, ) -> anyhow::Result<()> { // Add it to the directory entry let dir_key = slru_dir_to_key(kind); - let buf = self.get(dir_key).await?; + let buf = self.get(dir_key, ctx).await?; let mut dir = SlruSegmentDirectory::des(&buf)?; if !dir.segments.insert(segno) { @@ -988,10 +1033,15 @@ impl<'a> DatadirModification<'a> { } /// This method is used for marking truncated SLRU files - pub async fn drop_slru_segment(&mut self, kind: SlruKind, segno: u32) -> anyhow::Result<()> { + pub async fn drop_slru_segment( + &mut self, + kind: SlruKind, + segno: u32, + ctx: &RequestContext, + ) -> anyhow::Result<()> { // Remove it from the directory entry let dir_key = slru_dir_to_key(kind); - let buf = self.get(dir_key).await?; + let buf = self.get(dir_key, ctx).await?; let mut dir = SlruSegmentDirectory::des(&buf)?; if !dir.segments.remove(&segno) { @@ -1015,9 +1065,13 @@ impl<'a> DatadirModification<'a> { } /// This method is used for marking truncated SLRU files - pub async fn drop_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> { + pub async fn drop_twophase_file( + &mut self, + xid: TransactionId, + ctx: &RequestContext, + ) -> anyhow::Result<()> { // Remove it from the directory entry - let buf = self.get(TWOPHASEDIR_KEY).await?; + let buf = self.get(TWOPHASEDIR_KEY, ctx).await?; let mut dir = TwoPhaseDirectory::des(&buf)?; if !dir.xids.remove(&xid) { @@ -1111,7 +1165,7 @@ impl<'a> DatadirModification<'a> { // Internal helper functions to batch the modifications - async fn get(&self, key: Key) -> Result { + async fn get(&self, key: Key, ctx: &RequestContext) -> Result { // Have we already updated the same key? Read the pending updated // version in that case. // @@ -1132,7 +1186,7 @@ impl<'a> DatadirModification<'a> { } } else { let lsn = Lsn::max(self.tline.get_last_record_lsn(), self.lsn); - self.tline.get(key, lsn).await + self.tline.get(key, lsn, ctx).await } } @@ -1542,10 +1596,11 @@ pub fn create_test_timeline( tenant: &crate::tenant::Tenant, timeline_id: utils::id::TimelineId, pg_version: u32, + ctx: &RequestContext, ) -> anyhow::Result> { let tline = tenant - .create_empty_timeline(timeline_id, Lsn(8), pg_version)? - .initialize()?; + .create_empty_timeline(timeline_id, Lsn(8), pg_version, ctx)? + .initialize(ctx)?; let mut m = tline.begin_modification(Lsn(8)); m.init_empty()?; m.commit()?; diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 331d8e8ac8..09716ba0e0 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -171,6 +171,9 @@ task_local! { /// #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum TaskKind { + // Pageserver startup, i.e., `main` + Startup, + // libpq listener task. It just accepts connection and spawns a // PageRequestHandler task for each connection. LibpqEndpointListener, @@ -194,19 +197,26 @@ pub enum TaskKind { /// The `Connection` object is responsible for speaking the wire protocol. /// /// Walreceiver uses its own abstraction called `TaskHandle` to represent the activity of establishing and handling a connection. - /// That abstraction doesn't use `task_mgr` and hence, has no `TaskKind`. + /// That abstraction doesn't use `task_mgr`. /// The [`WalReceiverManager`] task ensures that this `TaskHandle` task does not outlive the [`WalReceiverManager`] task. + /// For the `RequestContext` that we hand to the TaskHandle, we use the [`WalReceiverConnectionHandler`] task kind. /// /// Once the connection is established, the `TaskHandle` task creates a - /// [`WalReceiverConnection`] task_mgr task that is responsible for polling + /// [`WalReceiverConnectionPoller`] task_mgr task that is responsible for polling /// the `Connection` object. /// A `CancellationToken` created by the `TaskHandle` task ensures - /// that the [`WalReceiverConnection`] task will cancel soon after as the `TaskHandle` is dropped. + /// that the [`WalReceiverConnectionPoller`] task will cancel soon after as the `TaskHandle` is dropped. WalReceiverManager, - /// The task that polls the `tokio-postgres::Connection` object. + /// The `TaskHandle` task that executes [`walreceiver_connection::handle_walreceiver_connection`]. + /// Not a `task_mgr` task, but we use this `TaskKind` for its `RequestContext`. /// See the comment on [`WalReceiverManager`]. - WalReceiverConnection, + WalReceiverConnectionHandler, + + /// The task that polls the `tokio-postgres::Connection` object. + /// Spawned by task [`WalReceiverConnectionHandler`]. + /// See the comment on [`WalReceiverManager`]. + WalReceiverConnectionPoller, // Garbage collection worker. One per tenant GarbageCollector, @@ -217,6 +227,8 @@ pub enum TaskKind { // Initial logical size calculation InitialLogicalSizeCalculation, + OndemandLogicalSizeCalculation, + // Task that flushes frozen in-memory layers to disk LayerFlushTask, @@ -239,6 +251,12 @@ pub enum TaskKind { DownloadAllRemoteLayers, // Task that calculates synthetis size for all active tenants CalculateSyntheticSize, + + // A request that comes in via the pageserver HTTP API. + MgmtRequest, + + #[cfg(test)] + UnitTest, } #[derive(Default)] diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index a503604ae8..1f0219aaa3 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -48,6 +48,7 @@ use std::time::{Duration, Instant}; use self::metadata::TimelineMetadata; use self::remote_timeline_client::RemoteTimelineClient; use crate::config::PageServerConf; +use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir; use crate::is_uninit_mark; use crate::metrics::{remove_tenant_metrics, STORAGE_TIME}; @@ -174,7 +175,7 @@ impl UninitializedTimeline<'_> { /// /// The new timeline is initialized in Active state, and its background jobs are /// started - pub fn initialize(self) -> anyhow::Result> { + pub fn initialize(self, _ctx: &RequestContext) -> anyhow::Result> { let mut timelines = self.owning_tenant.timelines.lock().unwrap(); self.initialize_with_lock(&mut timelines, true, true) } @@ -239,11 +240,12 @@ impl UninitializedTimeline<'_> { self, copyin_stream: &mut (impl Stream> + Sync + Send + Unpin), base_lsn: Lsn, + ctx: &RequestContext, ) -> anyhow::Result> { let raw_timeline = self.raw_timeline()?; let mut reader = tokio_util::io::StreamReader::new(copyin_stream); - import_datadir::import_basebackup_from_tar(raw_timeline, &mut reader, base_lsn) + import_datadir::import_basebackup_from_tar(raw_timeline, &mut reader, base_lsn, ctx) .await .context("Failed to import basebackup")?; @@ -261,9 +263,7 @@ impl UninitializedTimeline<'_> { .await .context("Failed to flush after basebackup import")?; - let timeline = self.initialize()?; - - Ok(timeline) + self.initialize(ctx) } fn raw_timeline(&self) -> anyhow::Result<&Arc> { @@ -449,6 +449,7 @@ impl Tenant { /// /// If the operation fails, the timeline is left in the tenant's hash map in Broken state. On success, /// it is marked as Active. + #[allow(clippy::too_many_arguments)] async fn timeline_init_and_sync( &self, timeline_id: TimelineId, @@ -457,6 +458,7 @@ impl Tenant { local_metadata: Option, ancestor: Option>, first_save: bool, + _ctx: &RequestContext, ) -> anyhow::Result<()> { let tenant_id = self.tenant_id; @@ -572,6 +574,7 @@ impl Tenant { conf: &'static PageServerConf, tenant_id: TenantId, remote_storage: GenericRemoteStorage, + ctx: &RequestContext, ) -> Arc { // XXX: Attach should provide the config, especially during tenant migration. // See https://github.com/neondatabase/neon/issues/1555 @@ -590,6 +593,7 @@ impl Tenant { // Do all the hard work in the background let tenant_clone = Arc::clone(&tenant); + let ctx = ctx.detached_child(TaskKind::Attach, DownloadBehavior::Warn); task_mgr::spawn( &tokio::runtime::Handle::current(), TaskKind::Attach, @@ -598,7 +602,7 @@ impl Tenant { "attach tenant", false, async move { - match tenant_clone.attach().await { + match tenant_clone.attach(ctx).await { Ok(_) => {} Err(e) => { tenant_clone.set_broken(&e.to_string()); @@ -614,8 +618,8 @@ impl Tenant { /// /// Background task that downloads all data for a tenant and brings it to Active state. /// - #[instrument(skip(self), fields(tenant_id=%self.tenant_id))] - async fn attach(self: &Arc) -> anyhow::Result<()> { + #[instrument(skip(self, ctx), fields(tenant_id=%self.tenant_id))] + async fn attach(self: &Arc, ctx: RequestContext) -> anyhow::Result<()> { // Create directory with marker file to indicate attaching state. // The load_local_tenants() function in tenant::mgr relies on the marker file // to determine whether a tenant has finished attaching. @@ -715,6 +719,7 @@ impl Tenant { index_parts.remove(&timeline_id).unwrap(), remote_metadata, remote_clients.remove(&timeline_id).unwrap(), + &ctx, ) .await .with_context(|| { @@ -764,6 +769,7 @@ impl Tenant { index_part: IndexPart, remote_metadata: TimelineMetadata, remote_client: RemoteTimelineClient, + ctx: &RequestContext, ) -> anyhow::Result<()> { info!("downloading index file for timeline {}", timeline_id); tokio::fs::create_dir_all(self.conf.timeline_path(&timeline_id, &self.tenant_id)) @@ -798,6 +804,7 @@ impl Tenant { local_metadata, ancestor, true, + ctx, ) .await } @@ -826,11 +833,12 @@ impl Tenant { /// If the loading fails for some reason, the Tenant will go into Broken /// state. /// - #[instrument(skip(conf, remote_storage), fields(tenant_id=%tenant_id))] + #[instrument(skip(conf, remote_storage, ctx), fields(tenant_id=%tenant_id))] pub fn spawn_load( conf: &'static PageServerConf, tenant_id: TenantId, remote_storage: Option, + ctx: &RequestContext, ) -> Arc { let tenant_conf = match Self::load_tenant_config(conf, tenant_id) { Ok(conf) => conf, @@ -854,6 +862,7 @@ impl Tenant { // Do all the hard work in a background task let tenant_clone = Arc::clone(&tenant); + let ctx = ctx.detached_child(TaskKind::InitialLoad, DownloadBehavior::Warn); let _ = task_mgr::spawn( &tokio::runtime::Handle::current(), TaskKind::InitialLoad, @@ -862,7 +871,7 @@ impl Tenant { "initial tenant load", false, async move { - match tenant_clone.load().await { + match tenant_clone.load(&ctx).await { Ok(()) => {} Err(err) => { tenant_clone.set_broken(&err.to_string()); @@ -883,8 +892,8 @@ impl Tenant { /// Background task to load in-memory data structures for this tenant, from /// files on disk. Used at pageserver startup. /// - #[instrument(skip(self), fields(tenant_id=%self.tenant_id))] - async fn load(self: &Arc) -> anyhow::Result<()> { + #[instrument(skip(self, ctx), fields(tenant_id=%self.tenant_id))] + async fn load(self: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { info!("loading tenant task"); utils::failpoint_sleep_millis_async!("before-loading-tenant"); @@ -995,7 +1004,7 @@ impl Tenant { // 1. "Timeline has no ancestor and no layer files" for (timeline_id, local_metadata) in sorted_timelines { - self.load_local_timeline(timeline_id, local_metadata) + self.load_local_timeline(timeline_id, local_metadata, ctx) .await .with_context(|| format!("load local timeline {timeline_id}"))?; } @@ -1012,11 +1021,12 @@ impl Tenant { /// Subroutine of `load_tenant`, to load an individual timeline /// /// NB: The parent is assumed to be already loaded! - #[instrument(skip(self, local_metadata), fields(timeline_id=%timeline_id))] + #[instrument(skip(self, local_metadata, ctx), fields(timeline_id=%timeline_id))] async fn load_local_timeline( &self, timeline_id: TimelineId, local_metadata: TimelineMetadata, + ctx: &RequestContext, ) -> anyhow::Result<()> { let ancestor = if let Some(ancestor_timeline_id) = local_metadata.ancestor_timeline() { let ancestor_timeline = self.get_timeline(ancestor_timeline_id, false) @@ -1060,6 +1070,7 @@ impl Tenant { Some(local_metadata), ancestor, false, + ctx, ) .await } @@ -1111,6 +1122,7 @@ impl Tenant { new_timeline_id: TimelineId, initdb_lsn: Lsn, pg_version: u32, + _ctx: &RequestContext, ) -> anyhow::Result { anyhow::ensure!( self.is_active(), @@ -1152,6 +1164,7 @@ impl Tenant { ancestor_timeline_id: Option, mut ancestor_start_lsn: Option, pg_version: u32, + ctx: &RequestContext, ) -> anyhow::Result>> { anyhow::ensure!( self.is_active(), @@ -1189,13 +1202,21 @@ impl Tenant { // decoding the new WAL might need to look up previous pages, relation // sizes etc. and that would get confused if the previous page versions // are not in the repository yet. - ancestor_timeline.wait_lsn(*lsn).await?; + ancestor_timeline.wait_lsn(*lsn, ctx).await?; } - self.branch_timeline(ancestor_timeline_id, new_timeline_id, ancestor_start_lsn) + self.branch_timeline( + ancestor_timeline_id, + new_timeline_id, + ancestor_start_lsn, + ctx, + ) + .await? + } + None => { + self.bootstrap_timeline(new_timeline_id, pg_version, ctx) .await? } - None => self.bootstrap_timeline(new_timeline_id, pg_version).await?, }; Ok(Some(loaded_timeline)) @@ -1219,6 +1240,7 @@ impl Tenant { target_timeline_id: Option, horizon: u64, pitr: Duration, + ctx: &RequestContext, ) -> anyhow::Result { anyhow::ensure!( self.is_active(), @@ -1233,7 +1255,7 @@ impl Tenant { let _timer = STORAGE_TIME .with_label_values(&["gc", &self.tenant_id.to_string(), &timeline_str]) .start_timer(); - self.gc_iteration_internal(target_timeline_id, horizon, pitr) + self.gc_iteration_internal(target_timeline_id, horizon, pitr, ctx) .await } } @@ -1242,7 +1264,7 @@ impl Tenant { /// This function is periodically called by compactor task. /// Also it can be explicitly requested per timeline through page server /// api's 'compact' command. - pub async fn compaction_iteration(&self) -> anyhow::Result<()> { + pub async fn compaction_iteration(&self, ctx: &RequestContext) -> anyhow::Result<()> { anyhow::ensure!( self.is_active(), "Cannot run compaction iteration on inactive tenant" @@ -1264,7 +1286,7 @@ impl Tenant { for (timeline_id, timeline) in &timelines_to_compact { timeline - .compact() + .compact(ctx) .instrument(info_span!("compact_timeline", timeline = %timeline_id)) .await?; } @@ -1297,7 +1319,11 @@ impl Tenant { } /// Removes timeline-related in-memory data - pub async fn delete_timeline(&self, timeline_id: TimelineId) -> anyhow::Result<()> { + pub async fn delete_timeline( + &self, + timeline_id: TimelineId, + _ctx: &RequestContext, + ) -> anyhow::Result<()> { // Transition the timeline into TimelineState::Stopping. // This should prevent new operations from starting. let timeline = { @@ -1869,12 +1895,13 @@ impl Tenant { target_timeline_id: Option, horizon: u64, pitr: Duration, + ctx: &RequestContext, ) -> anyhow::Result { let mut totals: GcResult = Default::default(); let now = Instant::now(); let gc_timelines = self - .refresh_gc_info_internal(target_timeline_id, horizon, pitr) + .refresh_gc_info_internal(target_timeline_id, horizon, pitr, ctx) .await?; utils::failpoint_sleep_millis_async!("gc_iteration_internal_after_getting_gc_timelines"); @@ -1915,7 +1942,10 @@ impl Tenant { /// [`Tenant::get_gc_horizon`]. /// /// This is usually executed as part of periodic gc, but can now be triggered more often. - pub async fn refresh_gc_info(&self) -> anyhow::Result>> { + pub async fn refresh_gc_info( + &self, + ctx: &RequestContext, + ) -> anyhow::Result>> { // since this method can now be called at different rates than the configured gc loop, it // might be that these configuration values get applied faster than what it was previously, // since these were only read from the gc task. @@ -1925,7 +1955,7 @@ impl Tenant { // refresh all timelines let target_timeline_id = None; - self.refresh_gc_info_internal(target_timeline_id, horizon, pitr) + self.refresh_gc_info_internal(target_timeline_id, horizon, pitr, ctx) .await } @@ -1934,6 +1964,7 @@ impl Tenant { target_timeline_id: Option, horizon: u64, pitr: Duration, + ctx: &RequestContext, ) -> anyhow::Result>> { // grab mutex to prevent new timelines from being created here. let gc_cs = self.gc_cs.lock().await; @@ -2005,7 +2036,9 @@ impl Tenant { )) .map(|&x| x.1) .collect(); - timeline.update_gc_info(branchpoints, cutoff, pitr).await?; + timeline + .update_gc_info(branchpoints, cutoff, pitr, ctx) + .await?; gc_timelines.push(timeline); } @@ -2020,6 +2053,7 @@ impl Tenant { src: TimelineId, dst: TimelineId, start_lsn: Option, + _ctx: &RequestContext, ) -> anyhow::Result> { // We need to hold this lock to prevent GC from starting at the same time. GC scans the directory to learn // about timelines, so otherwise a race condition is possible, where we create new timeline and GC @@ -2120,6 +2154,7 @@ impl Tenant { &self, timeline_id: TimelineId, pg_version: u32, + ctx: &RequestContext, ) -> anyhow::Result> { let timeline_uninit_mark = { let timelines = self.timelines.lock().unwrap(); @@ -2179,6 +2214,7 @@ impl Tenant { unfinished_timeline, pgdata_path, pgdata_lsn, + ctx, ) .await .with_context(|| { @@ -2350,7 +2386,10 @@ impl Tenant { /// /// Future is cancellation safe. Only one calculation can be running at once per tenant. #[instrument(skip_all, fields(tenant_id=%self.tenant_id))] - pub async fn gather_size_inputs(&self) -> anyhow::Result { + pub async fn gather_size_inputs( + &self, + ctx: &RequestContext, + ) -> anyhow::Result { let logical_sizes_at_once = self .conf .concurrent_tenant_size_logical_size_queries @@ -2362,15 +2401,15 @@ impl Tenant { // See more for on the issue #2748 condenced out of the initial PR review. let mut shared_cache = self.cached_logical_sizes.lock().await; - size::gather_inputs(self, logical_sizes_at_once, &mut shared_cache).await + size::gather_inputs(self, logical_sizes_at_once, &mut shared_cache, ctx).await } /// Calculate synthetic tenant size /// This is periodically called by background worker. /// result is cached in tenant struct #[instrument(skip_all, fields(tenant_id=%self.tenant_id))] - pub async fn calculate_synthetic_size(&self) -> anyhow::Result { - let inputs = self.gather_size_inputs().await?; + pub async fn calculate_synthetic_size(&self, ctx: &RequestContext) -> anyhow::Result { + let inputs = self.gather_size_inputs(ctx).await?; let size = inputs.calculate()?; @@ -2741,11 +2780,17 @@ pub mod harness { }) } - pub async fn load(&self) -> Arc { - self.try_load().await.expect("failed to load test tenant") + pub async fn load(&self) -> (Arc, RequestContext) { + let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error); + ( + self.try_load(&ctx) + .await + .expect("failed to load test tenant"), + ctx, + ) } - pub async fn try_load(&self) -> anyhow::Result> { + pub async fn try_load(&self, ctx: &RequestContext) -> anyhow::Result> { let walredo_mgr = Arc::new(TestRedoManager); let tenant = Arc::new(Tenant::new( @@ -2773,8 +2818,7 @@ pub mod harness { timelines_to_load.insert(timeline_id, timeline_metadata); } // FIXME starts background jobs - tenant.load().await?; - + tenant.load(ctx).await?; Ok(tenant) } @@ -2831,10 +2875,9 @@ mod tests { #[tokio::test] async fn test_basic() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_basic")?.load().await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = TenantHarness::create("test_basic")?.load().await; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; let writer = tline.writer(); writer.put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10")))?; @@ -2847,15 +2890,15 @@ mod tests { drop(writer); assert_eq!( - tline.get(*TEST_KEY, Lsn(0x10)).await?, + tline.get(*TEST_KEY, Lsn(0x10), &ctx).await?, TEST_IMG("foo at 0x10") ); assert_eq!( - tline.get(*TEST_KEY, Lsn(0x1f)).await?, + tline.get(*TEST_KEY, Lsn(0x1f), &ctx).await?, TEST_IMG("foo at 0x10") ); assert_eq!( - tline.get(*TEST_KEY, Lsn(0x20)).await?, + tline.get(*TEST_KEY, Lsn(0x20), &ctx).await?, TEST_IMG("foo at 0x20") ); @@ -2864,14 +2907,14 @@ mod tests { #[tokio::test] async fn no_duplicate_timelines() -> anyhow::Result<()> { - let tenant = TenantHarness::create("no_duplicate_timelines")? + let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines")? .load() .await; - let _ = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let timeline = + tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let _ = timeline.initialize(&ctx)?; - match tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION) { + match tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) { Ok(_) => panic!("duplicate timeline creation should fail"), Err(e) => assert_eq!( e.to_string(), @@ -2897,13 +2940,13 @@ mod tests { /// #[tokio::test] async fn test_branch() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_branch")?.load().await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; - let writer = tline.writer(); use std::str::from_utf8; + let (tenant, ctx) = TenantHarness::create("test_branch")?.load().await; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; + let writer = tline.writer(); + #[allow(non_snake_case)] let TEST_KEY_A: Key = Key::from_hex("112222222233333333444444445500000001").unwrap(); #[allow(non_snake_case)] @@ -2923,7 +2966,7 @@ mod tests { // Branch the history, modify relation differently on the new timeline tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x30))) + .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x30)), &ctx) .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) @@ -2934,15 +2977,15 @@ mod tests { // Check page contents on both branches assert_eq!( - from_utf8(&tline.get(TEST_KEY_A, Lsn(0x40)).await?)?, + from_utf8(&tline.get(TEST_KEY_A, Lsn(0x40), &ctx).await?)?, "foo at 0x40" ); assert_eq!( - from_utf8(&newtline.get(TEST_KEY_A, Lsn(0x40)).await?)?, + from_utf8(&newtline.get(TEST_KEY_A, Lsn(0x40), &ctx).await?)?, "bar at 0x40" ); assert_eq!( - from_utf8(&newtline.get(TEST_KEY_B, Lsn(0x40)).await?)?, + from_utf8(&newtline.get(TEST_KEY_B, Lsn(0x40), &ctx).await?)?, "foobar at 0x20" ); @@ -2994,13 +3037,12 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_garbage_collected_data() -> anyhow::Result<()> { - let tenant = + let (tenant, ctx) = TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? .load() .await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50 @@ -3008,12 +3050,12 @@ mod tests { // and compaction works. But it does set the 'cutoff' point so that the cross check // below should fail. tenant - .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO) + .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) .await?; // try to branch at lsn 25, should fail because we already garbage collected the data match tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x25))) + .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) .await { Ok(_) => panic!("branching should have failed"), @@ -3032,16 +3074,16 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? - .load() - .await; - - tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = + TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? + .load() + .await; + let tline = + tenant.create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx)?; + let _tline = tline.initialize(&ctx)?; // try to branch at lsn 0x25, should fail because initdb lsn is 0x50 match tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x25))) + .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) .await { Ok(_) => panic!("branching should have failed"), @@ -3083,40 +3125,40 @@ mod tests { #[tokio::test] async fn test_retain_data_in_parent_which_is_needed_for_child() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? - .load() - .await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = + TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? + .load() + .await; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40))) + .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) .expect("Should have a local timeline"); // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50 tenant - .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO) + .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) .await?; - assert!(newtline.get(*TEST_KEY, Lsn(0x25)).await.is_ok()); + assert!(newtline.get(*TEST_KEY, Lsn(0x25), &ctx).await.is_ok()); Ok(()) } #[tokio::test] async fn test_parent_keeps_data_forever_after_branching() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_parent_keeps_data_forever_after_branching")? - .load() - .await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = + TenantHarness::create("test_parent_keeps_data_forever_after_branching")? + .load() + .await; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40))) + .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) @@ -3126,12 +3168,12 @@ mod tests { // run gc on parent tenant - .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO) + .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) .await?; // Check that the data is still accessible on the branch. assert_eq!( - newtline.get(*TEST_KEY, Lsn(0x50)).await?, + newtline.get(*TEST_KEY, Lsn(0x50), &ctx).await?, TEST_IMG(&format!("foo at {}", Lsn(0x40))) ); @@ -3143,14 +3185,14 @@ mod tests { const TEST_NAME: &str = "timeline_load"; let harness = TenantHarness::create(TEST_NAME)?; { - let tenant = harness.load().await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0x8000), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = harness.load().await; + let tline = + tenant.create_empty_timeline(TIMELINE_ID, Lsn(0x8000), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; make_some_layers(tline.as_ref(), Lsn(0x8000)).await?; } - let tenant = harness.load().await; + let (tenant, _ctx) = harness.load().await; tenant .get_timeline(TIMELINE_ID, true) .expect("cannot load timeline"); @@ -3164,15 +3206,15 @@ mod tests { let harness = TenantHarness::create(TEST_NAME)?; // create two timelines { - let tenant = harness.load().await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = harness.load().await; + let tline = + tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40))) + .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) .await?; let newtline = tenant @@ -3183,7 +3225,7 @@ mod tests { } // check that both of them are initially unloaded - let tenant = harness.load().await; + let (tenant, _ctx) = harness.load().await; // check that both, child and ancestor are loaded let _child_tline = tenant @@ -3201,11 +3243,11 @@ mod tests { async fn corrupt_metadata() -> anyhow::Result<()> { const TEST_NAME: &str = "corrupt_metadata"; let harness = TenantHarness::create(TEST_NAME)?; - let tenant = harness.load().await; + let (tenant, ctx) = harness.load().await; tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)? + .initialize(&ctx)?; drop(tenant); let metadata_path = harness.timeline_path(&TIMELINE_ID).join(METADATA_FILE_NAME); @@ -3217,7 +3259,7 @@ mod tests { metadata_bytes[8] ^= 1; std::fs::write(metadata_path, metadata_bytes)?; - let err = harness.try_load().await.err().expect("should fail"); + let err = harness.try_load(&ctx).await.err().expect("should fail"); assert!(err .to_string() .starts_with("Failed to parse metadata bytes from path")); @@ -3241,10 +3283,9 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_images")?.load().await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = TenantHarness::create("test_images")?.load().await; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; let writer = tline.writer(); writer.put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10")))?; @@ -3252,7 +3293,7 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact().await?; + tline.compact(&ctx).await?; let writer = tline.writer(); writer.put(*TEST_KEY, Lsn(0x20), &Value::Image(TEST_IMG("foo at 0x20")))?; @@ -3260,7 +3301,7 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact().await?; + tline.compact(&ctx).await?; let writer = tline.writer(); writer.put(*TEST_KEY, Lsn(0x30), &Value::Image(TEST_IMG("foo at 0x30")))?; @@ -3268,7 +3309,7 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact().await?; + tline.compact(&ctx).await?; let writer = tline.writer(); writer.put(*TEST_KEY, Lsn(0x40), &Value::Image(TEST_IMG("foo at 0x40")))?; @@ -3276,26 +3317,26 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact().await?; + tline.compact(&ctx).await?; assert_eq!( - tline.get(*TEST_KEY, Lsn(0x10)).await?, + tline.get(*TEST_KEY, Lsn(0x10), &ctx).await?, TEST_IMG("foo at 0x10") ); assert_eq!( - tline.get(*TEST_KEY, Lsn(0x1f)).await?, + tline.get(*TEST_KEY, Lsn(0x1f), &ctx).await?, TEST_IMG("foo at 0x10") ); assert_eq!( - tline.get(*TEST_KEY, Lsn(0x20)).await?, + tline.get(*TEST_KEY, Lsn(0x20), &ctx).await?, TEST_IMG("foo at 0x20") ); assert_eq!( - tline.get(*TEST_KEY, Lsn(0x30)).await?, + tline.get(*TEST_KEY, Lsn(0x30), &ctx).await?, TEST_IMG("foo at 0x30") ); assert_eq!( - tline.get(*TEST_KEY, Lsn(0x40)).await?, + tline.get(*TEST_KEY, Lsn(0x40), &ctx).await?, TEST_IMG("foo at 0x40") ); @@ -3308,10 +3349,9 @@ mod tests { // #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_bulk_insert")?.load().await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = TenantHarness::create("test_bulk_insert")?.load().await; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; let mut lsn = Lsn(0x10); @@ -3340,10 +3380,10 @@ mod tests { let cutoff = tline.get_last_record_lsn(); tline - .update_gc_info(Vec::new(), cutoff, Duration::ZERO) + .update_gc_info(Vec::new(), cutoff, Duration::ZERO, &ctx) .await?; tline.freeze_and_flush().await?; - tline.compact().await?; + tline.compact(&ctx).await?; tline.gc().await?; } @@ -3352,10 +3392,9 @@ mod tests { #[tokio::test] async fn test_random_updates() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_random_updates")?.load().await; - let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + let (tenant, ctx) = TenantHarness::create("test_random_updates")?.load().await; + let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tline.initialize(&ctx)?; const NUM_KEYS: usize = 1000; @@ -3405,7 +3444,7 @@ mod tests { for (blknum, last_lsn) in updated.iter().enumerate() { test_key.field6 = blknum as u32; assert_eq!( - tline.get(test_key, lsn).await?, + tline.get(test_key, lsn, &ctx).await?, TEST_IMG(&format!("{} at {}", blknum, last_lsn)) ); } @@ -3413,10 +3452,10 @@ mod tests { // Perform a cycle of flush, compact, and GC let cutoff = tline.get_last_record_lsn(); tline - .update_gc_info(Vec::new(), cutoff, Duration::ZERO) + .update_gc_info(Vec::new(), cutoff, Duration::ZERO, &ctx) .await?; tline.freeze_and_flush().await?; - tline.compact().await?; + tline.compact(&ctx).await?; tline.gc().await?; } @@ -3425,12 +3464,12 @@ mod tests { #[tokio::test] async fn test_traverse_branches() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_traverse_branches")? + let (tenant, ctx) = TenantHarness::create("test_traverse_branches")? .load() .await; let mut tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)? + .initialize(&ctx)?; const NUM_KEYS: usize = 1000; @@ -3464,7 +3503,7 @@ mod tests { for _ in 0..50 { let new_tline_id = TimelineId::generate(); tenant - .branch_timeline(tline_id, new_tline_id, Some(lsn)) + .branch_timeline(tline_id, new_tline_id, Some(lsn), &ctx) .await?; tline = tenant .get_timeline(new_tline_id, true) @@ -3491,7 +3530,7 @@ mod tests { for (blknum, last_lsn) in updated.iter().enumerate() { test_key.field6 = blknum as u32; assert_eq!( - tline.get(test_key, lsn).await?, + tline.get(test_key, lsn, &ctx).await?, TEST_IMG(&format!("{} at {}", blknum, last_lsn)) ); } @@ -3499,10 +3538,10 @@ mod tests { // Perform a cycle of flush, compact, and GC let cutoff = tline.get_last_record_lsn(); tline - .update_gc_info(Vec::new(), cutoff, Duration::ZERO) + .update_gc_info(Vec::new(), cutoff, Duration::ZERO, &ctx) .await?; tline.freeze_and_flush().await?; - tline.compact().await?; + tline.compact(&ctx).await?; tline.gc().await?; } @@ -3511,12 +3550,12 @@ mod tests { #[tokio::test] async fn test_traverse_ancestors() -> anyhow::Result<()> { - let tenant = TenantHarness::create("test_traverse_ancestors")? + let (tenant, ctx) = TenantHarness::create("test_traverse_ancestors")? .load() .await; let mut tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)? - .initialize()?; + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)? + .initialize(&ctx)?; const NUM_KEYS: usize = 100; const NUM_TLINES: usize = 50; @@ -3532,7 +3571,7 @@ mod tests { for idx in 0..NUM_TLINES { let new_tline_id = TimelineId::generate(); tenant - .branch_timeline(tline_id, new_tline_id, Some(lsn)) + .branch_timeline(tline_id, new_tline_id, Some(lsn), &ctx) .await?; tline = tenant .get_timeline(new_tline_id, true) @@ -3566,7 +3605,7 @@ mod tests { println!("checking [{idx}][{blknum}] at {lsn}"); test_key.field6 = blknum as u32; assert_eq!( - tline.get(test_key, *lsn).await?, + tline.get(test_key, *lsn, &ctx).await?, TEST_IMG(&format!("{idx} {blknum} at {lsn}")) ); } diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index dce7cd8bae..9e9c2786b3 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -16,6 +16,7 @@ use remote_storage::GenericRemoteStorage; use utils::crashsafe; use crate::config::PageServerConf; +use crate::context::{DownloadBehavior, RequestContext}; use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::TenantConfOpt; use crate::tenant::{Tenant, TenantState}; @@ -43,6 +44,8 @@ pub async fn init_tenant_mgr( .await .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?; + let ctx = RequestContext::todo_child(TaskKind::Startup, DownloadBehavior::Warn); + loop { match dir_entries.next_entry().await { Ok(None) => break, @@ -86,6 +89,7 @@ pub async fn init_tenant_mgr( conf, &tenant_dir_path, remote_storage.clone(), + &ctx, ) { Ok(tenant) => { TENANTS.write().await.insert(tenant.tenant_id(), tenant); @@ -116,6 +120,7 @@ pub fn schedule_local_tenant_processing( conf: &'static PageServerConf, tenant_path: &Path, remote_storage: Option, + ctx: &RequestContext, ) -> anyhow::Result> { anyhow::ensure!( tenant_path.is_dir(), @@ -150,7 +155,7 @@ pub fn schedule_local_tenant_processing( let tenant = if conf.tenant_attaching_mark_file_path(&tenant_id).exists() { info!("tenant {tenant_id} has attaching mark file, resuming its attach operation"); if let Some(remote_storage) = remote_storage { - Tenant::spawn_attach(conf, tenant_id, remote_storage) + Tenant::spawn_attach(conf, tenant_id, remote_storage, ctx) } else { warn!("tenant {tenant_id} has attaching mark file, but pageserver has no remote storage configured"); Tenant::create_broken_tenant(conf, tenant_id) @@ -158,7 +163,7 @@ pub fn schedule_local_tenant_processing( } else { info!("tenant {tenant_id} is assumed to be loadable, starting load operation"); // Start loading the tenant into memory. It will initially be in Loading state. - Tenant::spawn_load(conf, tenant_id, remote_storage) + Tenant::spawn_load(conf, tenant_id, remote_storage, ctx) }; Ok(tenant) } @@ -207,6 +212,7 @@ pub async fn create_tenant( tenant_conf: TenantConfOpt, tenant_id: TenantId, remote_storage: Option, + ctx: &RequestContext, ) -> anyhow::Result>> { match TENANTS.write().await.entry(tenant_id) { hash_map::Entry::Occupied(_) => { @@ -218,7 +224,7 @@ pub async fn create_tenant( // If this section ever becomes contentious, introduce a new `TenantState::Creating`. let tenant_directory = super::create_tenant_files(conf, tenant_conf, tenant_id)?; let created_tenant = - schedule_local_tenant_processing(conf, &tenant_directory, remote_storage)?; + schedule_local_tenant_processing(conf, &tenant_directory, remote_storage, ctx)?; let crated_tenant_id = created_tenant.tenant_id(); anyhow::ensure!( tenant_id == crated_tenant_id, @@ -260,10 +266,14 @@ pub async fn get_tenant(tenant_id: TenantId, active_only: bool) -> anyhow::Resul } } -pub async fn delete_timeline(tenant_id: TenantId, timeline_id: TimelineId) -> anyhow::Result<()> { +pub async fn delete_timeline( + tenant_id: TenantId, + timeline_id: TimelineId, + ctx: &RequestContext, +) -> anyhow::Result<()> { match get_tenant(tenant_id, true).await { Ok(tenant) => { - tenant.delete_timeline(timeline_id).await?; + tenant.delete_timeline(timeline_id, ctx).await?; } Err(e) => anyhow::bail!("Cannot access tenant {tenant_id} in local tenant state: {e:?}"), } @@ -291,6 +301,7 @@ pub async fn load_tenant( conf: &'static PageServerConf, tenant_id: TenantId, remote_storage: Option, + ctx: &RequestContext, ) -> anyhow::Result<()> { run_if_no_tenant_in_memory(tenant_id, |vacant_entry| { let tenant_path = conf.tenant_path(&tenant_id); @@ -300,7 +311,7 @@ pub async fn load_tenant( .with_context(|| format!("Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading"))?; } - let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, remote_storage) + let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, remote_storage, ctx) .with_context(|| { format!("Failed to schedule tenant processing in path {tenant_path:?}") })?; @@ -349,6 +360,7 @@ pub async fn attach_tenant( conf: &'static PageServerConf, tenant_id: TenantId, remote_storage: GenericRemoteStorage, + ctx: &RequestContext, ) -> anyhow::Result<()> { run_if_no_tenant_in_memory(tenant_id, |vacant_entry| { let tenant_path = conf.tenant_path(&tenant_id); @@ -357,7 +369,7 @@ pub async fn attach_tenant( "Cannot attach tenant {tenant_id}, local tenant directory already exists" ); - let tenant = Tenant::spawn_attach(conf, tenant_id, remote_storage); + let tenant = Tenant::spawn_attach(conf, tenant_id, remote_storage, ctx); vacant_entry.insert(tenant); Ok(()) @@ -449,9 +461,9 @@ pub async fn immediate_gc( tenant_id: TenantId, timeline_id: TimelineId, gc_req: TimelineGcRequest, + ctx: &RequestContext, ) -> Result>, ApiError> { let guard = TENANTS.read().await; - let tenant = guard .get(&tenant_id) .map(Arc::clone) @@ -462,7 +474,8 @@ pub async fn immediate_gc( // Use tenant's pitr setting let pitr = tenant.get_pitr_interval(); - // Run in task_mgr to avoid race with detach operation + // Run in task_mgr to avoid race with tenant_detach operation + let ctx = ctx.detached_child(TaskKind::GarbageCollector, DownloadBehavior::Download); let (task_done, wait_task_done) = tokio::sync::oneshot::channel(); task_mgr::spawn( &tokio::runtime::Handle::current(), @@ -474,7 +487,7 @@ pub async fn immediate_gc( async move { fail::fail_point!("immediate_gc_task_pre"); let result = tenant - .gc_iteration(Some(timeline_id), gc_horizon, pitr) + .gc_iteration(Some(timeline_id), gc_horizon, pitr, &ctx) .instrument(info_span!("manual_gc", tenant = %tenant_id, timeline = %timeline_id)) .await; // FIXME: `gc_iteration` can return an error for multiple reasons; we should handle it @@ -497,6 +510,7 @@ pub async fn immediate_gc( pub async fn immediate_compact( tenant_id: TenantId, timeline_id: TimelineId, + ctx: &RequestContext, ) -> Result>, ApiError> { let guard = TENANTS.read().await; @@ -510,7 +524,8 @@ pub async fn immediate_compact( .get_timeline(timeline_id, true) .map_err(ApiError::NotFound)?; - // Run in task_mgr to avoid race with detach operation + // Run in task_mgr to avoid race with tenant_detach operation + let ctx = ctx.detached_child(TaskKind::Compaction, DownloadBehavior::Download); let (task_done, wait_task_done) = tokio::sync::oneshot::channel(); task_mgr::spawn( &tokio::runtime::Handle::current(), @@ -523,7 +538,7 @@ pub async fn immediate_compact( false, async move { let result = timeline - .compact() + .compact(&ctx) .instrument( info_span!("manual_compact", tenant = %tenant_id, timeline = %timeline_id), ) diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 013591caee..3f69017160 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1010,7 +1010,10 @@ impl RemoteTimelineClient { #[cfg(test)] mod tests { use super::*; - use crate::tenant::harness::{TenantHarness, TIMELINE_ID}; + use crate::{ + tenant::harness::{TenantHarness, TIMELINE_ID}, + DEFAULT_PG_VERSION, + }; use remote_storage::{RemoteStorageConfig, RemoteStorageKind}; use std::{collections::HashSet, path::Path}; use utils::lsn::Lsn; @@ -1064,9 +1067,19 @@ mod tests { // Test scheduling #[test] fn upload_scheduling() -> anyhow::Result<()> { + // Use a current-thread runtime in the test + let runtime = Box::leak(Box::new( + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?, + )); + let _entered = runtime.enter(); + let harness = TenantHarness::create("upload_scheduling")?; + let (tenant, ctx) = runtime.block_on(harness.load()); + let _timeline = + tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; let timeline_path = harness.timeline_path(&TIMELINE_ID); - std::fs::create_dir_all(&timeline_path)?; let remote_fs_dir = harness.conf.workdir.join("remote_fs"); std::fs::create_dir_all(remote_fs_dir)?; @@ -1084,14 +1097,6 @@ mod tests { storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), }; - // Use a current-thread runtime in the test - let runtime = Box::leak(Box::new( - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?, - )); - let _entered = runtime.enter(); - // Test outline: // // Schedule upload of a bunch of layers. Check that they are started immediately, not queued diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index 61cb32fc76..2fed4f88b3 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -6,6 +6,7 @@ use anyhow::Context; use tokio::sync::oneshot::error::RecvError; use tokio::sync::Semaphore; +use crate::context::RequestContext; use crate::pgdatadir_mapping::CalculateLogicalSizeError; use super::Tenant; @@ -181,6 +182,7 @@ pub(super) async fn gather_inputs( tenant: &Tenant, limit: &Arc, logical_size_cache: &mut HashMap<(TimelineId, Lsn), u64>, + ctx: &RequestContext, ) -> anyhow::Result { // with joinset, on drop, all of the tasks will just be de-scheduled, which we can use to // our advantage with `?` error handling. @@ -188,7 +190,7 @@ pub(super) async fn gather_inputs( // refresh is needed to update gc related pitr_cutoff and horizon_cutoff tenant - .refresh_gc_info() + .refresh_gc_info(ctx) .await .context("Failed to refresh gc_info before gathering inputs")?; @@ -329,7 +331,13 @@ pub(super) async fn gather_inputs( } else { let timeline = Arc::clone(&timeline); let parallel_size_calcs = Arc::clone(limit); - joinset.spawn(calculate_logical_size(parallel_size_calcs, timeline, *lsn)); + let ctx = ctx.attached_child(); + joinset.spawn(calculate_logical_size( + parallel_size_calcs, + timeline, + *lsn, + ctx, + )); } } @@ -387,6 +395,7 @@ pub(super) async fn gather_inputs( parallel_size_calcs, timeline.clone(), lsn, + ctx.attached_child(), )); if let Some(parent_id) = timeline.get_ancestor_timeline_id() { @@ -582,13 +591,14 @@ async fn calculate_logical_size( limit: Arc, timeline: Arc, lsn: utils::lsn::Lsn, + ctx: RequestContext, ) -> Result { let _permit = tokio::sync::Semaphore::acquire_owned(limit) .await .expect("global semaphore should not had been closed"); let size_res = timeline - .spawn_ondemand_logical_size_calculation(lsn) + .spawn_ondemand_logical_size_calculation(lsn, ctx) .instrument(info_span!("spawn_ondemand_logical_size_calculation")) .await?; Ok(TimelineAtLsnSizeResult(timeline, lsn, size_res)) diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index b7ad8fe791..b126545ee4 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -5,6 +5,7 @@ use std::ops::ControlFlow; use std::sync::Arc; use std::time::Duration; +use crate::context::{DownloadBehavior, RequestContext}; use crate::metrics::TENANT_TASK_EVENTS; use crate::task_mgr; use crate::task_mgr::{TaskKind, BACKGROUND_RUNTIME}; @@ -52,19 +53,20 @@ async fn compaction_loop(tenant_id: TenantId) { info!("starting"); TENANT_TASK_EVENTS.with_label_values(&["start"]).inc(); async { + let ctx = RequestContext::todo_child(TaskKind::Compaction, DownloadBehavior::Download); loop { trace!("waking up"); let tenant = tokio::select! { _ = task_mgr::shutdown_watcher() => { info!("received cancellation request"); - return; + return; }, tenant_wait_result = wait_for_active_tenant(tenant_id, wait_duration) => match tenant_wait_result { ControlFlow::Break(()) => return, ControlFlow::Continue(tenant) => tenant, }, - }; + }; let mut sleep_duration = tenant.get_compaction_period(); if sleep_duration == Duration::ZERO { @@ -73,7 +75,7 @@ async fn compaction_loop(tenant_id: TenantId) { sleep_duration = Duration::from_secs(10); } else { // Run compaction - if let Err(e) = tenant.compaction_iteration().await { + if let Err(e) = tenant.compaction_iteration(&ctx).await { sleep_duration = wait_duration; error!("Compaction failed, retrying in {:?}: {e:?}", sleep_duration); } @@ -103,6 +105,9 @@ async fn gc_loop(tenant_id: TenantId) { info!("starting"); TENANT_TASK_EVENTS.with_label_values(&["start"]).inc(); async { + // GC might require downloading, to find the cutoff LSN that corresponds to the + // cutoff specified as time. + let ctx = RequestContext::todo_child(TaskKind::GarbageCollector, DownloadBehavior::Download); loop { trace!("waking up"); @@ -127,7 +132,7 @@ async fn gc_loop(tenant_id: TenantId) { } else { // Run gc if gc_horizon > 0 { - if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval()).await + if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), &ctx).await { sleep_duration = wait_duration; error!("Gc failed, retrying in {:?}: {e:?}", sleep_duration); diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index b22a339839..134c8eed3c 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -23,6 +23,7 @@ use std::sync::atomic::{AtomicI64, Ordering as AtomicOrdering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, Weak}; use std::time::{Duration, Instant, SystemTime}; +use crate::context::{DownloadBehavior, RequestContext}; use crate::tenant::remote_timeline_client::{self, index::LayerFileMetadata}; use crate::tenant::storage_layer::{ DeltaFileName, DeltaLayerWriter, ImageFileName, ImageLayerWriter, InMemoryLayer, LayerFileName, @@ -377,6 +378,12 @@ pub enum PageReconstructError { #[error(transparent)] Other(#[from] anyhow::Error), // source and Display delegate to anyhow::Error + /// The operation would require downloading a layer that is missing locally. + NeedsDownload(Weak, Weak), + + /// The operation was cancelled + Cancelled, + /// An error happened replaying WAL records #[error(transparent)] WalRedo(#[from] crate::walredo::WalRedoError), @@ -386,6 +393,19 @@ impl std::fmt::Debug for PageReconstructError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { match self { Self::Other(err) => err.fmt(f), + Self::NeedsDownload(_tli, _layer) => write!(f, "needs download"), + Self::Cancelled => write!(f, "cancelled"), + Self::WalRedo(err) => err.fmt(f), + } + } +} + +impl std::fmt::Display for PageReconstructError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + match self { + Self::Other(err) => err.fmt(f), + Self::NeedsDownload(_tli, _layer) => write!(f, "needs download"), + Self::Cancelled => write!(f, "cancelled"), Self::WalRedo(err) => err.fmt(f), } } @@ -422,11 +442,24 @@ impl Timeline { /// an ancestor branch, for example, or waste a lot of cycles chasing the /// non-existing key. /// - pub async fn get(&self, key: Key, lsn: Lsn) -> Result { + pub async fn get( + &self, + key: Key, + lsn: Lsn, + ctx: &RequestContext, + ) -> Result { if !lsn.is_valid() { return Err(PageReconstructError::Other(anyhow::anyhow!("Invalid LSN"))); } + // XXX: structured stats collection for layer eviction here. + trace!( + "get page request for {}@{} from task kind {:?}", + key, + lsn, + ctx.task_kind() + ); + // Check the page cache. We will get back the most recent page with lsn <= `lsn`. // The cached image can be returned directly if there is no WAL between the cached image // and requested LSN. The cached image can also be used to reduce the amount of WAL needed @@ -450,7 +483,7 @@ impl Timeline { img: cached_page_img, }; - self.get_reconstruct_data(key, lsn, &mut reconstruct_state) + self.get_reconstruct_data(key, lsn, &mut reconstruct_state, ctx) .await?; self.metrics @@ -513,13 +546,25 @@ impl Timeline { /// You should call this before any of the other get_* or list_* functions. Calling /// those functions with an LSN that has been processed yet is an error. /// - pub async fn wait_lsn(&self, lsn: Lsn) -> anyhow::Result<()> { + pub async fn wait_lsn( + &self, + lsn: Lsn, + _ctx: &RequestContext, /* Prepare for use by cancellation */ + ) -> anyhow::Result<()> { anyhow::ensure!(self.is_active(), "Cannot wait for Lsn on inactive timeline"); // This should never be called from the WAL receiver, because that could lead // to a deadlock. anyhow::ensure!( - task_mgr::current_task_kind() != Some(TaskKind::WalReceiverConnection), + task_mgr::current_task_kind() != Some(TaskKind::WalReceiverManager), + "wait_lsn cannot be called in WAL receiver" + ); + anyhow::ensure!( + task_mgr::current_task_kind() != Some(TaskKind::WalReceiverConnectionHandler), + "wait_lsn cannot be called in WAL receiver" + ); + anyhow::ensure!( + task_mgr::current_task_kind() != Some(TaskKind::WalReceiverConnectionPoller), "wait_lsn cannot be called in WAL receiver" ); @@ -558,7 +603,7 @@ impl Timeline { self.flush_frozen_layers_and_wait().await } - pub async fn compact(&self) -> anyhow::Result<()> { + pub async fn compact(&self, ctx: &RequestContext) -> anyhow::Result<()> { let last_record_lsn = self.get_last_record_lsn(); // Last record Lsn could be zero in case the timeline was just created @@ -616,14 +661,16 @@ impl Timeline { .repartition( self.get_last_record_lsn(), self.get_compaction_target_size(), + ctx, ) .await { Ok((partitioning, lsn)) => { // 2. Create new image layers for partitions that have been modified // "enough". - let layer_paths_to_upload = - self.create_image_layers(&partitioning, lsn, false).await?; + let layer_paths_to_upload = self + .create_image_layers(&partitioning, lsn, false, ctx) + .await?; if let Some(remote_client) = &self.remote_client { for (path, layer_metadata) in layer_paths_to_upload { remote_client.schedule_layer_file_upload(&path, &layer_metadata)?; @@ -673,7 +720,10 @@ impl Timeline { /// the initial size calculation has not been run (gets triggered on the first size access). /// /// return size and boolean flag that shows if the size is exact - pub fn get_current_logical_size(self: &Arc) -> anyhow::Result<(u64, bool)> { + pub fn get_current_logical_size( + self: &Arc, + ctx: &RequestContext, + ) -> anyhow::Result<(u64, bool)> { let current_size = self.current_logical_size.current_size()?; debug!("Current size: {current_size:?}"); @@ -683,7 +733,7 @@ impl Timeline { (current_size, self.current_logical_size.initial_part_end) { is_exact = false; - self.try_spawn_size_init_task(init_lsn); + self.try_spawn_size_init_task(init_lsn, ctx); } Ok((size, is_exact)) @@ -917,22 +967,25 @@ impl Timeline { let layer_flush_start_rx = self.layer_flush_start_tx.subscribe(); let self_clone = Arc::clone(self); + info!("spawning flush loop"); task_mgr::spawn( - task_mgr::BACKGROUND_RUNTIME.handle(), - task_mgr::TaskKind::LayerFlushTask, - Some(self.tenant_id), - Some(self.timeline_id), - "layer flush task", - false, - async move { - self_clone.flush_loop(layer_flush_start_rx).await; - let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap(); - assert_eq!(*flush_loop_state, FlushLoopState::Running); - *flush_loop_state = FlushLoopState::Exited; - Ok(()) } - .instrument(info_span!(parent: None, "layer flush task", tenant = %self.tenant_id, timeline = %self.timeline_id)) - ); + task_mgr::BACKGROUND_RUNTIME.handle(), + task_mgr::TaskKind::LayerFlushTask, + Some(self.tenant_id), + Some(self.timeline_id), + "layer flush task", + false, + async move { + let background_ctx = RequestContext::todo_child(TaskKind::LayerFlushTask, DownloadBehavior::Error); + self_clone.flush_loop(layer_flush_start_rx, &background_ctx).await; + let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap(); + assert_eq!(*flush_loop_state, FlushLoopState::Running); + *flush_loop_state = FlushLoopState::Exited; + Ok(()) + } + .instrument(info_span!(parent: None, "layer flush task", tenant = %self.tenant_id, timeline = %self.timeline_id)) + ); *flush_loop_state = FlushLoopState::Running; } @@ -963,12 +1016,16 @@ impl Timeline { .unwrap_or(self.conf.default_tenant_conf.max_lsn_wal_lag); drop(tenant_conf_guard); let self_clone = Arc::clone(self); + let background_ctx = + // XXX: this is a detached_child. Plumb through the ctx from call sites. + RequestContext::todo_child(TaskKind::WalReceiverManager, DownloadBehavior::Error); spawn_connection_manager_task( self_clone, walreceiver_connect_timeout, lagging_wal_timeout, max_lsn_wal_lag, crate::config::SAFEKEEPER_AUTH_TOKEN.get().cloned(), + background_ctx, ); } @@ -1296,7 +1353,7 @@ impl Timeline { Ok(()) } - fn try_spawn_size_init_task(self: &Arc, init_lsn: Lsn) { + fn try_spawn_size_init_task(self: &Arc, init_lsn: Lsn, ctx: &RequestContext) { let permit = match Arc::clone(&self.current_logical_size.initial_size_computation) .try_acquire_owned() { @@ -1312,8 +1369,18 @@ impl Timeline { .initial_logical_size .get() .is_none()); + + info!( + "spawning logical size computation from context of task kind {:?}", + ctx.task_kind() + ); // We need to start the computation task. + // It gets a separate context since it will outlive the request that called this function. let self_clone = Arc::clone(self); + let background_ctx = ctx.detached_child( + TaskKind::InitialLogicalSizeCalculation, + DownloadBehavior::Download, + ); task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), task_mgr::TaskKind::InitialLogicalSizeCalculation, @@ -1323,7 +1390,9 @@ impl Timeline { false, // NB: don't log errors here, task_mgr will do that. async move { - let calculated_size = match self_clone.logical_size_calculation_task(init_lsn).await + let calculated_size = match self_clone + .logical_size_calculation_task(init_lsn, &background_ctx) + .await { Ok(s) => s, Err(CalculateLogicalSizeError::Cancelled) => { @@ -1358,18 +1427,27 @@ impl Timeline { pub fn spawn_ondemand_logical_size_calculation( self: &Arc, lsn: Lsn, + ctx: RequestContext, ) -> oneshot::Receiver> { let (sender, receiver) = oneshot::channel(); let self_clone = Arc::clone(self); + // XXX if our caller loses interest, i.e., ctx is cancelled, + // we should stop the size calculation work and return an error. + // That would require restructuring this function's API to + // return the result directly, instead of a Receiver for the result. + let ctx = ctx.detached_child( + TaskKind::OndemandLogicalSizeCalculation, + DownloadBehavior::Download, + ); task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), - task_mgr::TaskKind::InitialLogicalSizeCalculation, + task_mgr::TaskKind::OndemandLogicalSizeCalculation, Some(self.tenant_id), Some(self.timeline_id), "ondemand logical size calculation", false, async move { - let res = self_clone.logical_size_calculation_task(lsn).await; + let res = self_clone.logical_size_calculation_task(lsn, &ctx).await; let _ = sender.send(res).ok(); Ok(()) // Receiver is responsible for handling errors }, @@ -1381,6 +1459,7 @@ impl Timeline { async fn logical_size_calculation_task( self: &Arc, init_lsn: Lsn, + ctx: &RequestContext, ) -> Result { let mut timeline_state_updates = self.subscribe_for_state_updates(); let self_calculation = Arc::clone(self); @@ -1388,12 +1467,13 @@ impl Timeline { let calculation = async { let cancel = cancel.child_token(); + let ctx = ctx.attached_child(); tokio::task::spawn_blocking(move || { // Run in a separate thread since this can do a lot of // synchronous file IO without .await inbetween // if there are no RemoteLayers that would require downloading. let h = tokio::runtime::Handle::current(); - h.block_on(self_calculation.calculate_logical_size(init_lsn, cancel)) + h.block_on(self_calculation.calculate_logical_size(init_lsn, cancel, &ctx)) }) .await .context("Failed to spawn calculation result task")? @@ -1448,10 +1528,11 @@ impl Timeline { /// /// NOTE: counted incrementally, includes ancestors. This can be a slow operation, /// especially if we need to download remote layers. - async fn calculate_logical_size( + pub async fn calculate_logical_size( &self, up_to_lsn: Lsn, cancel: CancellationToken, + ctx: &RequestContext, ) -> Result { info!( "Calculating logical size for timeline {} at {}", @@ -1494,7 +1575,7 @@ impl Timeline { self.metrics.logical_size_histo.start_timer() }; let logical_size = self - .get_current_logical_size_non_incremental(up_to_lsn, cancel) + .get_current_logical_size_non_incremental(up_to_lsn, cancel, ctx) .await?; debug!("calculated logical size: {logical_size}"); timer.stop_and_record(); @@ -1571,6 +1652,7 @@ impl Timeline { key: Key, request_lsn: Lsn, reconstruct_state: &mut ValueReconstructState, + ctx: &RequestContext, ) -> Result<(), PageReconstructError> { // Start from the current timeline. let mut timeline_owned; @@ -1758,14 +1840,43 @@ impl Timeline { let remote_layer_as_persistent: Arc = Arc::clone(&remote_layer) as Arc; let id = remote_layer_as_persistent.traversal_id(); - info!("need remote layer {id}"); + info!( + "need remote layer {} for task kind {:?}", + id, + ctx.task_kind() + ); // The next layer doesn't exist locally. Need to download it. // (The control flow is a bit complicated here because we must drop the 'layers' // lock before awaiting on the Future.) - info!("on-demand downloading remote layer {id}"); - timeline.download_remote_layer(remote_layer).await?; - continue 'layer_map_search; + match ( + ctx.download_behavior(), + self.conf.ondemand_download_behavior_treat_error_as_warn, + ) { + (DownloadBehavior::Download, _) => { + info!( + "on-demand downloading remote layer {id} for task kind {:?}", + ctx.task_kind() + ); + timeline.download_remote_layer(remote_layer).await?; + continue 'layer_map_search; + } + (DownloadBehavior::Warn, _) | (DownloadBehavior::Error, true) => { + warn!( + "unexpectedly on-demand downloading remote layer {} for task kind {:?}", + id, + ctx.task_kind() + ); + timeline.download_remote_layer(remote_layer).await?; + continue 'layer_map_search; + } + (DownloadBehavior::Error, false) => { + return Err(PageReconstructError::NeedsDownload( + timeline.myself.clone(), + Arc::downgrade(&remote_layer), + )) + } + } } } } @@ -1887,7 +1998,11 @@ impl Timeline { } /// Layer flusher task's main loop. - async fn flush_loop(&self, mut layer_flush_start_rx: tokio::sync::watch::Receiver) { + async fn flush_loop( + &self, + mut layer_flush_start_rx: tokio::sync::watch::Receiver, + ctx: &RequestContext, + ) { info!("started flush loop"); loop { tokio::select! { @@ -1908,7 +2023,7 @@ impl Timeline { // drop 'layers' lock to allow concurrent reads and writes }; if let Some(layer_to_flush) = layer_to_flush { - if let Err(err) = self.flush_frozen_layer(layer_to_flush).await { + if let Err(err) = self.flush_frozen_layer(layer_to_flush, ctx).await { error!("could not flush frozen layer: {err:?}"); break Err(err); } @@ -1973,8 +2088,12 @@ impl Timeline { } /// Flush one frozen in-memory layer to disk, as a new delta layer. - #[instrument(skip(self, frozen_layer), fields(tenant_id=%self.tenant_id, timeline_id=%self.timeline_id, layer=%frozen_layer.short_id()))] - async fn flush_frozen_layer(&self, frozen_layer: Arc) -> anyhow::Result<()> { + #[instrument(skip(self, frozen_layer, ctx), fields(tenant_id=%self.tenant_id, timeline_id=%self.timeline_id, layer=%frozen_layer.short_id()))] + async fn flush_frozen_layer( + &self, + frozen_layer: Arc, + ctx: &RequestContext, + ) -> anyhow::Result<()> { // As a special case, when we have just imported an image into the repository, // instead of writing out a L0 delta layer, we directly write out image layer // files instead. This is possible as long as *all* the data imported into the @@ -1982,10 +2101,12 @@ impl Timeline { let lsn_range = frozen_layer.get_lsn_range(); let layer_paths_to_upload = if lsn_range.start == self.initdb_lsn && lsn_range.end == Lsn(self.initdb_lsn.0 + 1) { + // Note: The 'ctx' in use here has DownloadBehavior::Error. We should not + // require downloading anything during initial import. let (partitioning, _lsn) = self - .repartition(self.initdb_lsn, self.get_compaction_target_size()) + .repartition(self.initdb_lsn, self.get_compaction_target_size(), ctx) .await?; - self.create_image_layers(&partitioning, self.initdb_lsn, true) + self.create_image_layers(&partitioning, self.initdb_lsn, true, ctx) .await? } else { // normal case, write out a L0 delta layer file. @@ -2136,6 +2257,7 @@ impl Timeline { &self, lsn: Lsn, partition_size: u64, + ctx: &RequestContext, ) -> anyhow::Result<(KeyPartitioning, Lsn)> { { let partitioning_guard = self.partitioning.lock().unwrap(); @@ -2146,7 +2268,7 @@ impl Timeline { return Ok((partitioning_guard.0.clone(), partitioning_guard.1)); } } - let keyspace = self.collect_keyspace(lsn).await?; + let keyspace = self.collect_keyspace(lsn, ctx).await?; let partitioning = keyspace.partition(partition_size); let mut partitioning_guard = self.partitioning.lock().unwrap(); @@ -2206,6 +2328,7 @@ impl Timeline { partitioning: &KeyPartitioning, lsn: Lsn, force: bool, + ctx: &RequestContext, ) -> Result, PageReconstructError> { let timer = self.metrics.create_images_time_histo.start_timer(); let mut image_layers: Vec = Vec::new(); @@ -2230,7 +2353,7 @@ impl Timeline { for range in &partition.ranges { let mut key = range.start; while key < range.end { - let img = match self.get(key, lsn).await { + let img = match self.get(key, lsn, ctx).await { Ok(img) => img, Err(err) => { // If we fail to reconstruct a VM or FSM page, we can zero the @@ -2685,6 +2808,7 @@ impl Timeline { retain_lsns: Vec, cutoff_horizon: Lsn, pitr: Duration, + ctx: &RequestContext, ) -> anyhow::Result<()> { // First, calculate pitr_cutoff_timestamp and then convert it to LSN. // @@ -2697,7 +2821,7 @@ impl Timeline { if let Some(pitr_cutoff_timestamp) = now.checked_sub(pitr) { let pitr_timestamp = to_pg_timestamp(pitr_cutoff_timestamp); - match self.find_lsn_for_timestamp(pitr_timestamp).await? { + match self.find_lsn_for_timestamp(pitr_timestamp, ctx).await? { LsnForTimestamp::Present(lsn) => lsn, LsnForTimestamp::Future(lsn) => { // The timestamp is in the future. That sounds impossible, diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 0de2e6654d..3761c65668 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -29,6 +29,7 @@ use anyhow::Result; use bytes::{Buf, Bytes, BytesMut}; use tracing::*; +use crate::context::RequestContext; use crate::pgdatadir_mapping::*; use crate::tenant::PageReconstructError; use crate::tenant::Timeline; @@ -52,10 +53,14 @@ pub struct WalIngest<'a> { } impl<'a> WalIngest<'a> { - pub async fn new(timeline: &Timeline, startpoint: Lsn) -> anyhow::Result { + pub async fn new( + timeline: &'a Timeline, + startpoint: Lsn, + ctx: &'_ RequestContext, + ) -> anyhow::Result> { // Fetch the latest checkpoint into memory, so that we can compare with it // quickly in `ingest_record` and update it when it changes. - let checkpoint_bytes = timeline.get_checkpoint(startpoint).await?; + let checkpoint_bytes = timeline.get_checkpoint(startpoint, ctx).await?; let checkpoint = CheckPoint::decode(&checkpoint_bytes)?; trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value); @@ -80,6 +85,7 @@ impl<'a> WalIngest<'a> { lsn: Lsn, modification: &mut DatadirModification<'_>, decoded: &mut DecodedWALRecord, + ctx: &RequestContext, ) -> anyhow::Result<()> { modification.lsn = lsn; decode_wal_record(recdata, decoded, self.timeline.pg_version)?; @@ -97,7 +103,7 @@ impl<'a> WalIngest<'a> { if decoded.xl_rmid == pg_constants::RM_HEAP_ID || decoded.xl_rmid == pg_constants::RM_HEAP2_ID { - self.ingest_heapam_record(&mut buf, modification, decoded) + self.ingest_heapam_record(&mut buf, modification, decoded, ctx) .await?; } // Handle other special record types @@ -106,13 +112,14 @@ impl<'a> WalIngest<'a> { == pg_constants::XLOG_SMGR_CREATE { let create = XlSmgrCreate::decode(&mut buf); - self.ingest_xlog_smgr_create(modification, &create).await?; + self.ingest_xlog_smgr_create(modification, &create, ctx) + .await?; } else if decoded.xl_rmid == pg_constants::RM_SMGR_ID && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == pg_constants::XLOG_SMGR_TRUNCATE { let truncate = XlSmgrTruncate::decode(&mut buf); - self.ingest_xlog_smgr_truncate(modification, &truncate) + self.ingest_xlog_smgr_truncate(modification, &truncate, ctx) .await?; } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID { debug!( @@ -126,7 +133,7 @@ impl<'a> WalIngest<'a> { let createdb = XlCreateDatabase::decode(&mut buf); debug!("XLOG_DBASE_CREATE v14"); - self.ingest_xlog_dbase_create(modification, &createdb) + self.ingest_xlog_dbase_create(modification, &createdb, ctx) .await?; } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == postgres_ffi::v14::bindings::XLOG_DBASE_DROP @@ -134,7 +141,9 @@ impl<'a> WalIngest<'a> { let dropdb = XlDropDatabase::decode(&mut buf); for tablespace_id in dropdb.tablespace_ids { trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); - modification.drop_dbdir(tablespace_id, dropdb.db_id).await?; + modification + .drop_dbdir(tablespace_id, dropdb.db_id, ctx) + .await?; } } } else if self.timeline.pg_version == 15 { @@ -150,7 +159,7 @@ impl<'a> WalIngest<'a> { // So we can reuse XlCreateDatabase here. debug!("XLOG_DBASE_CREATE_FILE_COPY"); let createdb = XlCreateDatabase::decode(&mut buf); - self.ingest_xlog_dbase_create(modification, &createdb) + self.ingest_xlog_dbase_create(modification, &createdb, ctx) .await?; } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) == postgres_ffi::v15::bindings::XLOG_DBASE_DROP @@ -158,7 +167,9 @@ impl<'a> WalIngest<'a> { let dropdb = XlDropDatabase::decode(&mut buf); for tablespace_id in dropdb.tablespace_ids { trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); - modification.drop_dbdir(tablespace_id, dropdb.db_id).await?; + modification + .drop_dbdir(tablespace_id, dropdb.db_id, ctx) + .await?; } } } @@ -176,12 +187,13 @@ impl<'a> WalIngest<'a> { segno, rpageno, ZERO_PAGE.clone(), + ctx, ) .await?; } else { assert!(info == pg_constants::CLOG_TRUNCATE); let xlrec = XlClogTruncate::decode(&mut buf); - self.ingest_clog_truncate_record(modification, &xlrec) + self.ingest_clog_truncate_record(modification, &xlrec, ctx) .await?; } } else if decoded.xl_rmid == pg_constants::RM_XACT_ID { @@ -193,6 +205,7 @@ impl<'a> WalIngest<'a> { modification, &parsed_xact, info == pg_constants::XLOG_XACT_COMMIT, + ctx, ) .await?; } else if info == pg_constants::XLOG_XACT_COMMIT_PREPARED @@ -204,6 +217,7 @@ impl<'a> WalIngest<'a> { modification, &parsed_xact, info == pg_constants::XLOG_XACT_COMMIT_PREPARED, + ctx, ) .await?; // Remove twophase file. see RemoveTwoPhaseFile() in postgres code @@ -213,10 +227,12 @@ impl<'a> WalIngest<'a> { parsed_xact.xid, lsn, ); - modification.drop_twophase_file(parsed_xact.xid).await?; + modification + .drop_twophase_file(parsed_xact.xid, ctx) + .await?; } else if info == pg_constants::XLOG_XACT_PREPARE { modification - .put_twophase_file(decoded.xl_xid, Bytes::copy_from_slice(&buf[..])) + .put_twophase_file(decoded.xl_xid, Bytes::copy_from_slice(&buf[..]), ctx) .await?; } } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID { @@ -232,6 +248,7 @@ impl<'a> WalIngest<'a> { segno, rpageno, ZERO_PAGE.clone(), + ctx, ) .await?; } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE { @@ -244,6 +261,7 @@ impl<'a> WalIngest<'a> { segno, rpageno, ZERO_PAGE.clone(), + ctx, ) .await?; } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID { @@ -251,12 +269,12 @@ impl<'a> WalIngest<'a> { self.ingest_multixact_create_record(modification, &xlrec)?; } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID { let xlrec = XlMultiXactTruncate::decode(&mut buf); - self.ingest_multixact_truncate_record(modification, &xlrec) + self.ingest_multixact_truncate_record(modification, &xlrec, ctx) .await?; } } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID { let xlrec = XlRelmapUpdate::decode(&mut buf); - self.ingest_relmap_page(modification, &xlrec, decoded) + self.ingest_relmap_page(modification, &xlrec, decoded, ctx) .await?; } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID { let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; @@ -292,7 +310,7 @@ impl<'a> WalIngest<'a> { // Iterate through all the blocks that the record modifies, and // "put" a separate copy of the record for each block. for blk in decoded.blocks.iter() { - self.ingest_decoded_block(modification, lsn, decoded, blk) + self.ingest_decoded_block(modification, lsn, decoded, blk, ctx) .await?; } @@ -317,6 +335,7 @@ impl<'a> WalIngest<'a> { lsn: Lsn, decoded: &DecodedWALRecord, blk: &DecodedBkpBlock, + ctx: &RequestContext, ) -> Result<(), PageReconstructError> { let rel = RelTag { spcnode: blk.rnode_spcnode, @@ -359,14 +378,14 @@ impl<'a> WalIngest<'a> { page_set_lsn(&mut image, lsn) } assert_eq!(image.len(), BLCKSZ as usize); - self.put_rel_page_image(modification, rel, blk.blkno, image.freeze()) + self.put_rel_page_image(modification, rel, blk.blkno, image.freeze(), ctx) .await?; } else { let rec = NeonWalRecord::Postgres { will_init: blk.will_init || blk.apply_image, rec: decoded.record.clone(), }; - self.put_rel_wal_record(modification, rel, blk.blkno, rec) + self.put_rel_wal_record(modification, rel, blk.blkno, rec, ctx) .await?; } Ok(()) @@ -377,6 +396,7 @@ impl<'a> WalIngest<'a> { buf: &mut Bytes, modification: &mut DatadirModification<'_>, decoded: &mut DecodedWALRecord, + ctx: &RequestContext, ) -> anyhow::Result<()> { // Handle VM bit updates that are implicitly part of heap records. @@ -456,7 +476,7 @@ impl<'a> WalIngest<'a> { // replaying it would fail to find the previous image of the page, because // it doesn't exist. So check if the VM page(s) exist, and skip the WAL // record if it doesn't. - let vm_size = self.get_relsize(vm_rel, modification.lsn).await?; + let vm_size = self.get_relsize(vm_rel, modification.lsn, ctx).await?; if let Some(blknum) = new_vm_blk { if blknum >= vm_size { new_vm_blk = None; @@ -481,6 +501,7 @@ impl<'a> WalIngest<'a> { old_heap_blkno, flags: pg_constants::VISIBILITYMAP_VALID_BITS, }, + ctx, ) .await?; } else { @@ -496,6 +517,7 @@ impl<'a> WalIngest<'a> { old_heap_blkno: None, flags: pg_constants::VISIBILITYMAP_VALID_BITS, }, + ctx, ) .await?; } @@ -509,6 +531,7 @@ impl<'a> WalIngest<'a> { old_heap_blkno, flags: pg_constants::VISIBILITYMAP_VALID_BITS, }, + ctx, ) .await?; } @@ -524,6 +547,7 @@ impl<'a> WalIngest<'a> { &mut self, modification: &mut DatadirModification<'_>, rec: &XlCreateDatabase, + ctx: &RequestContext, ) -> anyhow::Result<()> { let db_id = rec.db_id; let tablespace_id = rec.tablespace_id; @@ -539,7 +563,7 @@ impl<'a> WalIngest<'a> { let rels = modification .tline - .list_rels(src_tablespace_id, src_db_id, req_lsn) + .list_rels(src_tablespace_id, src_db_id, req_lsn, ctx) .await?; debug!("ingest_xlog_dbase_create: {} rels", rels.len()); @@ -547,10 +571,10 @@ impl<'a> WalIngest<'a> { // Copy relfilemap let filemap = modification .tline - .get_relmap_file(src_tablespace_id, src_db_id, req_lsn) + .get_relmap_file(src_tablespace_id, src_db_id, req_lsn, ctx) .await?; modification - .put_relmap_file(tablespace_id, db_id, filemap) + .put_relmap_file(tablespace_id, db_id, filemap, ctx) .await?; let mut num_rels_copied = 0; @@ -561,7 +585,7 @@ impl<'a> WalIngest<'a> { let nblocks = modification .tline - .get_rel_size(src_rel, req_lsn, true) + .get_rel_size(src_rel, req_lsn, true, ctx) .await?; let dst_rel = RelTag { spcnode: tablespace_id, @@ -570,7 +594,7 @@ impl<'a> WalIngest<'a> { forknum: src_rel.forknum, }; - modification.put_rel_creation(dst_rel, nblocks).await?; + modification.put_rel_creation(dst_rel, nblocks, ctx).await?; // Copy content debug!("copying rel {} to {}, {} blocks", src_rel, dst_rel, nblocks); @@ -579,7 +603,7 @@ impl<'a> WalIngest<'a> { let content = modification .tline - .get_rel_page_at_lsn(src_rel, blknum, req_lsn, true) + .get_rel_page_at_lsn(src_rel, blknum, req_lsn, true, ctx) .await?; modification.put_rel_page_image(dst_rel, blknum, content)?; num_blocks_copied += 1; @@ -599,6 +623,7 @@ impl<'a> WalIngest<'a> { &mut self, modification: &mut DatadirModification<'_>, rec: &XlSmgrCreate, + ctx: &RequestContext, ) -> anyhow::Result<()> { let rel = RelTag { spcnode: rec.rnode.spcnode, @@ -606,7 +631,7 @@ impl<'a> WalIngest<'a> { relnode: rec.rnode.relnode, forknum: rec.forknum, }; - self.put_rel_creation(modification, rel).await?; + self.put_rel_creation(modification, rel, ctx).await?; Ok(()) } @@ -617,6 +642,7 @@ impl<'a> WalIngest<'a> { &mut self, modification: &mut DatadirModification<'_>, rec: &XlSmgrTruncate, + ctx: &RequestContext, ) -> anyhow::Result<()> { let spcnode = rec.rnode.spcnode; let dbnode = rec.rnode.dbnode; @@ -629,7 +655,7 @@ impl<'a> WalIngest<'a> { relnode, forknum: MAIN_FORKNUM, }; - self.put_rel_truncation(modification, rel, rec.blkno) + self.put_rel_truncation(modification, rel, rec.blkno, ctx) .await?; } if (rec.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 { @@ -648,10 +674,10 @@ impl<'a> WalIngest<'a> { modification.put_rel_page_image(rel, fsm_physical_page_no, ZERO_PAGE.clone())?; fsm_physical_page_no += 1; } - let nblocks = self.get_relsize(rel, modification.lsn).await?; + let nblocks = self.get_relsize(rel, modification.lsn, ctx).await?; if nblocks > fsm_physical_page_no { // check if something to do: FSM is larger than truncate position - self.put_rel_truncation(modification, rel, fsm_physical_page_no) + self.put_rel_truncation(modification, rel, fsm_physical_page_no, ctx) .await?; } } @@ -670,10 +696,10 @@ impl<'a> WalIngest<'a> { modification.put_rel_page_image(rel, vm_page_no, ZERO_PAGE.clone())?; vm_page_no += 1; } - let nblocks = self.get_relsize(rel, modification.lsn).await?; + let nblocks = self.get_relsize(rel, modification.lsn, ctx).await?; if nblocks > vm_page_no { // check if something to do: VM is larger than truncate position - self.put_rel_truncation(modification, rel, vm_page_no) + self.put_rel_truncation(modification, rel, vm_page_no, ctx) .await?; } } @@ -687,6 +713,7 @@ impl<'a> WalIngest<'a> { modification: &mut DatadirModification<'_>, parsed: &XlXactParsedRecord, is_commit: bool, + ctx: &RequestContext, ) -> anyhow::Result<()> { // Record update of CLOG pages let mut pageno = parsed.xid / pg_constants::CLOG_XACTS_PER_PAGE; @@ -745,10 +772,10 @@ impl<'a> WalIngest<'a> { let last_lsn = self.timeline.get_last_record_lsn(); if modification .tline - .get_rel_exists(rel, last_lsn, true) + .get_rel_exists(rel, last_lsn, true, ctx) .await? { - self.put_rel_drop(modification, rel).await?; + self.put_rel_drop(modification, rel, ctx).await?; } } } @@ -759,6 +786,7 @@ impl<'a> WalIngest<'a> { &mut self, modification: &mut DatadirModification<'_>, xlrec: &XlClogTruncate, + ctx: &RequestContext, ) -> anyhow::Result<()> { info!( "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {}", @@ -799,16 +827,15 @@ impl<'a> WalIngest<'a> { // it. So we use the previous record's LSN in the get calls // instead. let req_lsn = modification.tline.get_last_record_lsn(); - - let slru_segments = modification + for segno in modification .tline - .list_slru_segments(SlruKind::Clog, req_lsn) - .await?; - for segno in slru_segments { + .list_slru_segments(SlruKind::Clog, req_lsn, ctx) + .await? + { let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT; if slru_may_delete_clogsegment(segpage, xlrec.pageno) { modification - .drop_slru_segment(SlruKind::Clog, segno) + .drop_slru_segment(SlruKind::Clog, segno, ctx) .await?; trace!("Drop CLOG segment {:>04X}", segno); } @@ -900,6 +927,7 @@ impl<'a> WalIngest<'a> { &mut self, modification: &mut DatadirModification<'_>, xlrec: &XlMultiXactTruncate, + ctx: &RequestContext, ) -> Result<()> { self.checkpoint.oldestMulti = xlrec.end_trunc_off; self.checkpoint.oldestMultiDB = xlrec.oldest_multi_db; @@ -915,7 +943,7 @@ impl<'a> WalIngest<'a> { // contain, possibly partially, valid data. while segment != endsegment { modification - .drop_slru_segment(SlruKind::MultiXactMembers, segment as u32) + .drop_slru_segment(SlruKind::MultiXactMembers, segment as u32, ctx) .await?; /* move to next segment, handling wraparound correctly */ @@ -937,6 +965,7 @@ impl<'a> WalIngest<'a> { modification: &mut DatadirModification<'_>, xlrec: &XlRelmapUpdate, decoded: &DecodedWALRecord, + ctx: &RequestContext, ) -> Result<()> { let mut buf = decoded.record.clone(); buf.advance(decoded.main_data_offset); @@ -944,18 +973,22 @@ impl<'a> WalIngest<'a> { buf.advance(12); modification - .put_relmap_file(xlrec.tsid, xlrec.dbid, Bytes::copy_from_slice(&buf[..])) - .await?; - - Ok(()) + .put_relmap_file( + xlrec.tsid, + xlrec.dbid, + Bytes::copy_from_slice(&buf[..]), + ctx, + ) + .await } async fn put_rel_creation( &mut self, modification: &mut DatadirModification<'_>, rel: RelTag, + ctx: &RequestContext, ) -> Result<()> { - modification.put_rel_creation(rel, 0).await?; + modification.put_rel_creation(rel, 0, ctx).await?; Ok(()) } @@ -965,8 +998,10 @@ impl<'a> WalIngest<'a> { rel: RelTag, blknum: BlockNumber, img: Bytes, - ) -> anyhow::Result<()> { - self.handle_rel_extend(modification, rel, blknum).await?; + ctx: &RequestContext, + ) -> Result<(), PageReconstructError> { + self.handle_rel_extend(modification, rel, blknum, ctx) + .await?; modification.put_rel_page_image(rel, blknum, img)?; Ok(()) } @@ -977,8 +1012,10 @@ impl<'a> WalIngest<'a> { rel: RelTag, blknum: BlockNumber, rec: NeonWalRecord, - ) -> anyhow::Result<()> { - self.handle_rel_extend(modification, rel, blknum).await?; + ctx: &RequestContext, + ) -> Result<()> { + self.handle_rel_extend(modification, rel, blknum, ctx) + .await?; modification.put_rel_wal_record(rel, blknum, rec)?; Ok(()) } @@ -988,8 +1025,9 @@ impl<'a> WalIngest<'a> { modification: &mut DatadirModification<'_>, rel: RelTag, nblocks: BlockNumber, + ctx: &RequestContext, ) -> anyhow::Result<()> { - modification.put_rel_truncation(rel, nblocks).await?; + modification.put_rel_truncation(rel, nblocks, ctx).await?; Ok(()) } @@ -997,17 +1035,22 @@ impl<'a> WalIngest<'a> { &mut self, modification: &mut DatadirModification<'_>, rel: RelTag, + ctx: &RequestContext, ) -> Result<()> { - modification.put_rel_drop(rel).await?; + modification.put_rel_drop(rel, ctx).await?; Ok(()) } - async fn get_relsize(&mut self, rel: RelTag, lsn: Lsn) -> anyhow::Result { - let exists = self.timeline.get_rel_exists(rel, lsn, true).await?; - let nblocks = if !exists { + async fn get_relsize( + &mut self, + rel: RelTag, + lsn: Lsn, + ctx: &RequestContext, + ) -> anyhow::Result { + let nblocks = if !self.timeline.get_rel_exists(rel, lsn, true, ctx).await? { 0 } else { - self.timeline.get_rel_size(rel, lsn, true).await? + self.timeline.get_rel_size(rel, lsn, true, ctx).await? }; Ok(nblocks) } @@ -1017,23 +1060,28 @@ impl<'a> WalIngest<'a> { modification: &mut DatadirModification<'_>, rel: RelTag, blknum: BlockNumber, - ) -> anyhow::Result<()> { + ctx: &RequestContext, + ) -> Result<(), PageReconstructError> { let new_nblocks = blknum + 1; // Check if the relation exists. We implicitly create relations on first // record. // TODO: would be nice if to be more explicit about it let last_lsn = modification.lsn; - let old_nblocks = if !self.timeline.get_rel_exists(rel, last_lsn, true).await? { + let old_nblocks = if !self + .timeline + .get_rel_exists(rel, last_lsn, true, ctx) + .await? + { // create it with 0 size initially, the logic below will extend it - modification.put_rel_creation(rel, 0).await?; + modification.put_rel_creation(rel, 0, ctx).await?; 0 } else { - self.timeline.get_rel_size(rel, last_lsn, true).await? + self.timeline.get_rel_size(rel, last_lsn, true, ctx).await? }; if new_nblocks > old_nblocks { //info!("extending {} {} to {}", rel, old_nblocks, new_nblocks); - modification.put_rel_extend(rel, new_nblocks).await?; + modification.put_rel_extend(rel, new_nblocks, ctx).await?; // fill the gap with zeros for gap_blknum in old_nblocks..blknum { @@ -1050,8 +1098,9 @@ impl<'a> WalIngest<'a> { segno: u32, blknum: BlockNumber, img: Bytes, - ) -> anyhow::Result<()> { - self.handle_slru_extend(modification, kind, segno, blknum) + ctx: &RequestContext, + ) -> Result<()> { + self.handle_slru_extend(modification, kind, segno, blknum, ctx) .await?; modification.put_slru_page_image(kind, segno, blknum, img)?; Ok(()) @@ -1063,6 +1112,7 @@ impl<'a> WalIngest<'a> { kind: SlruKind, segno: u32, blknum: BlockNumber, + ctx: &RequestContext, ) -> anyhow::Result<()> { // we don't use a cache for this like we do for relations. SLRUS are explcitly // extended with ZEROPAGE records, not with commit records, so it happens @@ -1075,17 +1125,17 @@ impl<'a> WalIngest<'a> { let last_lsn = self.timeline.get_last_record_lsn(); let old_nblocks = if !self .timeline - .get_slru_segment_exists(kind, segno, last_lsn) + .get_slru_segment_exists(kind, segno, last_lsn, ctx) .await? { // create it with 0 size initially, the logic below will extend it modification - .put_slru_segment_creation(kind, segno, 0) + .put_slru_segment_creation(kind, segno, 0, ctx) .await?; 0 } else { self.timeline - .get_slru_segment_size(kind, segno, last_lsn) + .get_slru_segment_size(kind, segno, last_lsn, ctx) .await? }; @@ -1134,41 +1184,44 @@ mod tests { static ZERO_CHECKPOINT: Bytes = Bytes::from_static(&[0u8; SIZEOF_CHECKPOINT]); - async fn init_walingest_test(tline: &Timeline) -> Result { + async fn init_walingest_test<'a>( + tline: &'a Timeline, + ctx: &RequestContext, + ) -> Result> { let mut m = tline.begin_modification(Lsn(0x10)); m.put_checkpoint(ZERO_CHECKPOINT.clone())?; - m.put_relmap_file(0, 111, Bytes::from("")).await?; // dummy relmapper file + m.put_relmap_file(0, 111, Bytes::from(""), ctx).await?; // dummy relmapper file m.commit()?; - let walingest = WalIngest::new(tline, Lsn(0x10)).await?; + let walingest = WalIngest::new(tline, Lsn(0x10), ctx).await?; Ok(walingest) } #[tokio::test] async fn test_relsize() -> Result<()> { - let tenant = TenantHarness::create("test_relsize")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; - let mut walingest = init_walingest_test(&tline).await?; + let (tenant, ctx) = TenantHarness::create("test_relsize")?.load().await; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut m = tline.begin_modification(Lsn(0x20)); - walingest.put_rel_creation(&mut m, TESTREL_A).await?; + walingest.put_rel_creation(&mut m, TESTREL_A, &ctx).await?; walingest - .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 2")) + .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 2"), &ctx) .await?; m.commit()?; let mut m = tline.begin_modification(Lsn(0x30)); walingest - .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 3")) + .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 3"), &ctx) .await?; m.commit()?; let mut m = tline.begin_modification(Lsn(0x40)); walingest - .put_rel_page_image(&mut m, TESTREL_A, 1, TEST_IMG("foo blk 1 at 4")) + .put_rel_page_image(&mut m, TESTREL_A, 1, TEST_IMG("foo blk 1 at 4"), &ctx) .await?; m.commit()?; let mut m = tline.begin_modification(Lsn(0x50)); walingest - .put_rel_page_image(&mut m, TESTREL_A, 2, TEST_IMG("foo blk 2 at 5")) + .put_rel_page_image(&mut m, TESTREL_A, 2, TEST_IMG("foo blk 2 at 5"), &ctx) .await?; m.commit()?; @@ -1176,120 +1229,157 @@ mod tests { // The relation was created at LSN 2, not visible at LSN 1 yet. assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x10), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x10), false, &ctx) + .await?, false ); assert!(tline - .get_rel_size(TESTREL_A, Lsn(0x10), false) + .get_rel_size(TESTREL_A, Lsn(0x10), false, &ctx) .await .is_err()); - assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x20), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x20), false, &ctx) + .await?, true ); - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x20), false).await?, 1); - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x50), false).await?, 3); + assert_eq!( + tline + .get_rel_size(TESTREL_A, Lsn(0x20), false, &ctx) + .await?, + 1 + ); + assert_eq!( + tline + .get_rel_size(TESTREL_A, Lsn(0x50), false, &ctx) + .await?, + 3 + ); // Check page contents at each LSN assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x20), false) + .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x20), false, &ctx) .await?, TEST_IMG("foo blk 0 at 2") ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x30), false) + .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x30), false, &ctx) .await?, TEST_IMG("foo blk 0 at 3") ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x40), false) + .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x40), false, &ctx) .await?, TEST_IMG("foo blk 0 at 3") ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x40), false) + .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x40), false, &ctx) .await?, TEST_IMG("foo blk 1 at 4") ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x50), false) + .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x50), false, &ctx) .await?, TEST_IMG("foo blk 0 at 3") ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x50), false) + .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x50), false, &ctx) .await?, TEST_IMG("foo blk 1 at 4") ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 2, Lsn(0x50), false) + .get_rel_page_at_lsn(TESTREL_A, 2, Lsn(0x50), false, &ctx) .await?, TEST_IMG("foo blk 2 at 5") ); // Truncate last block let mut m = tline.begin_modification(Lsn(0x60)); - walingest.put_rel_truncation(&mut m, TESTREL_A, 2).await?; + walingest + .put_rel_truncation(&mut m, TESTREL_A, 2, &ctx) + .await?; m.commit()?; assert_current_logical_size(&tline, Lsn(0x60)); // Check reported size and contents after truncation - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x60), false).await?, 2); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x60), false) + .get_rel_size(TESTREL_A, Lsn(0x60), false, &ctx) + .await?, + 2 + ); + assert_eq!( + tline + .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x60), false, &ctx) .await?, TEST_IMG("foo blk 0 at 3") ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x60), false) + .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x60), false, &ctx) .await?, TEST_IMG("foo blk 1 at 4") ); // should still see the truncated block with older LSN - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x50), false).await?, 3); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 2, Lsn(0x50), false) + .get_rel_size(TESTREL_A, Lsn(0x50), false, &ctx) + .await?, + 3 + ); + assert_eq!( + tline + .get_rel_page_at_lsn(TESTREL_A, 2, Lsn(0x50), false, &ctx) .await?, TEST_IMG("foo blk 2 at 5") ); // Truncate to zero length let mut m = tline.begin_modification(Lsn(0x68)); - walingest.put_rel_truncation(&mut m, TESTREL_A, 0).await?; + walingest + .put_rel_truncation(&mut m, TESTREL_A, 0, &ctx) + .await?; m.commit()?; - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x68), false).await?, 0); + assert_eq!( + tline + .get_rel_size(TESTREL_A, Lsn(0x68), false, &ctx) + .await?, + 0 + ); // Extend from 0 to 2 blocks, leaving a gap let mut m = tline.begin_modification(Lsn(0x70)); walingest - .put_rel_page_image(&mut m, TESTREL_A, 1, TEST_IMG("foo blk 1")) + .put_rel_page_image(&mut m, TESTREL_A, 1, TEST_IMG("foo blk 1"), &ctx) .await?; m.commit()?; - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x70), false).await?, 2); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x70), false) + .get_rel_size(TESTREL_A, Lsn(0x70), false, &ctx) + .await?, + 2 + ); + assert_eq!( + tline + .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x70), false, &ctx) .await?, ZERO_PAGE ); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x70), false) + .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x70), false, &ctx) .await?, TEST_IMG("foo blk 1") ); @@ -1297,21 +1387,26 @@ mod tests { // Extend a lot more, leaving a big gap that spans across segments let mut m = tline.begin_modification(Lsn(0x80)); walingest - .put_rel_page_image(&mut m, TESTREL_A, 1500, TEST_IMG("foo blk 1500")) + .put_rel_page_image(&mut m, TESTREL_A, 1500, TEST_IMG("foo blk 1500"), &ctx) .await?; m.commit()?; - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x80), false).await?, 1501); + assert_eq!( + tline + .get_rel_size(TESTREL_A, Lsn(0x80), false, &ctx) + .await?, + 1501 + ); for blk in 2..1500 { assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, blk, Lsn(0x80), false) + .get_rel_page_at_lsn(TESTREL_A, blk, Lsn(0x80), false, &ctx) .await?, ZERO_PAGE ); } assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, 1500, Lsn(0x80), false) + .get_rel_page_at_lsn(TESTREL_A, 1500, Lsn(0x80), false, &ctx) .await?, TEST_IMG("foo blk 1500") ); @@ -1323,31 +1418,40 @@ mod tests { // and then created it again within the same layer. #[tokio::test] async fn test_drop_extend() -> Result<()> { - let tenant = TenantHarness::create("test_drop_extend")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; - let mut walingest = init_walingest_test(&tline).await?; + let (tenant, ctx) = TenantHarness::create("test_drop_extend")?.load().await; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut m = tline.begin_modification(Lsn(0x20)); walingest - .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 2")) + .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 2"), &ctx) .await?; m.commit()?; // Check that rel exists and size is correct assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x20), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x20), false, &ctx) + .await?, true ); - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x20), false).await?, 1); + assert_eq!( + tline + .get_rel_size(TESTREL_A, Lsn(0x20), false, &ctx) + .await?, + 1 + ); // Drop rel let mut m = tline.begin_modification(Lsn(0x30)); - walingest.put_rel_drop(&mut m, TESTREL_A).await?; + walingest.put_rel_drop(&mut m, TESTREL_A, &ctx).await?; m.commit()?; // Check that rel is not visible anymore assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x30), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x30), false, &ctx) + .await?, false ); @@ -1357,16 +1461,23 @@ mod tests { // Re-create it let mut m = tline.begin_modification(Lsn(0x40)); walingest - .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 4")) + .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 4"), &ctx) .await?; m.commit()?; // Check that rel exists and size is correct assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x40), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x40), false, &ctx) + .await?, true ); - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x40), false).await?, 1); + assert_eq!( + tline + .get_rel_size(TESTREL_A, Lsn(0x40), false, &ctx) + .await?, + 1 + ); Ok(()) } @@ -1376,9 +1487,9 @@ mod tests { // and then extended it again within the same layer. #[tokio::test] async fn test_truncate_extend() -> Result<()> { - let tenant = TenantHarness::create("test_truncate_extend")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; - let mut walingest = init_walingest_test(&tline).await?; + let (tenant, ctx) = TenantHarness::create("test_truncate_extend")?.load().await; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let mut walingest = init_walingest_test(&tline, &ctx).await?; // Create a 20 MB relation (the size is arbitrary) let relsize = 20 * 1024 * 1024 / 8192; @@ -1386,27 +1497,33 @@ mod tests { for blkno in 0..relsize { let data = format!("foo blk {} at {}", blkno, Lsn(0x20)); walingest - .put_rel_page_image(&mut m, TESTREL_A, blkno, TEST_IMG(&data)) + .put_rel_page_image(&mut m, TESTREL_A, blkno, TEST_IMG(&data), &ctx) .await?; } m.commit()?; // The relation was created at LSN 20, not visible at LSN 1 yet. assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x10), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x10), false, &ctx) + .await?, false ); assert!(tline - .get_rel_size(TESTREL_A, Lsn(0x10), false) + .get_rel_size(TESTREL_A, Lsn(0x10), false, &ctx) .await .is_err()); assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x20), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x20), false, &ctx) + .await?, true ); assert_eq!( - tline.get_rel_size(TESTREL_A, Lsn(0x20), false).await?, + tline + .get_rel_size(TESTREL_A, Lsn(0x20), false, &ctx) + .await?, relsize ); @@ -1416,7 +1533,7 @@ mod tests { let data = format!("foo blk {} at {}", blkno, lsn); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, blkno, lsn, false) + .get_rel_page_at_lsn(TESTREL_A, blkno, lsn, false, &ctx) .await?, TEST_IMG(&data) ); @@ -1425,18 +1542,25 @@ mod tests { // Truncate relation so that second segment was dropped // - only leave one page let mut m = tline.begin_modification(Lsn(0x60)); - walingest.put_rel_truncation(&mut m, TESTREL_A, 1).await?; + walingest + .put_rel_truncation(&mut m, TESTREL_A, 1, &ctx) + .await?; m.commit()?; // Check reported size and contents after truncation - assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x60), false).await?, 1); + assert_eq!( + tline + .get_rel_size(TESTREL_A, Lsn(0x60), false, &ctx) + .await?, + 1 + ); for blkno in 0..1 { let lsn = Lsn(0x20); let data = format!("foo blk {} at {}", blkno, lsn); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x60), false) + .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x60), false, &ctx) .await?, TEST_IMG(&data) ); @@ -1444,7 +1568,9 @@ mod tests { // should still see all blocks with older LSN assert_eq!( - tline.get_rel_size(TESTREL_A, Lsn(0x50), false).await?, + tline + .get_rel_size(TESTREL_A, Lsn(0x50), false, &ctx) + .await?, relsize ); for blkno in 0..relsize { @@ -1452,7 +1578,7 @@ mod tests { let data = format!("foo blk {} at {}", blkno, lsn); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x50), false) + .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x50), false, &ctx) .await?, TEST_IMG(&data) ); @@ -1465,17 +1591,21 @@ mod tests { for blkno in 0..relsize { let data = format!("foo blk {} at {}", blkno, lsn); walingest - .put_rel_page_image(&mut m, TESTREL_A, blkno, TEST_IMG(&data)) + .put_rel_page_image(&mut m, TESTREL_A, blkno, TEST_IMG(&data), &ctx) .await?; } m.commit()?; assert_eq!( - tline.get_rel_exists(TESTREL_A, Lsn(0x80), false).await?, + tline + .get_rel_exists(TESTREL_A, Lsn(0x80), false, &ctx) + .await?, true ); assert_eq!( - tline.get_rel_size(TESTREL_A, Lsn(0x80), false).await?, + tline + .get_rel_size(TESTREL_A, Lsn(0x80), false, &ctx) + .await?, relsize ); // Check relation content @@ -1484,7 +1614,7 @@ mod tests { let data = format!("foo blk {} at {}", blkno, lsn); assert_eq!( tline - .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x80), false) + .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x80), false, &ctx) .await?, TEST_IMG(&data) ); @@ -1497,9 +1627,9 @@ mod tests { /// split into multiple 1 GB segments in Postgres. #[tokio::test] async fn test_large_rel() -> Result<()> { - let tenant = TenantHarness::create("test_large_rel")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION)?; - let mut walingest = init_walingest_test(&tline).await?; + let (tenant, ctx) = TenantHarness::create("test_large_rel")?.load().await; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut lsn = 0x10; for blknum in 0..RELSEG_SIZE + 1 { @@ -1507,7 +1637,7 @@ mod tests { let mut m = tline.begin_modification(Lsn(lsn)); let img = TEST_IMG(&format!("foo blk {} at {}", blknum, Lsn(lsn))); walingest - .put_rel_page_image(&mut m, TESTREL_A, blknum as BlockNumber, img) + .put_rel_page_image(&mut m, TESTREL_A, blknum as BlockNumber, img, &ctx) .await?; m.commit()?; } @@ -1515,7 +1645,7 @@ mod tests { assert_current_logical_size(&tline, Lsn(lsn)); assert_eq!( - tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?, + tline.get_rel_size(TESTREL_A, Lsn(lsn), false, &ctx).await?, RELSEG_SIZE + 1 ); @@ -1523,11 +1653,11 @@ mod tests { lsn += 0x10; let mut m = tline.begin_modification(Lsn(lsn)); walingest - .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE) + .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE, &ctx) .await?; m.commit()?; assert_eq!( - tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?, + tline.get_rel_size(TESTREL_A, Lsn(lsn), false, &ctx).await?, RELSEG_SIZE ); assert_current_logical_size(&tline, Lsn(lsn)); @@ -1536,11 +1666,11 @@ mod tests { lsn += 0x10; let mut m = tline.begin_modification(Lsn(lsn)); walingest - .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE - 1) + .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE - 1, &ctx) .await?; m.commit()?; assert_eq!( - tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?, + tline.get_rel_size(TESTREL_A, Lsn(lsn), false, &ctx).await?, RELSEG_SIZE - 1 ); assert_current_logical_size(&tline, Lsn(lsn)); @@ -1552,11 +1682,11 @@ mod tests { lsn += 0x10; let mut m = tline.begin_modification(Lsn(lsn)); walingest - .put_rel_truncation(&mut m, TESTREL_A, size as BlockNumber) + .put_rel_truncation(&mut m, TESTREL_A, size as BlockNumber, &ctx) .await?; m.commit()?; assert_eq!( - tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?, + tline.get_rel_size(TESTREL_A, Lsn(lsn), false, &ctx).await?, size as BlockNumber ); diff --git a/pageserver/src/walreceiver/connection_manager.rs b/pageserver/src/walreceiver/connection_manager.rs index 2214cd79dd..3af408c61b 100644 --- a/pageserver/src/walreceiver/connection_manager.rs +++ b/pageserver/src/walreceiver/connection_manager.rs @@ -11,6 +11,7 @@ use std::{collections::HashMap, num::NonZeroU64, ops::ControlFlow, sync::Arc, time::Duration}; +use crate::context::RequestContext; use crate::task_mgr::TaskKind; use crate::task_mgr::WALRECEIVER_RUNTIME; use crate::tenant::Timeline; @@ -46,6 +47,7 @@ pub fn spawn_connection_manager_task( lagging_wal_timeout: Duration, max_lsn_wal_lag: NonZeroU64, auth_token: Option>, + ctx: RequestContext, ) { let mut broker_client = get_broker_client().clone(); @@ -78,6 +80,7 @@ pub fn spawn_connection_manager_task( loop_step_result = connection_manager_loop_step( &mut broker_client, &mut walreceiver_state, + &ctx, ) => match loop_step_result { ControlFlow::Continue(()) => continue, ControlFlow::Break(()) => { @@ -101,6 +104,7 @@ pub fn spawn_connection_manager_task( async fn connection_manager_loop_step( broker_client: &mut BrokerClientChannel, walreceiver_state: &mut WalreceiverState, + ctx: &RequestContext, ) -> ControlFlow<(), ()> { let mut timeline_state_updates = walreceiver_state.timeline.subscribe_for_state_updates(); @@ -236,6 +240,7 @@ async fn connection_manager_loop_step( .change_connection( new_candidate.safekeeper_id, new_candidate.wal_source_connconf, + ctx, ) .await } @@ -401,12 +406,17 @@ impl WalreceiverState { &mut self, new_sk_id: NodeId, new_wal_source_connconf: PgConnectionConfig, + ctx: &RequestContext, ) { self.drop_old_connection(true).await; let id = self.id; let connect_timeout = self.wal_connect_timeout; let timeline = Arc::clone(&self.timeline); + let ctx = ctx.detached_child( + TaskKind::WalReceiverConnectionHandler, + ctx.download_behavior(), + ); let connection_handle = TaskHandle::spawn(move |events_sender, cancellation| { async move { super::walreceiver_connection::handle_walreceiver_connection( @@ -415,6 +425,7 @@ impl WalreceiverState { events_sender, cancellation, connect_timeout, + ctx, ) .await .context("walreceiver connection handling failure") @@ -1245,18 +1256,18 @@ mod tests { const DUMMY_SAFEKEEPER_HOST: &str = "safekeeper_connstr"; async fn dummy_state(harness: &TenantHarness<'_>) -> WalreceiverState { + let (tenant, ctx) = harness.load().await; + let timeline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION, &ctx) + .expect("Failed to create an empty timeline for dummy wal connection manager"); + let timeline = timeline.initialize(&ctx).unwrap(); + WalreceiverState { id: TenantTimelineId { tenant_id: harness.tenant_id, timeline_id: TIMELINE_ID, }, - timeline: harness - .load() - .await - .create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION) - .expect("Failed to create an empty timeline for dummy wal connection manager") - .initialize() - .unwrap(), + timeline, wal_connect_timeout: Duration::from_secs(1), lagging_wal_timeout: Duration::from_secs(1), max_lsn_wal_lag: NonZeroU64::new(1024 * 1024).unwrap(), diff --git a/pageserver/src/walreceiver/walreceiver_connection.rs b/pageserver/src/walreceiver/walreceiver_connection.rs index 246bfb837a..1836195fd4 100644 --- a/pageserver/src/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/walreceiver/walreceiver_connection.rs @@ -22,6 +22,7 @@ use tokio_postgres::{replication::ReplicationStream, Client}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; +use crate::context::RequestContext; use crate::{metrics::LIVE_CONNECTIONS_COUNT, walreceiver::TaskStateUpdate}; use crate::{ task_mgr, @@ -62,6 +63,7 @@ pub async fn handle_walreceiver_connection( events_sender: watch::Sender>, cancellation: CancellationToken, connect_timeout: Duration, + ctx: RequestContext, ) -> anyhow::Result<()> { // Connect to the database in replication mode. info!("connecting to {wal_source_connconf:?}"); @@ -103,10 +105,14 @@ pub async fn handle_walreceiver_connection( // The connection object performs the actual communication with the database, // so spawn it off to run on its own. + let _connection_ctx = ctx.detached_child( + TaskKind::WalReceiverConnectionPoller, + ctx.download_behavior(), + ); let connection_cancellation = cancellation.clone(); task_mgr::spawn( WALRECEIVER_RUNTIME.handle(), - TaskKind::WalReceiverConnection, + TaskKind::WalReceiverConnectionPoller, Some(timeline.tenant_id), Some(timeline.timeline_id), "walreceiver connection", @@ -121,7 +127,7 @@ pub async fn handle_walreceiver_connection( } } }, - + // Future: replace connection_cancellation with connection_ctx cancellation _ = connection_cancellation.cancelled() => info!("Connection cancelled"), } Ok(()) @@ -184,7 +190,7 @@ pub async fn handle_walreceiver_connection( let mut waldecoder = WalStreamDecoder::new(startpoint, timeline.pg_version); - let mut walingest = WalIngest::new(timeline.as_ref(), startpoint).await?; + let mut walingest = WalIngest::new(timeline.as_ref(), startpoint, &ctx).await?; while let Some(replication_message) = { select! { @@ -255,7 +261,7 @@ pub async fn handle_walreceiver_connection( ensure!(lsn.is_aligned()); walingest - .ingest_record(recdata.clone(), lsn, &mut modification, &mut decoded) + .ingest_record(recdata, lsn, &mut modification, &mut decoded, &ctx) .await .with_context(|| format!("could not ingest record at {lsn}"))?; @@ -333,7 +339,7 @@ pub async fn handle_walreceiver_connection( // Send the replication feedback message. // Regular standby_status_update fields are put into this message. let (timeline_logical_size, _) = timeline - .get_current_logical_size() + .get_current_logical_size(&ctx) .context("Status update creation failed to get current logical size")?; let status_update = ReplicationFeedback { current_timeline_size: timeline_logical_size, From 8963d830fbab37bb9a5f6f68ccaa0ffa340aed09 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 25 Jan 2023 14:55:25 +0100 Subject: [PATCH 037/102] add script to download all remote layers (#3294) For use in production in case on-demand download turns out to be problematic during tenant_attach, or when we eventually introduce layer eviction. Co-authored-by: Dmitry Rodionov --- poetry.lock | 247 +++++++++++++++++++++++- pyproject.toml | 1 + scripts/force_layer_download.py | 324 ++++++++++++++++++++++++++++++++ 3 files changed, 567 insertions(+), 5 deletions(-) create mode 100644 scripts/force_layer_download.py diff --git a/poetry.lock b/poetry.lock index edbcddd576..fc37124184 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,21 @@ +[[package]] +name = "aiohttp" +version = "3.7.0" +description = "Async http client/server framework (asyncio)" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +async-timeout = ">=3.0,<4.0" +attrs = ">=17.3.0" +chardet = ">=2.0,<4.0" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["aiodns", "brotlipy", "cchardet"] + [[package]] name = "aiopg" version = "1.3.4" @@ -41,11 +59,11 @@ six = ">=1.9.0" [[package]] name = "async-timeout" -version = "4.0.2" +version = "3.0.1" description = "Timeout context manager for asyncio programs" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.5.3" [[package]] name = "asyncpg" @@ -560,6 +578,14 @@ networkx = ">=2.4,<3.0" pyyaml = ">5.4" sarif-om = ">=1.0.4,<1.1.0" +[[package]] +name = "chardet" +version = "3.0.4" +description = "Universal encoding detector for Python 2 and 3" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "charset-normalizer" version = "2.1.0" @@ -939,6 +965,14 @@ server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)" ssm = ["PyYAML (>=5.1)", "dataclasses"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "mypy" version = "0.991" @@ -1580,6 +1614,18 @@ category = "main" optional = false python-versions = ">=3.4" +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + [[package]] name = "zipp" version = "3.8.1" @@ -1595,9 +1641,44 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "af44b269c235a6fd59dacb4ff9e05cbc13a79b57254a8d5d4bde934bd5691a70" +content-hash = "0f7289ef9439d1d7cd36b07efb53741b773669b0f860189c800270b7def0c241" [metadata.files] +aiohttp = [ + {file = "aiohttp-3.7.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:72fe89f7e14939e896d984c4b592580f8cdfa7497feb1c0c24639a9c60be3eb9"}, + {file = "aiohttp-3.7.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:fdf778d4c4bf976e69a37213fe8083613d0851976ddcf485bd7c0650a43d3852"}, + {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fee7b5e68939ffc09f9b29f167ed49c8b50de3eee0a1d8108b439ddd9963af46"}, + {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:dd64634713be409202058f2ea267dfbcdd74b387b8793425f21ef0266d45d0e9"}, + {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:713dd7fd70ddda9dc8d014c49dd0e55b58afe4e0cddb8722c7501f53edf30c3f"}, + {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:d31c43f7c4948ce01957f9a1ceee0784e067778477557ebccdf805398331c1a1"}, + {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:5e26d6003eb6df304608d9fd9c9437065a8532d869a3ffcbd8113a3d710f8239"}, + {file = "aiohttp-3.7.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bf08462cddd10ddd8ffe5cb5c1638bfa051290909ebedb31c06e46578b9b7529"}, + {file = "aiohttp-3.7.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:07bacf6721db51a4c6160ed3031a2a97910647969dafd7c653f600f3b542f463"}, + {file = "aiohttp-3.7.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:245b58e30bc889d18b783db2f09ef1d814f466e15c84325410827451297003a0"}, + {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b392e5c3e122586c49cd8b9426f577bf4d51958933b839d158d28b69515af74e"}, + {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:5b5c320621a171aa85f96909af28fbb5286bd6842066db3062b083ba92261256"}, + {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:97d2341d1360dbe2c5b1d94922f7d68f9ce2ded1daab88b9bdeb49ce419cdc1b"}, + {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:beda23f292716887532661dc19abb9db2302ccfbd671a080cd8f4be7463d0841"}, + {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cbcaae9a6f14f762348d19b2dce8162772c0b0a1739314e18492a308a22caf96"}, + {file = "aiohttp-3.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7a49ef7b691babc83db126db874fbf26ba2f781899b91399f9ff8b235f059245"}, + {file = "aiohttp-3.7.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f56892f57310415cf6a179eec3ea6c7a82a9d37fbc00894943ea3154011a6d2a"}, + {file = "aiohttp-3.7.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:df1274b7620c32d3b15bfb0a8fb3165dd6cdc9c39f4db74d162f051c80826542"}, + {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a04ba359dc5f2e21b96bfc90c4a7665441441ba61b52e992b7799493889a3419"}, + {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:f548d7976d168f0f45ac5909ca5f606ae3f6f7aa1725b22504004a053b29a7d0"}, + {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:deef02e2a9f5095463098c7c22d5566f20a6e4e14fc0996c0c2efc74d461b680"}, + {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:fe44c96bc380588d36729392b602470d88a7c18e646e95dd4348cafe3900d91d"}, + {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:9210532e6e95b40d22a33415bb84423eef3f633b2d2339b97f3b26438eebc466"}, + {file = "aiohttp-3.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:a586e476a251483d222c73dfb2f27df90bc4ea1b8c7da9396236510e0d4046c8"}, + {file = "aiohttp-3.7.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:900012c5f12ff72b1453229afe288ddc9135176df8b3b3cc5b8f6cfde912aaa4"}, + {file = "aiohttp-3.7.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:064d5f0738bcbab3e0c0ecf85c93b5ee1e07e124f994eaa03bf73687f3ecd9da"}, + {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0a2edf27865e66a33f64fa793cd14d0aae8127ce20a858539e97c25b600556dc"}, + {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:eaa8ae734639d5a0a3b5e33a154b8bfef384cdc090706f95c387cae8b21af764"}, + {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:a8a42f05491d9c04a77806875a68f84fea9af7a59d47b7897cb166632f74606c"}, + {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:b19ded3f6957693b97ba8372aacb5b0021639bbd5e77b1e960796bcef5431969"}, + {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:cefbd7ce7d1f1db43749a077e4970e29e2b631f367c9eff3862c3c886b4218dd"}, + {file = "aiohttp-3.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:7d64f7dfd4e326d9b0d11b07fcd5ebf78844ba3c8f7699f38b50b0e0db0ae68f"}, + {file = "aiohttp-3.7.0.tar.gz", hash = "sha256:176f1d2b2bc07044f4ed583216578a72a2bd35dffdeb92e0517d0aaa29d29549"}, +] aiopg = [ {file = "aiopg-1.3.4-py3-none-any.whl", hash = "sha256:b5b74a124831aad71608c3c203479db90bac4a7eb3f8982bc48c3d3e6f1e57bf"}, {file = "aiopg-1.3.4.tar.gz", hash = "sha256:23f9e4cd9f28e9d91a6de3b4fb517e8bed25511cd954acccba9fe3a702d9b7d0"}, @@ -1611,8 +1692,8 @@ allure-python-commons = [ {file = "allure_python_commons-2.10.0-py3-none-any.whl", hash = "sha256:2a717e8ca8d296bf89cd57f38fc3c21893bd7ea8cd02a6ae5420e6d1a6eda5d0"}, ] async-timeout = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, + {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, + {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, ] asyncpg = [ {file = "asyncpg-0.27.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fca608d199ffed4903dce1bcd97ad0fe8260f405c1c225bdf0002709132171c2"}, @@ -1787,6 +1868,10 @@ cfn-lint = [ {file = "cfn-lint-0.61.3.tar.gz", hash = "sha256:3806e010d77901f5e935496df690c10e39676434a738fce1a1161cf9c7bd36a2"}, {file = "cfn_lint-0.61.3-py3-none-any.whl", hash = "sha256:8e9522fad0c7c98b31ecbdd4724f8d8a5787457cc0f71e62ae0d11104d6e52ab"}, ] +chardet = [ + {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, + {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, +] charset-normalizer = [ {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, @@ -1960,6 +2045,82 @@ moto = [ {file = "moto-3.1.18-py3-none-any.whl", hash = "sha256:b6eb096e7880c46ac44d6d90988c0043e31462115cfdc913a0ee8f470bd9555c"}, {file = "moto-3.1.18.tar.gz", hash = "sha256:1e05276a62aa5a4aa821b441647c2cbaa2ea175388980b10d5de88d41b327cf7"}, ] +multidict = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] mypy = [ {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, @@ -2412,6 +2573,82 @@ xmltodict = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] +yarl = [ + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"}, + {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"}, + {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"}, + {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"}, + {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"}, + {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"}, + {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"}, + {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946"}, + {file = "yarl-1.8.2-cp38-cp38-win32.whl", hash = "sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165"}, + {file = "yarl-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588"}, + {file = "yarl-1.8.2-cp39-cp39-win32.whl", hash = "sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83"}, + {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, + {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, +] zipp = [ {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, diff --git a/pyproject.toml b/pyproject.toml index b4fb7a9e7d..a817e9dda5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ psutil = "^5.9.4" types-psutil = "^5.9.5.4" types-toml = "^0.10.8" pytest-httpserver = "^1.0.6" +aiohttp = "3.7" [tool.poetry.dev-dependencies] flake8 = "^5.0.4" diff --git a/scripts/force_layer_download.py b/scripts/force_layer_download.py new file mode 100644 index 0000000000..5472d86d8f --- /dev/null +++ b/scripts/force_layer_download.py @@ -0,0 +1,324 @@ +import argparse +import asyncio +import json +import logging +import signal +import sys +from collections import defaultdict +from dataclasses import dataclass +from typing import Any, Awaitable, Dict, List, Tuple + +import aiohttp + + +class ClientException(Exception): + pass + + +class Client: + def __init__(self, pageserver_api_endpoint: str, max_concurrent_layer_downloads: int): + self.endpoint = pageserver_api_endpoint + self.max_concurrent_layer_downloads = max_concurrent_layer_downloads + self.sess = aiohttp.ClientSession() + + async def close(self): + await self.sess.close() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_t, exc_v, exc_tb): + await self.close() + + async def parse_response(self, resp, expected_type): + body = await resp.json() + if not resp.ok: + raise ClientException(f"Response: {resp} Body: {body}") + + if not isinstance(body, expected_type): + raise ClientException(f"expecting {expected_type.__name__}") + return body + + async def get_tenant_ids(self): + resp = await self.sess.get(f"{self.endpoint}/v1/tenant") + payload = await self.parse_response(resp=resp, expected_type=list) + return [t["id"] for t in payload] + + async def get_timeline_ids(self, tenant_id): + resp = await self.sess.get(f"{self.endpoint}/v1/tenant/{tenant_id}/timeline") + payload = await self.parse_response(resp=resp, expected_type=list) + return [t["timeline_id"] for t in payload] + + async def timeline_spawn_download_remote_layers(self, tenant_id, timeline_id, ongoing_ok=False): + resp = await self.sess.post( + f"{self.endpoint}/v1/tenant/{tenant_id}/timeline/{timeline_id}/download_remote_layers", + json={"max_concurrent_downloads": self.max_concurrent_layer_downloads}, + ) + body = await resp.json() + if resp.status == 409: + if not ongoing_ok: + raise ClientException("download already ongoing") + # response body has same shape for ongoing and newly created + elif not resp.ok: + raise ClientException(f"Response: {resp} Body: {body}") + + if not isinstance(body, dict): + raise ClientException("expecting dict") + + return body + + async def timeline_poll_download_remote_layers_status( + self, + tenant_id, + timeline_id, + ): + resp = await self.sess.get( + f"{self.endpoint}/v1/tenant/{tenant_id}/timeline/{timeline_id}/download_remote_layers", + ) + body = await resp.json() + + if resp.status == 404: + return None + elif not resp.ok: + raise ClientException(f"Response: {resp} Body: {body}") + + return body + + +@dataclass +class Completed: + """The status dict returned by the API""" + + status: Dict[str, Any] + + +sigint_received = asyncio.Event() + + +async def do_timeline(client: Client, tenant_id, timeline_id): + """ + Spawn download_remote_layers task for given timeline, + then poll until the download has reached a terminal state. + + If the terminal state is not 'Completed', the method raises an exception. + The caller is responsible for inspecting `failed_download_count`. + + If there is already a task going on when this method is invoked, + it raises an exception. + """ + + # Don't start new downloads if user pressed SIGINT. + # This task will show up as "raised_exception" in the report. + if sigint_received.is_set(): + raise Exception("not starting because SIGINT received") + + # run downloads to completion + + status = await client.timeline_poll_download_remote_layers_status(tenant_id, timeline_id) + if status is not None and status["state"] == "Running": + raise Exception("download is already running") + + spawned = await client.timeline_spawn_download_remote_layers( + tenant_id, timeline_id, ongoing_ok=False + ) + + while True: + st = await client.timeline_poll_download_remote_layers_status(tenant_id, timeline_id) + logging.info(f"{tenant_id}:{timeline_id} state is: {st}") + + if spawned["task_id"] != st["task_id"]: + raise ClientException("download task ids changed while polling") + + if st["state"] == "Running": + await asyncio.sleep(10) + continue + + if st["state"] != "Completed": + raise ClientException( + f"download task reached terminal state != Completed: {st['state']}" + ) + + return Completed(st) + + +def handle_sigint(): + logging.info("SIGINT received, asyncio event set. Will not start new downloads.") + global sigint_received + sigint_received.set() + + +async def main(args): + async with Client(args.pageserver_http_endpoint, args.max_concurrent_layer_downloads) as client: + exit_code = await main_impl(args, args.report_output, client) + + return exit_code + + +async def taskq_handler(task_q, result_q): + while True: + try: + (id, fut) = task_q.get_nowait() + except asyncio.QueueEmpty: + logging.debug("taskq_handler observed empty task_q, returning") + return + logging.info(f"starting task {id}") + try: + res = await fut + except Exception as e: + res = e + result_q.put_nowait((id, res)) + + +async def print_progress(result_q, tasks): + while True: + await asyncio.sleep(10) + logging.info(f"{result_q.qsize()} / {len(tasks)} tasks done") + + +async def main_impl(args, report_out, client: Client): + """ + Returns OS exit status. + """ + tenant_and_timline_ids: List[Tuple[str, str]] = [] + # fill tenant_and_timline_ids based on spec + for spec in args.what: + comps = spec.split(":") + if comps == ["ALL"]: + logging.info("get tenant list") + tenant_ids = await client.get_tenant_ids() + get_timeline_id_coros = [client.get_timeline_ids(tenant_id) for tenant_id in tenant_ids] + gathered = await asyncio.gather(*get_timeline_id_coros, return_exceptions=True) + assert len(tenant_ids) == len(gathered) + tenant_and_timline_ids = [] + for tid, tlids in zip(tenant_ids, gathered): + for tlid in tlids: + tenant_and_timline_ids.append((tid, tlid)) + elif len(comps) == 1: + tid = comps[0] + tlids = await client.get_timeline_ids(tid) + for tlid in tlids: + tenant_and_timline_ids.append((tid, tlid)) + elif len(comps) == 2: + tenant_and_timline_ids.append((comps[0], comps[1])) + else: + raise ValueError(f"invalid what-spec: {spec}") + + logging.info("expanded spec:") + for tid, tlid in tenant_and_timline_ids: + logging.info(f"{tid}:{tlid}") + + logging.info("remove duplicates after expanding spec") + tmp = list(set(tenant_and_timline_ids)) + assert len(tmp) <= len(tenant_and_timline_ids) + if len(tmp) != len(tenant_and_timline_ids): + logging.info(f"spec had {len(tenant_and_timline_ids) - len(tmp)} duplicates") + tenant_and_timline_ids = tmp + + logging.info("create tasks and process them at specified concurrency") + task_q: asyncio.Queue[Tuple[str, Awaitable[Any]]] = asyncio.Queue() + tasks = { + f"{tid}:{tlid}": do_timeline(client, tid, tlid) for tid, tlid in tenant_and_timline_ids + } + for task in tasks.items(): + task_q.put_nowait(task) + + result_q: asyncio.Queue[Tuple[str, Any]] = asyncio.Queue() + taskq_handlers = [] + for _ in range(0, args.concurrent_tasks): + taskq_handlers.append(taskq_handler(task_q, result_q)) + + print_progress_task = asyncio.create_task(print_progress(result_q, tasks)) + + await asyncio.gather(*taskq_handlers) + print_progress_task.cancel() + + logging.info("all tasks handled, generating report") + + results = [] + while True: + try: + results.append(result_q.get_nowait()) + except asyncio.QueueEmpty: + break + assert task_q.empty() + + report = defaultdict(list) + for id, result in results: + logging.info(f"result for {id}: {result}") + if isinstance(result, Completed): + if result.status["failed_download_count"] == 0: + report["completed_without_errors"].append(id) + else: + report["completed_with_download_errors"].append(id) + elif isinstance(result, Exception): + report["raised_exception"].append(id) + else: + raise ValueError("unexpected result type") + json.dump(report, report_out) + + logging.info("--------------------------------------------------------------------------------") + + report_success = len(report["completed_without_errors"]) == len(tenant_and_timline_ids) + if not report_success: + logging.error("One or more tasks encountered errors.") + else: + logging.info("All tasks reported success.") + logging.info("Inspect log for details and report file for JSON summary.") + + return report_success + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--report-output", + type=argparse.FileType("w"), + default="-", + help="where to write report output (default: stdout)", + ) + parser.add_argument( + "--pageserver-http-endpoint", + default="http://localhost:9898", + help="pageserver http endpoint, (default http://localhost:9898)", + ) + parser.add_argument( + "--concurrent-tasks", + required=False, + default=5, + type=int, + help="Max concurrent download tasks created & polled by this script", + ) + parser.add_argument( + "--max-concurrent-layer-downloads", + dest="max_concurrent_layer_downloads", + required=False, + default=8, + type=int, + help="Max concurrent download tasks spawned by pageserver. Each layer is a separate task.", + ) + + parser.add_argument( + "what", + nargs="+", + help="what to download: ALL|tenant_id|tenant_id:timeline_id", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="enable verbose logging", + ) + args = parser.parse_args() + + level = logging.INFO + if args.verbose: + level = logging.DEBUG + logging.basicConfig( + format="%(asctime)s,%(msecs)03d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s", + datefmt="%Y-%m-%d:%H:%M:%S", + level=level, + ) + + loop = asyncio.get_event_loop() + + loop.add_signal_handler(signal.SIGINT, handle_sigint) + sys.exit(asyncio.run(main(args))) From 00f1f54b7a74840e8dc9bfda5421866c4a9c07a1 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Wed, 25 Jan 2023 12:10:52 +0100 Subject: [PATCH 038/102] Leave one Dockerfile --- .github/workflows/build_and_test.yml | 2 +- ...ompute-node-v14 => Dockerfile.compute-node | 8 +- Dockerfile.compute-node-v15 | 239 ------------------ 3 files changed, 3 insertions(+), 246 deletions(-) rename Dockerfile.compute-node-v14 => Dockerfile.compute-node (98%) delete mode 100644 Dockerfile.compute-node-v15 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 918e568e27..1088f67710 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -583,7 +583,7 @@ jobs: run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - name: Kaniko build compute node with extensions - run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-${{ matrix.version }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} + run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --build-arg PG_VERSION=${{ matrix.version }} --dockerfile Dockerfile.compute-node --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} vm-compute-node-image: runs-on: [ self-hosted, dev, x64 ] diff --git a/Dockerfile.compute-node-v14 b/Dockerfile.compute-node similarity index 98% rename from Dockerfile.compute-node-v14 rename to Dockerfile.compute-node index fb115ca641..3d6cfd4e6b 100644 --- a/Dockerfile.compute-node-v14 +++ b/Dockerfile.compute-node @@ -1,8 +1,3 @@ -# -# This file is identical to the Dockerfile.compute-node-v15 file -# except for the version of Postgres that is built. -# - ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com ARG IMAGE=rust ARG TAG=pinned @@ -24,7 +19,8 @@ RUN apt update && \ # ######################################################################################### FROM build-deps AS pg-build -COPY vendor/postgres-v14 postgres +ARG PG_VERSION +COPY vendor/postgres-${PG_VERSION} postgres RUN cd postgres && \ ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ diff --git a/Dockerfile.compute-node-v15 b/Dockerfile.compute-node-v15 deleted file mode 100644 index efbe0c6872..0000000000 --- a/Dockerfile.compute-node-v15 +++ /dev/null @@ -1,239 +0,0 @@ -# -# This file is identical to the Dockerfile.compute-node-v14 file -# except for the version of Postgres that is built. -# - -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com -ARG IMAGE=rust -ARG TAG=pinned - -######################################################################################### -# -# Layer "build-deps" -# -######################################################################################### -FROM debian:bullseye-slim AS build-deps -RUN apt update && \ - apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ - zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev - -######################################################################################### -# -# Layer "pg-build" -# Build Postgres from the neon postgres repository. -# -######################################################################################### -FROM build-deps AS pg-build -COPY vendor/postgres-v15 postgres -RUN cd postgres && \ - ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \ - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \ - # Install headers - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \ - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install && \ - # Enable some of contrib extensions - echo 'trusted = true' >> /usr/local/pgsql/share/extension/bloom.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrowlocks.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/intagg.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgstattuple.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control - -######################################################################################### -# -# Layer "postgis-build" -# Build PostGIS from the upstream PostGIS mirror. -# -######################################################################################### -FROM build-deps AS postgis-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ -RUN apt update && \ - apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc - -RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \ - tar xvzf postgis-3.3.1.tar.gz && \ - cd postgis-3.3.1 && \ - ./autogen.sh && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ - ./configure && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ - cd extensions/postgis && \ - make clean && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_raster.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer_data_us.control - -######################################################################################### -# -# Layer "plv8-build" -# Build plv8 -# -######################################################################################### -FROM build-deps AS plv8-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ -RUN apt update && \ - apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils - -# https://github.com/plv8/plv8/issues/475: -# v8 uses gold for linking and sets `--thread-count=4` which breaks -# gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607) -# Install newer gold version manually as debian-testing binutils version updates -# libc version, which in turn breaks other extension built against non-testing libc. -RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \ - tar xvzf binutils-2.38.tar.gz && \ - cd binutils-2.38 && \ - cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \ - cd ../bfd && ./configure && make bfdver.h && \ - cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \ - cp /usr/local/bin/ld.gold /usr/bin/gold - -# Sed is used to patch for https://github.com/plv8/plv8/issues/503 -RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \ - tar xvzf v3.1.4.tar.gz && \ - cd plv8-3.1.4 && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ - sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \ - make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \ - rm -rf /plv8-* && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control - -######################################################################################### -# -# Layer "h3-pg-build" -# Build h3_pg -# -######################################################################################### -FROM build-deps AS h3-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - -# packaged cmake is too old -RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \ - -q -O /tmp/cmake-install.sh \ - && chmod u+x /tmp/cmake-install.sh \ - && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \ - && rm /tmp/cmake-install.sh - -RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \ - tar xvzf h3.tgz && \ - cd h3-4.0.1 && \ - mkdir build && \ - cd build && \ - cmake .. -DCMAKE_BUILD_TYPE=Release && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - DESTDIR=/h3 make install && \ - cp -R /h3/usr / && \ - rm -rf build - -RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3-pg.tgz && \ - tar xvzf h3-pg.tgz && \ - cd h3-pg-4.0.1 && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control - -######################################################################################### -# -# Layer "unit-pg-build" -# compile unit extension -# -######################################################################################### -FROM build-deps AS unit-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - -RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz && \ - tar xvzf 7.7.tar.gz && \ - cd postgresql-unit-7.7 && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/unit.control - -######################################################################################### -# -# Layer "neon-pg-ext-build" -# compile neon extensions -# -######################################################################################### -FROM build-deps AS neon-pg-ext-build -COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY --from=h3-pg-build /h3/usr / -COPY --from=unit-pg-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY pgxn/ pgxn/ - -RUN make -j $(getconf _NPROCESSORS_ONLN) \ - PG_CONFIG=/usr/local/pgsql/bin/pg_config \ - -C pgxn/neon \ - -s install - -######################################################################################### -# -# Compile and run the Neon-specific `compute_ctl` binary -# -######################################################################################### -FROM $REPOSITORY/$IMAGE:$TAG AS compute-tools -USER nonroot -# Copy entire project to get Cargo.* files with proper dependencies for the whole project -COPY --chown=nonroot . . -RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto - -######################################################################################### -# -# Clean up postgres folder before inclusion -# -######################################################################################### -FROM neon-pg-ext-build AS postgres-cleanup-layer -COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql - -# Remove binaries from /bin/ that we won't use (or would manually copy & install otherwise) -RUN cd /usr/local/pgsql/bin && rm ecpg raster2pgsql shp2pgsql pgtopo_export pgtopo_import pgsql2shp - -# Remove headers that we won't need anymore - we've completed installation of all extensions -RUN rm -r /usr/local/pgsql/include - -# Remove static postgresql libraries - all compilation is finished, so we -# can now remove these files - they must be included in other binaries by now -# if they were to be used by other libraries. -RUN rm /usr/local/pgsql/lib/lib*.a - -######################################################################################### -# -# Final layer -# Put it all together into the final image -# -######################################################################################### -FROM debian:bullseye-slim -# Add user postgres -RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \ - echo "postgres:test_console_pass" | chpasswd && \ - mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \ - chown -R postgres:postgres /var/db/postgres && \ - chmod 0750 /var/db/postgres/compute && \ - echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig - -COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local -COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl - -# Install: -# libreadline8 for psql -# libossp-uuid16 for extension ossp-uuid -# libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS -RUN apt update && \ - apt install --no-install-recommends -y \ - libreadline8 \ - libossp-uuid16 \ - libgeos-c1v5 \ - libgdal28 \ - libproj19 \ - libprotobuf-c1 \ - gdb && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -USER postgres -ENTRYPOINT ["/usr/local/bin/compute_ctl"] From 0c7276ae136daee382714e5acd0799f18bacc26e Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Wed, 25 Jan 2023 16:23:30 +0200 Subject: [PATCH 039/102] Expect timeline being stopped during the detach smoke test (#3442) Found this error recently: https://neon-github-public-dev.s3.amazonaws.com/reports/main/release/4005062867/index.html#categories/d2116d7e3b88302f27b3d646396b385b/18590f7063e91b53/?attachment=69e899c74f1cbfc5 I could not reproduce it locally, since always received `gc target timeline does not exist` instead, so that test is quite lucky. Still, the error is pretty valid to appear in this context, so do not fail the test if it's found in the logs. --- test_runner/regress/test_tenant_detach.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index ef6ac1527e..1beca58298 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -272,8 +272,10 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): bogus_timeline_id = TimelineId.generate() pageserver_http.timeline_gc(tenant_id, bogus_timeline_id, 0) - # the error will be printed to the log too + # the error will be printed to the log too env.pageserver.allowed_errors.append(".*gc target timeline does not exist.*") + # Timelines get stopped during detach, ignore the gc calls that error, whitnessing that + env.pageserver.allowed_errors.append(".*InternalServerError\\(timeline is Stopping.*") # Detach while running manual GC. # It should wait for manual GC to finish because it runs in a task associated with the tenant. From 4b8dbea5c11d30ffe0b2762330e18453601c45eb Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Wed, 25 Jan 2023 17:15:56 +0100 Subject: [PATCH 040/102] Add production link proxy to new account (#3444) This PR setup link proxy in us-east-2 region, but do not redirect pg.neon.tech DNS name to it Will keep old link proxy for the time of migration --- .../prod-us-east-2-delta.neon-proxy-link.yaml | 59 +++++++++++++++++++ .github/workflows/build_and_test.yml | 12 +++- 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 .github/helm-values/prod-us-east-2-delta.neon-proxy-link.yaml diff --git a/.github/helm-values/prod-us-east-2-delta.neon-proxy-link.yaml b/.github/helm-values/prod-us-east-2-delta.neon-proxy-link.yaml new file mode 100644 index 0000000000..eff24302bb --- /dev/null +++ b/.github/helm-values/prod-us-east-2-delta.neon-proxy-link.yaml @@ -0,0 +1,59 @@ +# Helm chart values for neon-proxy-link. +# This is a YAML-formatted file. + +image: + repository: neondatabase/neon + +settings: + authBackend: "link" + authEndpoint: "https://console.neon.tech/authenticate_proxy_request/" + uri: "https://console.neon.tech/psql_session/" + domain: "pg.neon.tech" + sentryEnvironment: "production" + +# -- Additional labels for zenith-proxy pods +podLabels: + zenith_service: proxy + zenith_env: production + zenith_region: us-east-2 + zenith_region_slug: us-east-2 + +service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internal + external-dns.alpha.kubernetes.io/hostname: neon-proxy-link-mgmt.delta.us-east-2.aws.neon.tech + +exposedService: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + external-dns.alpha.kubernetes.io/hostname: neon-proxy-link.delta.us-east-2.aws.neon.tech + +extraManifests: + - apiVersion: operator.victoriametrics.com/v1beta1 + kind: VMServiceScrape + metadata: + name: "{{ include \"neon-proxy.fullname\" . }}" + labels: + helm.sh/chart: neon-proxy-{{ .Chart.Version }} + app.kubernetes.io/name: neon-proxy + app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/managed-by: Helm + namespace: "{{ .Release.Namespace }}" + spec: + selector: + matchLabels: + app.kubernetes.io/name: "neon-proxy" + endpoints: + - port: http + path: /metrics + interval: 10s + scrapeTimeout: 10s + namespaceSelector: + matchNames: + - "{{ .Release.Namespace }}" diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1088f67710..553471e1a0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1149,12 +1149,16 @@ jobs: include: - target_region: us-east-2 target_cluster: prod-us-east-2-delta + deploy_link_proxy: true - target_region: us-west-2 target_cluster: prod-us-west-2-eta + deploy_link_proxy: false - target_region: eu-central-1 target_cluster: prod-eu-central-1-gamma + deploy_link_proxy: false - target_region: ap-southeast-1 target_cluster: prod-ap-southeast-1-epsilon + deploy_link_proxy: false environment: name: prod-${{ matrix.target_region }} steps: @@ -1169,11 +1173,17 @@ jobs: helm repo add neondatabase https://neondatabase.github.io/helm-charts aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - name: Re-deploy proxy + - name: Re-deploy scram proxy run: | DOCKER_TAG=${{needs.tag.outputs.build-tag}} helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + - name: Re-deploy link proxy + if: matrix.deploy_link_proxy + run: | + DOCKER_TAG=${{needs.tag.outputs.build-tag}} + helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + deploy-storage-broker-prod-new: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest From e3efb0d854b754f02a543b2af59f3b519ad8d19f Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Wed, 25 Jan 2023 18:29:06 +0100 Subject: [PATCH 041/102] Fix bug while creating unit extension (#3447) after executing ```sql CREATE EXTENSION unit; ``` I saw such error ``` ERROR: could not open file "/usr/local/pgsql/share/extension/unit_prefixes.data" for reading: No such file or directory (SQLSTATE 58P01) ``` Co-authored-by: Anastasia Lubennikova --- Dockerfile.compute-node | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 3d6cfd4e6b..936f368833 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -147,6 +147,11 @@ RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz & cd postgresql-unit-7.7 && \ make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + # unit extension's "create extension" script relies on absolute install path to fill some reference tables. + # We move the extension from '/usr/local/pgsql/' to '/usr/local/' after it is build. So we need to adjust the path. + # This one-liner removes pgsql/ part of the path. + # NOTE: Other extensions that rely on MODULEDIR variable after building phase will need the same fix. + find /usr/local/pgsql/share/extension/ -name "unit*.sql" -print0 | xargs -0 sed -i "s|pgsql/||g" && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/unit.control ######################################################################################### From 0a095894035b4a1ebf660ad04fadbe77d5185475 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Wed, 25 Jan 2023 15:18:41 -0500 Subject: [PATCH 042/102] Increase gc period to 1h (#3432) --- pageserver/src/tenant/config.rs | 7 ++++++- test_runner/regress/test_tenant_conf.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index c95a98fbc7..e66ee0ae36 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -28,7 +28,12 @@ pub mod defaults { pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10; pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024; - pub const DEFAULT_GC_PERIOD: &str = "100 s"; + + // Large DEFAULT_GC_PERIOD is fine as long as PITR_INTERVAL is larger. + // If there's a need to decrease this value, first make sure that GC + // doesn't hold a layer map write lock for non-trivial operations. + // Relevant: https://github.com/neondatabase/neon/issues/3394 + pub const DEFAULT_GC_PERIOD: &str = "1 hr"; pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3; pub const DEFAULT_PITR_INTERVAL: &str = "7 days"; pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "2 seconds"; diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 29cdcb18ce..f6a7977c32 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -57,7 +57,7 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" "compaction_period": 20, "compaction_threshold": 10, "gc_horizon": 67108864, - "gc_period": 100, + "gc_period": 60 * 60, "image_creation_threshold": 3, "pitr_interval": 604800, # 7 days }.items() From cd5732d9d8ccd291f39ed41250072acdce3012e6 Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Thu, 26 Jan 2023 10:46:06 +0100 Subject: [PATCH 043/102] Gen3 runners (#3220) https://github.com/neondatabase/cloud/issues/2738 Co-authored-by: Rory de Zoete Co-authored-by: Rory de Zoete --- .../actions/run-python-test-set/action.yml | 4 +- .github/workflows/build_and_test.yml | 150 ++++++++++-------- 2 files changed, 87 insertions(+), 67 deletions(-) diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 990c7e25a9..29b04a3478 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -123,8 +123,8 @@ runs: exit 1 fi if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then - # -n4 uses four processes to run tests via pytest-xdist - EXTRA_PARAMS="-n4 $EXTRA_PARAMS" + # -n16 uses sixteen processes to run tests via pytest-xdist + EXTRA_PARAMS="-n16 $EXTRA_PARAMS" # --dist=loadgroup points tests marked with @pytest.mark.xdist_group # to the same worker to make @pytest.mark.order work with xdist diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 553471e1a0..8f1730056e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -19,10 +19,12 @@ concurrency: env: RUST_BACKTRACE: 1 COPT: '-Werror' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} jobs: tag: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned outputs: build-tag: ${{steps.build-tag.outputs.tag}} @@ -50,7 +52,7 @@ jobs: id: build-tag check-codestyle-python: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cloud:pinned options: --init @@ -85,7 +87,7 @@ jobs: run: poetry run mypy . check-codestyle-rust: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -97,16 +99,16 @@ jobs: submodules: true fetch-depth: 1 - - name: Restore cargo deps cache - id: cache_cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry/ - !~/.cargo/registry/src - ~/.cargo/git/ - target/ - key: v1-${{ runner.os }}-cargo-clippy-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} +# Disabled for now +# - name: Restore cargo deps cache +# id: cache_cargo +# uses: actions/cache@v3 +# with: +# path: | +# !~/.cargo/registry/src +# ~/.cargo/git/ +# target/ +# key: v1-${{ runner.os }}-cargo-clippy-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} # Some of our rust modules use FFI and need those to be checked - name: Get postgres headers @@ -133,7 +135,7 @@ jobs: run: cargo deny check build-neon: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -141,7 +143,6 @@ jobs: fail-fast: false matrix: build_type: [ debug, release ] - env: BUILD_TYPE: ${{ matrix.build_type }} GIT_VERSION: ${{ github.sha }} @@ -194,24 +195,26 @@ jobs: echo "cov_prefix=${cov_prefix}" >> $GITHUB_ENV echo "CARGO_FEATURES=${CARGO_FEATURES}" >> $GITHUB_ENV echo "CARGO_FLAGS=${CARGO_FLAGS}" >> $GITHUB_ENV + echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo" >> $GITHUB_ENV + # Disabled for now # Don't include the ~/.cargo/registry/src directory. It contains just # uncompressed versions of the crates in ~/.cargo/registry/cache # directory, and it's faster to let 'cargo' to rebuild it from the # compressed crates. - - name: Cache cargo deps - id: cache_cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry/ - !~/.cargo/registry/src - ~/.cargo/git/ - target/ - # Fall back to older versions of the key, if no cache for current Cargo.lock was found - key: | - v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} - v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}- +# - name: Cache cargo deps +# id: cache_cargo +# uses: actions/cache@v3 +# with: +# path: | +# ~/.cargo/registry/ +# !~/.cargo/registry/src +# ~/.cargo/git/ +# target/ +# # Fall back to older versions of the key, if no cache for current Cargo.lock was found +# key: | +# v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} +# v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}- - name: Cache postgres v14 build id: cache_pg_14 @@ -301,7 +304,7 @@ jobs: uses: ./.github/actions/save-coverage-data regress-tests: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -334,7 +337,7 @@ jobs: uses: ./.github/actions/save-coverage-data benchmarks: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -365,7 +368,7 @@ jobs: # while coverage is currently collected for the debug ones merge-allure-report: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -402,7 +405,7 @@ jobs: DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json coverage-report: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -418,16 +421,17 @@ jobs: submodules: true fetch-depth: 1 - - name: Restore cargo deps cache - id: cache_cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry/ - !~/.cargo/registry/src - ~/.cargo/git/ - target/ - key: v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} +# Disabled for now +# - name: Restore cargo deps cache +# id: cache_cargo +# uses: actions/cache@v3 +# with: +# path: | +# ~/.cargo/registry/ +# !~/.cargo/registry/src +# ~/.cargo/git/ +# target/ +# key: v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} - name: Get Neon artifact uses: ./.github/actions/download @@ -477,7 +481,7 @@ jobs: }" trigger-e2e-tests: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned options: --init @@ -522,9 +526,10 @@ jobs: }" neon-image: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] needs: [ tag ] - container: gcr.io/kaniko-project/executor:v1.9.0-debug + # https://github.com/GoogleContainerTools/kaniko/issues/2005 + container: gcr.io/kaniko-project/executor:v1.7.0-debug defaults: run: shell: sh -eu {0} @@ -540,12 +545,16 @@ jobs: run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - name: Kaniko build neon - run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} + run: /kaniko/executor --reproducible --snapshotMode=redo --skip-unused-stages --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} + + # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ecr': Permission denied + - name: Cleanup ECR folder + run: rm -rf ~/.ecr compute-tools-image: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] needs: [ tag ] - container: gcr.io/kaniko-project/executor:v1.9.0-debug + container: gcr.io/kaniko-project/executor:v1.7.0-debug defaults: run: shell: sh -eu {0} @@ -558,11 +567,14 @@ jobs: run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - name: Kaniko build compute tools - run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} + run: /kaniko/executor --reproducible --snapshotMode=redo --skip-unused-stages --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} + + - name: Cleanup ECR folder + run: rm -rf ~/.ecr compute-node-image: - runs-on: [ self-hosted, dev, x64 ] - container: gcr.io/kaniko-project/executor:v1.9.0-debug + runs-on: [ self-hosted, gen3, large ] + container: gcr.io/kaniko-project/executor:v1.7.0-debug needs: [ tag ] strategy: fail-fast: false @@ -583,10 +595,13 @@ jobs: run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - name: Kaniko build compute node with extensions - run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --build-arg PG_VERSION=${{ matrix.version }} --dockerfile Dockerfile.compute-node --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} + run: /kaniko/executor --reproducible --snapshotMode=redo --skip-unused-stages --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --build-arg PG_VERSION=${{ matrix.version }} --dockerfile Dockerfile.compute-node --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} + + - name: Cleanup ECR folder + run: rm -rf ~/.ecr vm-compute-node-image: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, large ] needs: [ tag, compute-node-image ] strategy: fail-fast: false @@ -631,7 +646,7 @@ jobs: test-images: needs: [ tag, neon-image, compute-node-image, compute-tools-image ] - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] steps: - name: Checkout @@ -673,7 +688,7 @@ jobs: docker compose -f ./docker-compose/docker-compose.yml down promote-images: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] needs: [ tag, test-images, vm-compute-node-image ] if: github.event_name != 'workflow_dispatch' container: amazon/aws-cli @@ -681,6 +696,8 @@ jobs: fail-fast: false matrix: name: [ neon, compute-node-v14, vm-compute-node-v14, compute-node-v15, vm-compute-node-v15, compute-tools] + env: + AWS_DEFAULT_REGION: eu-central-1 steps: - name: Promote image to latest @@ -689,7 +706,7 @@ jobs: aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST" push-docker-hub: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] needs: [ promote-images, tag ] container: golang:1.19-bullseye @@ -776,8 +793,11 @@ jobs: crane tag neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} latest + - name: Cleanup ECR folder + run: rm -rf ~/.ecr + calculate-deploy-targets: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] if: | github.ref_name == 'release' && github.event_name != 'workflow_dispatch' @@ -795,7 +815,7 @@ jobs: fi deploy: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly @@ -843,7 +863,7 @@ jobs: rm -f neon_install.tar.gz .neon_current_version deploy-new: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly @@ -883,7 +903,7 @@ jobs: rm -f neon_install.tar.gz .neon_current_version deploy-pr-test-new: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly @@ -958,7 +978,7 @@ jobs: rm -f neon_install.tar.gz .neon_current_version deploy-proxy: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] @@ -1003,7 +1023,7 @@ jobs: deploy-storage-broker: name: deploy storage broker on old staging and old prod - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] @@ -1045,7 +1065,7 @@ jobs: helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace ${{ matrix.storage_broker_ns }} --create-namespace --install --atomic -f .github/helm-values/${{ matrix.storage_broker_config }}.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s deploy-proxy-new: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. needs: [ push-docker-hub, tag, regress-tests ] @@ -1098,7 +1118,7 @@ jobs: helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s deploy-storage-broker-dev-new: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. needs: [ push-docker-hub, tag, regress-tests ] @@ -1225,7 +1245,7 @@ jobs: helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s promote-compatibility-data: - runs-on: [ self-hosted, dev, x64 ] + runs-on: [ self-hosted, gen3, small ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init From dc64962ffcf8bd1991ce1bc8d10d860498c4475a Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 19 Jan 2023 13:29:19 +0100 Subject: [PATCH 044/102] tenant::mgr: explicit tracking of initializing & shutting-down states This patch wrap the tenants hashmap into an enum that represents the tenant manager's three major states: - Initializing - Open for business - Shutting down. See the enum doc comments for details. In response, all the users of `TENANTS` are now forced to distinguish those states. The only major change is in `run_if_no_tenant_in_memory`, which, before this patch, was used by the /attach and /load endpoints. This patch rewrites that method under the name `tenant_map_insert`, replacing the anyhow::Result with a std Result and a dedicated error type. Introducing this error types allows using `tenant_map_insert` in `tenant_create`, thereby unifying all code paths that create tenants objects to use `tenant_map_insert`. This is beneficial because we can now systematically prevent tenants from being created, attached, or `/load`ed during pageserver shutdown. The management API remains available, but the endpoints that create new tenants will fail with an error. More work would need to be done to properly distinguish these errors through HTTP status codes such as 503. --- pageserver/src/consumption_metrics.rs | 10 +- pageserver/src/http/routes.rs | 61 +++++---- pageserver/src/tenant/mgr.rs | 190 +++++++++++++++++++------- 3 files changed, 179 insertions(+), 82 deletions(-) diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 17432234ed..d848ec5ee5 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -115,7 +115,7 @@ pub async fn collect_metrics_iteration( ); // get list of tenants - let tenants = mgr::list_tenants().await; + let tenants = mgr::list_tenants().await?; // iterate through list of Active tenants and collect metrics for (tenant_id, tenant_state) in tenants { @@ -276,7 +276,13 @@ pub async fn calculate_synthetic_size_worker( }, _ = ticker.tick() => { - let tenants = mgr::list_tenants().await; + let tenants = match mgr::list_tenants().await { + Ok(tenants) => tenants, + Err(e) => { + warn!("cannot get tenant list: {e:#}"); + continue; + } + }; // iterate through list of Active tenants and collect metrics for (tenant_id, tenant_state) in tenants { diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 541c779efd..750a242361 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -16,6 +16,7 @@ use crate::context::{DownloadBehavior, RequestContext}; use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; use crate::tenant::config::TenantConfOpt; +use crate::tenant::mgr::TenantMapInsertError; use crate::tenant::{PageReconstructError, Timeline}; use crate::{config::PageServerConf, tenant::mgr}; use utils::{ @@ -99,6 +100,18 @@ fn apierror_from_prerror(err: PageReconstructError) -> ApiError { } } +fn apierror_from_tenant_map_insert_error(e: TenantMapInsertError) -> ApiError { + match e { + TenantMapInsertError::StillInitializing | TenantMapInsertError::ShuttingDown => { + ApiError::InternalServerError(anyhow::Error::new(e)) + } + TenantMapInsertError::TenantAlreadyExists(id, state) => { + ApiError::Conflict(format!("tenant {id} already exists, state: {state:?}")) + } + TenantMapInsertError::Closure(e) => ApiError::InternalServerError(e), + } +} + // Helper function to construct a TimelineInfo struct for a timeline async fn build_timeline_info( timeline: &Arc, @@ -356,11 +369,10 @@ async fn tenant_attach_handler(request: Request) -> Result, let state = get_state(&request); if let Some(remote_storage) = &state.remote_storage { - // FIXME: distinguish between "Tenant already exists" and other errors mgr::attach_tenant(state.conf, tenant_id, remote_storage.clone(), &ctx) .instrument(info_span!("tenant_attach", tenant = %tenant_id)) .await - .map_err(ApiError::InternalServerError)?; + .map_err(apierror_from_tenant_map_insert_error)?; } else { return Err(ApiError::BadRequest(anyhow!( "attach_tenant is not possible because pageserver was configured without remote storage" @@ -414,7 +426,7 @@ async fn tenant_load_handler(request: Request) -> Result, A mgr::load_tenant(state.conf, tenant_id, state.remote_storage.clone(), &ctx) .instrument(info_span!("load", tenant = %tenant_id)) .await - .map_err(ApiError::InternalServerError)?; + .map_err(apierror_from_tenant_map_insert_error)?; json_response(StatusCode::ACCEPTED, ()) } @@ -441,6 +453,8 @@ async fn tenant_list_handler(request: Request) -> Result, A let response_data = mgr::list_tenants() .instrument(info_span!("tenant_list")) .await + .map_err(anyhow::Error::new) + .map_err(ApiError::InternalServerError)? .iter() .map(|(id, state)| TenantInfo { id: *id, @@ -638,31 +652,24 @@ async fn tenant_create_handler(mut request: Request) -> Result { - // We created the tenant. Existing API semantics are that the tenant - // is Active when this function returns. - if let res @ Err(_) = tenant.wait_to_become_active().await { - // This shouldn't happen because we just created the tenant directory - // in tenant::mgr::create_tenant, and there aren't any remote timelines - // to load, so, nothing can really fail during load. - // Don't do cleanup because we don't know how we got here. - // The tenant will likely be in `Broken` state and subsequent - // calls will fail. - res.context("created tenant failed to become active") - .map_err(ApiError::InternalServerError)?; - } - json_response( - StatusCode::CREATED, - TenantCreateResponse(tenant.tenant_id()), - )? - } - None => json_response(StatusCode::CONFLICT, ())?, - }) + // We created the tenant. Existing API semantics are that the tenant + // is Active when this function returns. + if let res @ Err(_) = new_tenant.wait_to_become_active().await { + // This shouldn't happen because we just created the tenant directory + // in tenant::mgr::create_tenant, and there aren't any remote timelines + // to load, so, nothing can really fail during load. + // Don't do cleanup because we don't know how we got here. + // The tenant will likely be in `Broken` state and subsequent + // calls will fail. + res.context("created tenant failed to become active") + .map_err(ApiError::InternalServerError)?; + } + json_response( + StatusCode::CREATED, + TenantCreateResponse(new_tenant.tenant_id()), + ) } async fn tenant_config_handler(mut request: Request) -> Result, ApiError> { diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 9e9c2786b3..e7b71d7dd7 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -25,8 +25,35 @@ use crate::IGNORED_TENANT_FILE_NAME; use utils::fs_ext::PathExt; use utils::id::{TenantId, TimelineId}; -static TENANTS: Lazy>>> = - Lazy::new(|| RwLock::new(HashMap::new())); +/// The tenants known to the pageserver. +/// The enum variants are used to distinguish the different states that the pageserver can be in. +enum TenantsMap { + /// [`init_tenant_mgr`] is not done yet. + Initializing, + /// [`init_tenant_mgr`] is done, all on-disk tenants have been loaded. + /// New tenants can be added using [`tenant_map_insert`]. + Open(HashMap>), + /// The pageserver has entered shutdown mode via [`shutdown_all_tenants`]. + /// Existing tenants are still accessible, but no new tenants can be created. + ShuttingDown(HashMap>), +} + +impl TenantsMap { + fn get(&self, tenant_id: &TenantId) -> Option<&Arc> { + match self { + TenantsMap::Initializing => None, + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.get(tenant_id), + } + } + fn remove(&mut self, tenant_id: &TenantId) -> Option> { + match self { + TenantsMap::Initializing => None, + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.remove(tenant_id), + } + } +} + +static TENANTS: Lazy> = Lazy::new(|| RwLock::new(TenantsMap::Initializing)); /// Initialize repositories with locally available timelines. /// Timelines that are only partially available locally (remote storage has more data than this pageserver) @@ -37,9 +64,10 @@ pub async fn init_tenant_mgr( remote_storage: Option, ) -> anyhow::Result<()> { // Scan local filesystem for attached tenants - let mut number_of_tenants = 0; let tenants_dir = conf.tenants_path(); + let mut tenants = HashMap::new(); + let mut dir_entries = fs::read_dir(&tenants_dir) .await .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?; @@ -92,8 +120,7 @@ pub async fn init_tenant_mgr( &ctx, ) { Ok(tenant) => { - TENANTS.write().await.insert(tenant.tenant_id(), tenant); - number_of_tenants += 1; + tenants.insert(tenant.tenant_id(), tenant); } Err(e) => { error!("Failed to collect tenant files from dir {tenants_dir:?} for entry {dir_entry:?}, reason: {e:#}"); @@ -112,7 +139,11 @@ pub async fn init_tenant_mgr( } } - info!("Processed {number_of_tenants} local tenants at startup"); + info!("Processed {} local tenants at startup", tenants.len()); + + let mut tenants_map = TENANTS.write().await; + assert!(matches!(&*tenants_map, &TenantsMap::Initializing)); + *tenants_map = TenantsMap::Open(tenants); Ok(()) } @@ -171,21 +202,44 @@ pub fn schedule_local_tenant_processing( /// /// Shut down all tenants. This runs as part of pageserver shutdown. /// +/// NB: We leave the tenants in the map, so that they remain accessible through +/// the management API until we shut it down. If we removed the shut-down tenants +/// from the tenants map, the management API would return 404 for these tenants, +/// because TenantsMap::get() now returns `None`. +/// That could be easily misinterpreted by control plane, the consumer of the +/// management API. For example, it could attach the tenant on a different pageserver. +/// We would then be in split-brain once this pageserver restarts. pub async fn shutdown_all_tenants() { + // Prevent new tenants from being created. let tenants_to_shut_down = { let mut m = TENANTS.write().await; - let mut tenants_to_shut_down = Vec::with_capacity(m.len()); - for (_, tenant) in m.drain() { - if tenant.is_active() { - // updates tenant state, forbidding new GC and compaction iterations from starting - tenant.set_stopping(); - tenants_to_shut_down.push(tenant) + match &mut *m { + TenantsMap::Initializing => { + *m = TenantsMap::ShuttingDown(HashMap::default()); + info!("tenants map is empty"); + return; + } + TenantsMap::Open(tenants) => { + let tenants_clone = tenants.clone(); + *m = TenantsMap::ShuttingDown(std::mem::take(tenants)); + tenants_clone + } + TenantsMap::ShuttingDown(_) => { + error!("already shutting down, this function isn't supposed to be called more than once"); + return; } } - drop(m); - tenants_to_shut_down }; + let mut tenants_to_freeze_and_flush = Vec::with_capacity(tenants_to_shut_down.len()); + for (_, tenant) in tenants_to_shut_down { + if tenant.is_active() { + // updates tenant state, forbidding new GC and compaction iterations from starting + tenant.set_stopping(); + tenants_to_freeze_and_flush.push(tenant); + } + } + // Shut down all existing walreceiver connections and stop accepting the new ones. task_mgr::shutdown_tasks(Some(TaskKind::WalReceiverManager), None, None).await; @@ -197,7 +251,7 @@ pub async fn shutdown_all_tenants() { // should be no more activity in any of the repositories. // // On error, log it but continue with the shutdown for other tenants. - for tenant in tenants_to_shut_down { + for tenant in tenants_to_freeze_and_flush { let tenant_id = tenant.tenant_id(); debug!("shutdown tenant {tenant_id}"); @@ -213,27 +267,22 @@ pub async fn create_tenant( tenant_id: TenantId, remote_storage: Option, ctx: &RequestContext, -) -> anyhow::Result>> { - match TENANTS.write().await.entry(tenant_id) { - hash_map::Entry::Occupied(_) => { - debug!("tenant {tenant_id} already exists"); - Ok(None) - } - hash_map::Entry::Vacant(v) => { - // Hold the write_tenants() lock, since all of this is local IO. - // If this section ever becomes contentious, introduce a new `TenantState::Creating`. - let tenant_directory = super::create_tenant_files(conf, tenant_conf, tenant_id)?; - let created_tenant = - schedule_local_tenant_processing(conf, &tenant_directory, remote_storage, ctx)?; - let crated_tenant_id = created_tenant.tenant_id(); - anyhow::ensure!( +) -> Result, TenantMapInsertError> { + tenant_map_insert(tenant_id, |vacant_entry| { + // We're holding the tenants lock in write mode while doing local IO. + // If this section ever becomes contentious, introduce a new `TenantState::Creating` + // and do the work in that state. + let tenant_directory = super::create_tenant_files(conf, tenant_conf, tenant_id)?; + let created_tenant = + schedule_local_tenant_processing(conf, &tenant_directory, remote_storage, ctx)?; + let crated_tenant_id = created_tenant.tenant_id(); + anyhow::ensure!( tenant_id == crated_tenant_id, "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {crated_tenant_id})", ); - v.insert(Arc::clone(&created_tenant)); - Ok(Some(created_tenant)) - } - } + vacant_entry.insert(Arc::clone(&created_tenant)); + Ok(created_tenant) + }).await } pub async fn update_tenant_config( @@ -302,8 +351,8 @@ pub async fn load_tenant( tenant_id: TenantId, remote_storage: Option, ctx: &RequestContext, -) -> anyhow::Result<()> { - run_if_no_tenant_in_memory(tenant_id, |vacant_entry| { +) -> Result<(), TenantMapInsertError> { + tenant_map_insert(tenant_id, |vacant_entry| { let tenant_path = conf.tenant_path(&tenant_id); let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(tenant_id); if tenant_ignore_mark.exists() { @@ -340,16 +389,24 @@ pub async fn ignore_tenant( .await } +#[derive(Debug, thiserror::Error)] +pub enum TenantMapListError { + #[error("tenant map is still initiailizing")] + Initializing, +} + /// /// Get list of tenants, for the mgmt API /// -pub async fn list_tenants() -> Vec<(TenantId, TenantState)> { - TENANTS - .read() - .await - .iter() +pub async fn list_tenants() -> Result, TenantMapListError> { + let tenants = TENANTS.read().await; + let m = match &*tenants { + TenantsMap::Initializing => return Err(TenantMapListError::Initializing), + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m, + }; + Ok(m.iter() .map(|(id, tenant)| (*id, tenant.current_state())) - .collect() + .collect()) } /// Execute Attach mgmt API command. @@ -361,8 +418,8 @@ pub async fn attach_tenant( tenant_id: TenantId, remote_storage: GenericRemoteStorage, ctx: &RequestContext, -) -> anyhow::Result<()> { - run_if_no_tenant_in_memory(tenant_id, |vacant_entry| { +) -> Result<(), TenantMapInsertError> { + tenant_map_insert(tenant_id, |vacant_entry| { let tenant_path = conf.tenant_path(&tenant_id); anyhow::ensure!( !tenant_path.exists(), @@ -371,24 +428,51 @@ pub async fn attach_tenant( let tenant = Tenant::spawn_attach(conf, tenant_id, remote_storage, ctx); vacant_entry.insert(tenant); - Ok(()) }) .await } -async fn run_if_no_tenant_in_memory(tenant_id: TenantId, run: F) -> anyhow::Result +#[derive(Debug, thiserror::Error)] +pub enum TenantMapInsertError { + #[error("tenant map is still initializing")] + StillInitializing, + #[error("tenant map is shutting down")] + ShuttingDown, + #[error("tenant {0} already exists, state: {1:?}")] + TenantAlreadyExists(TenantId, TenantState), + #[error(transparent)] + Closure(#[from] anyhow::Error), +} + +/// Give the given closure access to the tenants map entry for the given `tenant_id`, iff that +/// entry is vacant. The closure is responsible for creating the tenant object and inserting +/// it into the tenants map through the vacnt entry that it receives as argument. +/// +/// NB: the closure should return quickly because the current implementation of tenants map +/// serializes access through an `RwLock`. +async fn tenant_map_insert( + tenant_id: TenantId, + insert_fn: F, +) -> Result where F: FnOnce(hash_map::VacantEntry>) -> anyhow::Result, { - match TENANTS.write().await.entry(tenant_id) { - hash_map::Entry::Occupied(e) => { - anyhow::bail!( - "tenant {tenant_id} already exists, state: {:?}", - e.get().current_state() - ) - } - hash_map::Entry::Vacant(v) => run(v), + let mut guard = TENANTS.write().await; + let m = match &mut *guard { + TenantsMap::Initializing => return Err(TenantMapInsertError::StillInitializing), + TenantsMap::ShuttingDown(_) => return Err(TenantMapInsertError::ShuttingDown), + TenantsMap::Open(m) => m, + }; + match m.entry(tenant_id) { + hash_map::Entry::Occupied(e) => Err(TenantMapInsertError::TenantAlreadyExists( + tenant_id, + e.get().current_state(), + )), + hash_map::Entry::Vacant(v) => match insert_fn(v) { + Ok(v) => Ok(v), + Err(e) => Err(TenantMapInsertError::Closure(e)), + }, } } From 4bcbb7793d5b3cde3fd2b4f5e642690992a5710d Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:30:53 +0100 Subject: [PATCH 045/102] Revert docker hub job (#3453) Regression fix as permissions aren't configured properly on gen3 for this job. Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8f1730056e..202e7240c8 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -706,7 +706,7 @@ jobs: aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST" push-docker-hub: - runs-on: [ self-hosted, gen3, small ] + runs-on: [ self-hosted, dev, x64 ] needs: [ promote-images, tag ] container: golang:1.19-bullseye From 006ee5f94a2d6c308056d28012ffbdb75f7f8802 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 26 Jan 2023 14:13:20 +0200 Subject: [PATCH 046/102] Configure 'compute_ctl' to use OpenTelemetry exporter. This allows tracing the startup actions e.g. with Jaeger (https://www.jaegertracing.io/). We use the "tracing-opentelemetry" crate, which turns tracing spans into OpenTelemetry spans, so you can use the usual "#[instrument]" directives to add tracing. I put the tracing initialization code to a separate crate, `tracing-utils`, so that we can reuse it in other programs. We probably want to set up tracing in the same way in all our programs. Co-authored-by: Joonas Koivunen --- Cargo.lock | 152 +++++++++++++++++++++++++ Cargo.toml | 5 + compute_tools/Cargo.toml | 2 + compute_tools/src/bin/compute_ctl.rs | 6 +- compute_tools/src/logger.rs | 24 +++- libs/tracing-utils/Cargo.toml | 16 +++ libs/tracing-utils/src/lib.rs | 161 +++++++++++++++++++++++++++ workspace_hack/Cargo.toml | 6 + 8 files changed, 367 insertions(+), 5 deletions(-) create mode 100644 libs/tracing-utils/Cargo.toml create mode 100644 libs/tracing-utils/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c76861267f..91515df5bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -839,7 +839,9 @@ dependencies = [ "tokio", "tokio-postgres", "tracing", + "tracing-opentelemetry", "tracing-subscriber", + "tracing-utils", "url", "workspace_hack", ] @@ -1140,6 +1142,19 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown 0.12.3", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.3.3" @@ -2162,6 +2177,108 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc79add46364183ece1a4542592ca593e6421c60807232f5b8f7a31703825d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry_api", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" +dependencies = [ + "async-trait", + "futures", + "futures-util", + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "prost", + "reqwest", + "thiserror", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" +dependencies = [ + "futures", + "futures-util", + "opentelemetry", + "prost", + "tonic", + "tonic-build", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "os_info" version = "3.5.1" @@ -4002,6 +4119,20 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -4033,6 +4164,21 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-utils" +version = "0.1.0" +dependencies = [ + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "reqwest", + "tokio", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", + "workspace_hack", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -4470,7 +4616,9 @@ dependencies = [ "crossbeam-utils", "either", "fail", + "futures", "futures-channel", + "futures-executor", "futures-task", "futures-util", "indexmap", @@ -4486,6 +4634,9 @@ dependencies = [ "rand", "regex", "regex-syntax", + "reqwest", + "ring", + "rustls", "scopeguard", "serde", "serde_json", @@ -4493,6 +4644,7 @@ dependencies = [ "syn", "tokio", "tokio-util", + "tonic", "tower", "tracing", "tracing-core", diff --git a/Cargo.toml b/Cargo.toml index 57f4b1d981..e6695c4246 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,10 @@ nix = "0.26" notify = "5.0.0" num-traits = "0.2.15" once_cell = "1.13" +opentelemetry = "0.18.0" +opentelemetry-otlp = { version = "0.11.0", default_features=false, features = ["http-proto", "trace", "http", "reqwest-client"] } +opentelemetry-semantic-conventions = "0.10.0" +tracing-opentelemetry = "0.18.0" parking_lot = "0.12" pin-project-lite = "0.2" prometheus = {version = "0.13", default_features=false, features = ["process"]} # removes protobuf dependency @@ -125,6 +129,7 @@ remote_storage = { version = "0.1", path = "./libs/remote_storage/" } safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" } storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy. tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" } +tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" } utils = { version = "0.1", path = "./libs/utils/" } ## Common library dependency diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 4536604bdf..0fabd23965 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -19,7 +19,9 @@ tar.workspace = true tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } tokio-postgres.workspace = true tracing.workspace = true +tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true +tracing-utils.workspace = true url.workspace = true workspace_hack.workspace = true diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index e5ab8eb153..0e7f38bf84 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -53,7 +53,7 @@ use compute_tools::spec::*; use url::Url; fn main() -> Result<()> { - init_logger(DEFAULT_LOG_LEVEL)?; + init_tracing_and_logging(DEFAULT_LOG_LEVEL)?; let matches = cli().get_matches(); @@ -159,6 +159,10 @@ fn main() -> Result<()> { info!("shutting down"); } + // Shutdown trace pipeline gracefully, so that it has a chance to send any + // pending traces before we exit. + tracing_utils::shutdown_tracing(); + exit(exit_code.unwrap_or(1)) } diff --git a/compute_tools/src/logger.rs b/compute_tools/src/logger.rs index 57e5496e86..1b5cf647b0 100644 --- a/compute_tools/src/logger.rs +++ b/compute_tools/src/logger.rs @@ -1,21 +1,37 @@ -use anyhow::Result; +use tracing_opentelemetry::OpenTelemetryLayer; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::prelude::*; -/// Initialize `env_logger` using either `default_level` or +/// Initialize logging to stderr, and OpenTelemetry tracing and exporter. +/// +/// Logging is configured using either `default_log_level` or /// `RUST_LOG` environment variable as default log level. -pub fn init_logger(default_level: &str) -> Result<()> { +/// +/// OpenTelemetry is configured with OTLP/HTTP exporter. It picks up +/// configuration from environment variables. For example, to change the destination, +/// set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318`. See +/// `tracing-utils` package description. +/// +pub fn init_tracing_and_logging(default_log_level: &str) -> anyhow::Result<()> { + // Initialize Logging let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(default_level)); + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(default_log_level)); let fmt_layer = tracing_subscriber::fmt::layer() .with_target(false) .with_writer(std::io::stderr); + // Initialize OpenTelemetry + let otlp_layer = + tracing_utils::init_tracing_without_runtime("compute_ctl").map(OpenTelemetryLayer::new); + + // Put it all together tracing_subscriber::registry() .with(env_filter) + .with(otlp_layer) .with(fmt_layer) .init(); + tracing::info!("logging and tracing started"); Ok(()) } diff --git a/libs/tracing-utils/Cargo.toml b/libs/tracing-utils/Cargo.toml new file mode 100644 index 0000000000..9d5cfb0d56 --- /dev/null +++ b/libs/tracing-utils/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "tracing-utils" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +opentelemetry = { workspace = true, features=["rt-tokio"] } +opentelemetry-otlp = { workspace = true, default_features=false, features = ["http-proto", "trace", "http", "reqwest-client"] } +opentelemetry-semantic-conventions.workspace = true +reqwest = { workspace = true, default-features = false, features = ["rustls-tls"] } +tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } +tracing.workspace = true +tracing-opentelemetry.workspace = true +tracing-subscriber.workspace = true +workspace_hack = { version = "0.1", path = "../../workspace_hack" } diff --git a/libs/tracing-utils/src/lib.rs b/libs/tracing-utils/src/lib.rs new file mode 100644 index 0000000000..179f54c659 --- /dev/null +++ b/libs/tracing-utils/src/lib.rs @@ -0,0 +1,161 @@ +//! Helper functions to set up OpenTelemetry tracing. +//! +//! This comes in two variants, depending on whether you have a Tokio runtime available. +//! If you do, call `init_tracing()`. It sets up the trace processor and exporter to use +//! the current tokio runtime. If you don't have a runtime available, or you don't want +//! to share the runtime with the tracing tasks, call `init_tracing_without_runtime()` +//! instead. It sets up a dedicated single-threaded Tokio runtime for the tracing tasks. +//! +//! Example: +//! +//! ```rust,no_run +//! use tracing_subscriber::prelude::*; +//! use tracing_opentelemetry::OpenTelemetryLayer; +//! +//! #[tokio::main] +//! async fn main() { +//! // Set up logging to stderr +//! let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() +//! .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); +//! let fmt_layer = tracing_subscriber::fmt::layer() +//! .with_target(false) +//! .with_writer(std::io::stderr); +//! +//! // Initialize OpenTelemetry. Exports tracing spans as OpenTelemetry traces +//! let otlp_layer = tracing_utils::init_tracing("my_application").await.map(OpenTelemetryLayer::new); +//! +//! // Put it all together +//! tracing_subscriber::registry() +//! .with(env_filter) +//! .with(otlp_layer) +//! .with(fmt_layer) +//! .init(); +//! } +//! ``` + +use opentelemetry::sdk::Resource; +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::{OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}; + +pub use tracing_opentelemetry::OpenTelemetryLayer; + +/// Set up OpenTelemetry exporter, using configuration from environment variables. +/// +/// `service_name` is set as the OpenTelemetry 'service.name' resource (see +/// ) +/// +/// We try to follow the conventions for the environment variables specified in +/// +/// +/// However, we only support a subset of those options: +/// +/// - OTEL_SDK_DISABLED is supported. The default is "false", meaning tracing +/// is enabled by default. Set it to "true" to disable. +/// +/// - We use the OTLP exporter, with HTTP protocol. Most of the OTEL_EXPORTER_OTLP_* +/// settings specified in +/// +/// are supported, as they are handled by the `opentelemetry-otlp` crate. +/// Settings related to other exporters have no effect. +/// +/// - Some other settings are supported by the `opentelemetry` crate. +/// +/// If you need some other setting, please test if it works first. And perhaps +/// add a comment in the list above to save the effort of testing for the next +/// person. +/// +/// This doesn't block, but is marked as 'async' to hint that this must be called in +/// asynchronous execution context. +pub async fn init_tracing(service_name: &str) -> Option { + if std::env::var("OTEL_SDK_DISABLED") == Ok("true".to_string()) { + return None; + }; + Some(init_tracing_internal(service_name.to_string())) +} + +/// Like `init_tracing`, but creates a separate tokio Runtime for the tracing +/// tasks. +pub fn init_tracing_without_runtime( + service_name: &str, +) -> Option { + if std::env::var("OTEL_SDK_DISABLED") == Ok("true".to_string()) { + return None; + }; + + // The opentelemetry batch processor and the OTLP exporter needs a Tokio + // runtime. Create a dedicated runtime for them. One thread should be + // enough. + // + // (Alternatively, instead of batching, we could use the "simple + // processor", which doesn't need Tokio, and use "reqwest-blocking" + // feature for the OTLP exporter, which also doesn't need Tokio. However, + // batching is considered best practice, and also I have the feeling that + // the non-Tokio codepaths in the opentelemetry crate are less used and + // might be more buggy, so better to stay on the well-beaten path.) + // + // We leak the runtime so that it keeps running after we exit the + // function. + let runtime = Box::leak(Box::new( + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("otlp runtime thread") + .worker_threads(1) + .build() + .unwrap(), + )); + let _guard = runtime.enter(); + + Some(init_tracing_internal(service_name.to_string())) +} + +fn init_tracing_internal(service_name: String) -> opentelemetry::sdk::trace::Tracer { + // Set up exporter from the OTEL_EXPORTER_* environment variables + let mut exporter = opentelemetry_otlp::new_exporter().http().with_env(); + + // XXX opentelemetry-otlp v0.18.0 has a bug in how it uses the + // OTEL_EXPORTER_OTLP_ENDPOINT env variable. According to the + // OpenTelemetry spec at + // , + // the full exporter URL is formed by appending "/v1/traces" to the value + // of OTEL_EXPORTER_OTLP_ENDPOINT. However, opentelemetry-otlp only does + // that with the grpc-tonic exporter. Other exporters, like the HTTP + // exporter, use the URL from OTEL_EXPORTER_OTLP_ENDPOINT as is, without + // appending "/v1/traces". + // + // See https://github.com/open-telemetry/opentelemetry-rust/pull/950 + // + // Work around that by checking OTEL_EXPORTER_OTLP_ENDPOINT, and setting + // the endpoint url with the "/v1/traces" path ourselves. If the bug is + // fixed in a later version, we can remove this code. But if we don't + // remember to remove this, it won't do any harm either, as the crate will + // just ignore the OTEL_EXPORTER_OTLP_ENDPOINT setting when the endpoint + // is set directly with `with_endpoint`. + if std::env::var(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).is_err() { + if let Ok(mut endpoint) = std::env::var(OTEL_EXPORTER_OTLP_ENDPOINT) { + if !endpoint.ends_with('/') { + endpoint.push('/'); + } + endpoint.push_str("v1/traces"); + exporter = exporter.with_endpoint(endpoint); + } + } + + opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + .with_trace_config( + opentelemetry::sdk::trace::config().with_resource(Resource::new(vec![KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAME, + service_name, + )])), + ) + .install_batch(opentelemetry::runtime::Tokio) + .expect("could not initialize opentelemetry exporter") +} + +// Shutdown trace pipeline gracefully, so that it has a chance to send any +// pending traces before we exit. +pub fn shutdown_tracing() { + opentelemetry::global::shutdown_tracer_provider(); +} diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index acc38cfb9d..3a852b2207 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -20,7 +20,9 @@ clap = { version = "4", features = ["derive", "string"] } crossbeam-utils = { version = "0.8" } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } +futures = { version = "0.3" } futures-channel = { version = "0.3", features = ["sink"] } +futures-executor = { version = "0.3" } futures-task = { version = "0.3", default-features = false, features = ["std"] } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } indexmap = { version = "1", default-features = false, features = ["std"] } @@ -36,12 +38,16 @@ prost = { version = "0.11" } rand = { version = "0.8", features = ["small_rng"] } regex = { version = "1" } regex-syntax = { version = "0.6" } +reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] } +ring = { version = "0.16", features = ["std"] } +rustls = { version = "0.20", features = ["dangerous_configuration"] } scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } socket2 = { version = "0.4", default-features = false, features = ["all"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "sync", "time"] } tokio-util = { version = "0.7", features = ["codec", "io"] } +tonic = { version = "0.8", features = ["tls-roots"] } tower = { version = "0.4", features = ["balance", "buffer", "limit", "retry", "timeout", "util"] } tracing = { version = "0.1", features = ["log"] } tracing-core = { version = "0.1" } From 3e94fd5af31930eac0a1e5ec6f599d4fa68e4515 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 26 Jan 2023 14:15:49 +0200 Subject: [PATCH 047/102] Inherit OpenTelemetry context for compute startup from cloud console. This allows fine-grained distributed tracing of the 'start_compute' operation from the cloud console. The startup actions performed by 'compute_ctl' are now performed in a child of the 'start_compute' context, so you can trace through the whole compute start operation. This needs a corresponding change in the cloud console to fill in the 'startup_tracing_context' field in the json spec. If it's missing, the startup operations are simply traced as a separate trace, without a parent. --- Cargo.lock | 1 + compute_tools/Cargo.toml | 1 + compute_tools/src/bin/compute_ctl.rs | 26 ++++++++++++++++++++++++++ compute_tools/src/spec.rs | 3 +++ 4 files changed, 31 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 91515df5bb..f28eedfa56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -831,6 +831,7 @@ dependencies = [ "futures", "hyper", "notify", + "opentelemetry", "postgres", "regex", "serde", diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 0fabd23965..f8c3481f57 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -11,6 +11,7 @@ clap.workspace = true futures.workspace = true hyper = { workspace = true, features = ["full"] } notify.workspace = true +opentelemetry.workspace = true postgres.workspace = true regex.workspace = true serde.workspace = true diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 0e7f38bf84..2c42662020 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -84,6 +84,29 @@ fn main() -> Result<()> { } }; + // Extract OpenTelemetry context for the startup actions from the spec, and + // attach it to the current tracing context. + // + // This is used to propagate the context for the 'start_compute' operation + // from the neon control plane. This allows linking together the wider + // 'start_compute' operation that creates the compute container, with the + // startup actions here within the container. + // + // Switch to the startup context here, and exit it once the startup has + // completed and Postgres is up and running. + // + // NOTE: This is supposed to only cover the *startup* actions. Once + // postgres is configured and up-and-running, we exit this span. Any other + // actions that are performed on incoming HTTP requests, for example, are + // performed in separate spans. + let startup_context_guard = if let Some(ref carrier) = spec.startup_tracing_context { + use opentelemetry::propagation::TextMapPropagator; + use opentelemetry::sdk::propagation::TraceContextPropagator; + Some(TraceContextPropagator::new().extract(carrier).attach()) + } else { + None + }; + let pageserver_connstr = spec .cluster .settings @@ -140,6 +163,9 @@ fn main() -> Result<()> { // Wait for the child Postgres process forever. In this state Ctrl+C will // propagate to Postgres and it will be shut down as well. if let Some(mut pg) = pg { + // Startup is finished, exit the startup tracing span + drop(startup_context_guard); + let ecode = pg .wait() .expect("failed to start waiting on Postgres process"); diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 4c7cca545c..8e249f3722 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::path::Path; use std::str::FromStr; @@ -22,6 +23,8 @@ pub struct ComputeSpec { /// Expected cluster state at the end of transition process. pub cluster: Cluster, pub delta_operations: Option>, + + pub startup_tracing_context: Option>, } /// Cluster state seen from the perspective of the external tools From 0c0e15b81df0e118ddab4f85fb9ee2cdb729ee4f Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 26 Jan 2023 14:23:07 +0200 Subject: [PATCH 048/102] compute_ctl: Extract tracing context from incoming HTTP requests. This allows tracing the handling of HTTP requests as part of the caller's trace. --- Cargo.lock | 1 + compute_tools/src/http/api.rs | 24 +++++++-- libs/tracing-utils/Cargo.toml | 1 + libs/tracing-utils/src/http.rs | 96 ++++++++++++++++++++++++++++++++++ libs/tracing-utils/src/lib.rs | 7 +++ 5 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 libs/tracing-utils/src/http.rs diff --git a/Cargo.lock b/Cargo.lock index f28eedfa56..2985a654f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4169,6 +4169,7 @@ dependencies = [ name = "tracing-utils" version = "0.1.0" dependencies = [ + "hyper", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index f2a49f332c..74d733424d 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -8,11 +8,17 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; use serde_json; use tracing::{error, info}; +use tracing_utils::http::OtelName; use crate::compute::ComputeNode; // Service function to handle all available routes. -async fn routes(req: Request, compute: Arc) -> Response { +async fn routes(req: Request, compute: &Arc) -> Response { + // + // NOTE: The URI path is currently included in traces. That's OK because + // it doesn't contain any variable parts or sensitive information. But + // please keep that in mind if you change the routing here. + // match (req.method(), req.uri().path()) { // Serialized compute state. (&Method::GET, "/status") => { @@ -30,7 +36,7 @@ async fn routes(req: Request, compute: Arc) -> Response (&Method::POST, "/check_writability") => { info!("serving /check_writability POST request"); - let res = crate::checker::check_writability(&compute).await; + let res = crate::checker::check_writability(compute).await; match res { Ok(_) => Response::new(Body::from("true")), Err(e) => Response::new(Body::from(e.to_string())), @@ -56,7 +62,19 @@ async fn serve(state: Arc) { async move { Ok::<_, Infallible>(service_fn(move |req: Request| { let state = state.clone(); - async move { Ok::<_, Infallible>(routes(req, state).await) } + async move { + Ok::<_, Infallible>( + // NOTE: We include the URI path in the string. It + // doesn't contain any variable parts or sensitive + // information in this API. + tracing_utils::http::tracing_handler( + req, + |req| routes(req, &state), + OtelName::UriPath, + ) + .await, + ) + } })) } }); diff --git a/libs/tracing-utils/Cargo.toml b/libs/tracing-utils/Cargo.toml index 9d5cfb0d56..8c3d3f9063 100644 --- a/libs/tracing-utils/Cargo.toml +++ b/libs/tracing-utils/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +hyper.workspace = true opentelemetry = { workspace = true, features=["rt-tokio"] } opentelemetry-otlp = { workspace = true, default_features=false, features = ["http-proto", "trace", "http", "reqwest-client"] } opentelemetry-semantic-conventions.workspace = true diff --git a/libs/tracing-utils/src/http.rs b/libs/tracing-utils/src/http.rs new file mode 100644 index 0000000000..3f80f49de1 --- /dev/null +++ b/libs/tracing-utils/src/http.rs @@ -0,0 +1,96 @@ +//! Tracing wrapper for Hyper HTTP server + +use hyper::HeaderMap; +use hyper::{Body, Request, Response}; +use std::future::Future; +use tracing::Instrument; +use tracing_opentelemetry::OpenTelemetrySpanExt; + +/// Configuration option for what to use as the "otel.name" field in the traces. +pub enum OtelName<'a> { + /// Use a constant string + Constant(&'a str), + + /// Use the path from the request. + /// + /// That's very useful information, but is not appropriate if the + /// path contains parameters that differ on ever request, or worse, + /// sensitive information like usernames or email addresses. + /// + /// See + UriPath, +} + +/// Handle an incoming HTTP request using the given handler function, +/// with OpenTelemetry tracing. +/// +/// This runs 'handler' on the request in a new span, with fields filled in +/// from the request. Notably, if the request contains tracing information, +/// it is propagated to the span, so that this request is traced as part of +/// the same trace. +/// +/// XXX: Usually, this is handled by existing libraries, or built +/// directly into HTTP servers. However, I couldn't find one for Hyper, +/// so I had to write our own. OpenTelemetry website has a registry of +/// instrumentation libraries at: +/// https://opentelemetry.io/registry/?language=rust&component=instrumentation +/// If a Hyper crate appears, consider switching to that. +pub async fn tracing_handler( + req: Request, + handler: F, + otel_name: OtelName<'_>, +) -> Response +where + F: Fn(Request) -> R, + R: Future>, +{ + // Create a tracing span, with context propagated from the incoming + // request if any. + // + // See list of standard fields defined for HTTP requests at + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md + // We only fill in a few of the most useful ones here. + let otel_name = match otel_name { + OtelName::Constant(s) => s, + OtelName::UriPath => req.uri().path(), + }; + + let span = tracing::info_span!( + "http request", + otel.name= %otel_name, + http.method = %req.method(), + http.status_code = tracing::field::Empty, + ); + let parent_ctx = extract_remote_context(req.headers()); + span.set_parent(parent_ctx); + + // Handle the request within the span + let response = handler(req).instrument(span.clone()).await; + + // Fill in the fields from the response code + let status = response.status(); + span.record("http.status_code", status.as_str()); + span.record( + "otel.status_code", + if status.is_success() { "OK" } else { "ERROR" }, + ); + + response +} + +// Extract remote tracing context from the HTTP headers +fn extract_remote_context(headers: &HeaderMap) -> opentelemetry::Context { + struct HeaderExtractor<'a>(&'a HeaderMap); + + impl<'a> opentelemetry::propagation::Extractor for HeaderExtractor<'a> { + fn get(&self, key: &str) -> Option<&str> { + self.0.get(key).and_then(|value| value.to_str().ok()) + } + + fn keys(&self) -> Vec<&str> { + self.0.keys().map(|value| value.as_str()).collect() + } + } + let extractor = HeaderExtractor(headers); + opentelemetry::global::get_text_map_propagator(|propagator| propagator.extract(&extractor)) +} diff --git a/libs/tracing-utils/src/lib.rs b/libs/tracing-utils/src/lib.rs index 179f54c659..de0e2ad799 100644 --- a/libs/tracing-utils/src/lib.rs +++ b/libs/tracing-utils/src/lib.rs @@ -40,6 +40,8 @@ use opentelemetry_otlp::{OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_ pub use tracing_opentelemetry::OpenTelemetryLayer; +pub mod http; + /// Set up OpenTelemetry exporter, using configuration from environment variables. /// /// `service_name` is set as the OpenTelemetry 'service.name' resource (see @@ -141,6 +143,11 @@ fn init_tracing_internal(service_name: String) -> opentelemetry::sdk::trace::Tra } } + // Propagate trace information in the standard W3C TraceContext format. + opentelemetry::global::set_text_map_propagator( + opentelemetry::sdk::propagation::TraceContextPropagator::new(), + ); + opentelemetry_otlp::new_pipeline() .tracing() .with_exporter(exporter) From b858d70f194d9724ebcfe74052f548240d614810 Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Thu, 26 Jan 2023 14:26:23 +0100 Subject: [PATCH 049/102] Update promote job (#3455) To fix errors such as: `An error occurred (ImageAlreadyExistsException) when calling the PutImage operation: Image with digest 'sha256:da6d8ad97d84e3aec4e6a240c3a35868b626692ee5d199cdd3fe45d29a8e54df' and tag 'latest' already exists in the repository with name 'compute-node-v14' in registry with id '369495373322'` Co-authored-by: Rory de Zoete Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 37 ++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 202e7240c8..17bd8eee65 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -690,20 +690,37 @@ jobs: promote-images: runs-on: [ self-hosted, gen3, small ] needs: [ tag, test-images, vm-compute-node-image ] + container: golang:1.19-bullseye if: github.event_name != 'workflow_dispatch' - container: amazon/aws-cli - strategy: - fail-fast: false - matrix: - name: [ neon, compute-node-v14, vm-compute-node-v14, compute-node-v15, vm-compute-node-v15, compute-tools] - env: - AWS_DEFAULT_REGION: eu-central-1 steps: - - name: Promote image to latest + - name: Install Crane & ECR helper + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' run: | - export MANIFEST=$(aws ecr batch-get-image --repository-name ${{ matrix.name }} --image-ids imageTag=${{needs.tag.outputs.build-tag}} --query 'images[].imageManifest' --output text) - aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST" + go install github.com/google/go-containerregistry/cmd/crane@31786c6cbb82d6ec4fb8eb79cd9387905130534e # v0.11.0 + go install github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@69c85dc22db6511932bbf119e1a0cc5c90c69a7f # v0.6.0 + + - name: Configure ECR login + run: | + mkdir /github/home/.docker/ + echo "{\"credsStore\":\"ecr-login\"}" > /github/home/.docker/config.json + + - name: Add latest tag to images + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + run: | + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} latest + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} latest + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} latest + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v14:${{needs.tag.outputs.build-tag}} latest + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} latest + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} latest + + - name: Cleanup ECR folder + run: rm -rf ~/.ecr push-docker-hub: runs-on: [ self-hosted, dev, x64 ] From 2ecd0e1f0071aa16cd01c1c9c713d513164f6979 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Thu, 26 Jan 2023 16:18:57 +0100 Subject: [PATCH 050/102] Decommission link proxy from old account (#3454) --- .github/helm-values/production.proxy.yaml | 59 ----------------------- .github/workflows/build_and_test.yml | 1 - 2 files changed, 60 deletions(-) delete mode 100644 .github/helm-values/production.proxy.yaml diff --git a/.github/helm-values/production.proxy.yaml b/.github/helm-values/production.proxy.yaml deleted file mode 100644 index dbaf3cd096..0000000000 --- a/.github/helm-values/production.proxy.yaml +++ /dev/null @@ -1,59 +0,0 @@ -settings: - authBackend: "link" - authEndpoint: "https://console.neon.tech/authenticate_proxy_request/" - uri: "https://console.neon.tech/psql_session/" - sentryEnvironment: "production" - -# -- Additional labels for zenith-proxy pods -podLabels: - zenith_service: proxy - zenith_env: production - zenith_region: us-west-2 - zenith_region_slug: oregon - -service: - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: external - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip - service.beta.kubernetes.io/aws-load-balancer-scheme: internal - external-dns.alpha.kubernetes.io/hostname: proxy-release.local - type: LoadBalancer - -exposedService: - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: external - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip - service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing - external-dns.alpha.kubernetes.io/hostname: connect.neon.tech,pg.neon.tech - -metrics: - enabled: true - serviceMonitor: - enabled: true - selector: - release: kube-prometheus-stack - -extraManifests: - - apiVersion: operator.victoriametrics.com/v1beta1 - kind: VMServiceScrape - metadata: - name: "{{ include \"neon-proxy.fullname\" . }}" - labels: - helm.sh/chart: neon-proxy-{{ .Chart.Version }} - app.kubernetes.io/name: neon-proxy - app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}" - app.kubernetes.io/version: "{{ .Chart.AppVersion }}" - app.kubernetes.io/managed-by: Helm - namespace: "{{ .Release.Namespace }}" - spec: - selector: - matchLabels: - app.kubernetes.io/name: "neon-proxy" - endpoints: - - port: http - path: /metrics - interval: 10s - scrapeTimeout: 10s - namespaceSelector: - matchNames: - - "{{ .Release.Namespace }}" diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 17bd8eee65..d8e4e06210 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1035,7 +1035,6 @@ jobs: - name: Re-deploy proxy run: | DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade ${{ matrix.proxy_job }} neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s deploy-storage-broker: From bf63f129ae1b1822dfb2ec689e483f39c772c9a5 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 27 Jan 2023 00:49:02 +0200 Subject: [PATCH 051/102] Make 'branch_timeline' function more clear. Change the signature so that it takes an Arc reference to the source timeline, instead of just the ID. All the callers have an Arc reference at hand, so this is more convenient for everyone. Reorder the code a bit and improve the comments, to make it more clear what it does and why. --- pageserver/src/pgdatadir_mapping.rs | 2 +- pageserver/src/tenant.rs | 113 ++++++++++++++-------------- 2 files changed, 56 insertions(+), 59 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 6eea023af1..6f9035305d 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -1653,7 +1653,7 @@ mod tests { assert!(tline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&TESTREL_A)); // Create a branch, check that the relation is visible there - repo.branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Lsn(0x30))?; + repo.branch_timeline(&tline, NEW_TIMELINE_ID, Lsn(0x30))?; let newtline = match repo.get_timeline(NEW_TIMELINE_ID)?.local_timeline() { Some(timeline) => timeline, None => panic!("Should have a local timeline"), diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 1f0219aaa3..59450cf277 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1205,13 +1205,8 @@ impl Tenant { ancestor_timeline.wait_lsn(*lsn, ctx).await?; } - self.branch_timeline( - ancestor_timeline_id, - new_timeline_id, - ancestor_start_lsn, - ctx, - ) - .await? + self.branch_timeline(&ancestor_timeline, new_timeline_id, ancestor_start_lsn, ctx) + .await? } None => { self.bootstrap_timeline(new_timeline_id, pg_version, ctx) @@ -2050,54 +2045,53 @@ impl Tenant { /// Branch an existing timeline async fn branch_timeline( &self, - src: TimelineId, - dst: TimelineId, + src_timeline: &Arc, + dst_id: TimelineId, start_lsn: Option, _ctx: &RequestContext, ) -> anyhow::Result> { - // We need to hold this lock to prevent GC from starting at the same time. GC scans the directory to learn - // about timelines, so otherwise a race condition is possible, where we create new timeline and GC - // concurrently removes data that is needed by the new timeline. - let _gc_cs = self.gc_cs.lock().await; - let timeline_uninit_mark = { - let timelines = self.timelines.lock().unwrap(); - self.create_timeline_uninit_mark(dst, &timelines)? - }; - - // In order for the branch creation task to not wait for GC/compaction, - // we need to make sure that the starting LSN of the child branch is not out of scope midway by - // - // 1. holding the GC lock to prevent overwritting timeline's GC data - // 2. checking both the latest GC cutoff LSN and latest GC info of the source timeline - // - // Step 2 is to avoid initializing the new branch using data removed by past GC iterations - // or in-queue GC iterations. - - let src_timeline = self.get_timeline(src, false).with_context(|| { - format!( - "No ancestor {} found for timeline {}/{}", - src, self.tenant_id, dst - ) - })?; - - let latest_gc_cutoff_lsn = src_timeline.get_latest_gc_cutoff_lsn(); + let src_id = src_timeline.timeline_id; // If no start LSN is specified, we branch the new timeline from the source timeline's last record LSN let start_lsn = start_lsn.unwrap_or_else(|| { let lsn = src_timeline.get_last_record_lsn(); - info!("branching timeline {dst} from timeline {src} at last record LSN: {lsn}"); + info!("branching timeline {dst_id} from timeline {src_id} at last record LSN: {lsn}"); lsn }); - // Check if the starting LSN is out of scope because it is less than - // 1. the latest GC cutoff LSN or - // 2. the planned GC cutoff LSN, which is from an in-queue GC iteration. + // First acquire the GC lock so that another task cannot advance the GC + // cutoff in 'gc_info', and make 'start_lsn' invalid, while we are + // creating the branch. + let _gc_cs = self.gc_cs.lock().await; + + // Create a placeholder for the new branch. This will error + // out if the new timeline ID is already in use. + let timeline_uninit_mark = { + let timelines = self.timelines.lock().unwrap(); + self.create_timeline_uninit_mark(dst_id, &timelines)? + }; + + // Ensure that `start_lsn` is valid, i.e. the LSN is within the PITR + // horizon on the source timeline + // + // We check it against both the planned GC cutoff stored in 'gc_info', + // and the 'latest_gc_cutoff' of the last GC that was performed. The + // planned GC cutoff in 'gc_info' is normally larger than + // 'latest_gc_cutoff_lsn', but beware of corner cases like if you just + // changed the GC settings for the tenant to make the PITR window + // larger, but some of the data was already removed by an earlier GC + // iteration. + + // check against last actual 'latest_gc_cutoff' first + let latest_gc_cutoff_lsn = src_timeline.get_latest_gc_cutoff_lsn(); src_timeline .check_lsn_is_in_scope(start_lsn, &latest_gc_cutoff_lsn) .context(format!( "invalid branch start lsn: less than latest GC cutoff {}", *latest_gc_cutoff_lsn, ))?; + + // and then the planned GC cutoff { let gc_info = src_timeline.gc_info.read().unwrap(); let cutoff = min(gc_info.pitr_cutoff, gc_info.horizon_cutoff); @@ -2108,6 +2102,12 @@ impl Tenant { } } + // + // The branch point is valid, and we are still holding the 'gc_cs' lock + // so that GC cannot advance the GC cutoff until we are finished. + // Proceed with the branch creation. + // + // Determine prev-LSN for the new timeline. We can only determine it if // the timeline was branched at the current end of the source timeline. let RecordLsn { @@ -2126,7 +2126,7 @@ impl Tenant { let metadata = TimelineMetadata::new( start_lsn, dst_prev, - Some(src), + Some(src_id), start_lsn, *src_timeline.latest_gc_cutoff_lsn.read(), // FIXME: should we hold onto this guard longer? src_timeline.initdb_lsn, @@ -2135,15 +2135,15 @@ impl Tenant { let mut timelines = self.timelines.lock().unwrap(); let new_timeline = self .prepare_timeline( - dst, + dst_id, metadata, timeline_uninit_mark, false, - Some(src_timeline), + Some(Arc::clone(src_timeline)), )? .initialize_with_lock(&mut timelines, true, true)?; drop(timelines); - info!("branched timeline {dst} from {src} at {start_lsn}"); + info!("branched timeline {dst_id} from {src_id} at {start_lsn}"); Ok(new_timeline) } @@ -2966,7 +2966,7 @@ mod tests { // Branch the history, modify relation differently on the new timeline tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x30)), &ctx) + .branch_timeline(&tline, NEW_TIMELINE_ID, Some(Lsn(0x30)), &ctx) .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) @@ -3055,7 +3055,7 @@ mod tests { // try to branch at lsn 25, should fail because we already garbage collected the data match tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) + .branch_timeline(&tline, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) .await { Ok(_) => panic!("branching should have failed"), @@ -3078,12 +3078,13 @@ mod tests { TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? .load() .await; - let tline = - tenant.create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx)?; - let _tline = tline.initialize(&ctx)?; + + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx)? + .initialize(&ctx)?; // try to branch at lsn 0x25, should fail because initdb lsn is 0x50 match tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) + .branch_timeline(&tline, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) .await { Ok(_) => panic!("branching should have failed"), @@ -3134,7 +3135,7 @@ mod tests { make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) + .branch_timeline(&tline, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) @@ -3158,7 +3159,7 @@ mod tests { make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) + .branch_timeline(&tline, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) @@ -3214,7 +3215,7 @@ mod tests { make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant - .branch_timeline(TIMELINE_ID, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) + .branch_timeline(&tline, NEW_TIMELINE_ID, Some(Lsn(0x40)), &ctx) .await?; let newtline = tenant @@ -3499,16 +3500,14 @@ mod tests { keyspace.add_key(test_key); } - let mut tline_id = TIMELINE_ID; for _ in 0..50 { let new_tline_id = TimelineId::generate(); tenant - .branch_timeline(tline_id, new_tline_id, Some(lsn), &ctx) + .branch_timeline(&tline, new_tline_id, Some(lsn), &ctx) .await?; tline = tenant .get_timeline(new_tline_id, true) .expect("Should have the branched timeline"); - tline_id = new_tline_id; for _ in 0..NUM_KEYS { lsn = Lsn(lsn.0 + 0x10); @@ -3565,18 +3564,16 @@ mod tests { let mut updated = [[Lsn(0); NUM_KEYS]; NUM_TLINES]; let mut lsn = Lsn(0); - let mut tline_id = TIMELINE_ID; #[allow(clippy::needless_range_loop)] for idx in 0..NUM_TLINES { let new_tline_id = TimelineId::generate(); tenant - .branch_timeline(tline_id, new_tline_id, Some(lsn), &ctx) + .branch_timeline(&tline, new_tline_id, Some(lsn), &ctx) .await?; tline = tenant .get_timeline(new_tline_id, true) .expect("Should have the branched timeline"); - tline_id = new_tline_id; for _ in 0..NUM_KEYS { lsn = Lsn(lsn.0 + 0x10); From fb721cdfa56746e5b1b03c148d3df41d08ce056e Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Fri, 27 Jan 2023 11:05:05 +0100 Subject: [PATCH 052/102] Setup legacy scram proxy in the new account (#3461) This setup proxies with *.cloud.neon.tech certificate in the us-west-2 region of the new account, we are not switching to them here yet --- ...us-west-2-eta.neon-proxy-scram-legacy.yaml | 61 +++++++++++++++++++ .github/workflows/build_and_test.yml | 10 +++ 2 files changed, 71 insertions(+) create mode 100644 .github/helm-values/prod-us-west-2-eta.neon-proxy-scram-legacy.yaml diff --git a/.github/helm-values/prod-us-west-2-eta.neon-proxy-scram-legacy.yaml b/.github/helm-values/prod-us-west-2-eta.neon-proxy-scram-legacy.yaml new file mode 100644 index 0000000000..3a5cde4b01 --- /dev/null +++ b/.github/helm-values/prod-us-west-2-eta.neon-proxy-scram-legacy.yaml @@ -0,0 +1,61 @@ +# Helm chart values for neon-proxy-scram. +# This is a YAML-formatted file. + +image: + repository: neondatabase/neon + +settings: + authBackend: "console" + authEndpoint: "http://console-release.local/management/api/v2" + domain: "*.cloud.neon.tech" + sentryEnvironment: "production" + wssPort: 8443 + metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events" + metricCollectionInterval: "10min" + +# -- Additional labels for neon-proxy pods +podLabels: + zenith_service: proxy-scram + zenith_env: prod + zenith_region: us-west-2 + zenith_region_slug: us-west-2 + +exposedService: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + external-dns.alpha.kubernetes.io/hostname: neon-proxy-scram-legacy.eta.us-west-2.aws.neon.tech + httpsPort: 443 + +#metrics: +# enabled: true +# serviceMonitor: +# enabled: true +# selector: +# release: kube-prometheus-stack + +extraManifests: + - apiVersion: operator.victoriametrics.com/v1beta1 + kind: VMServiceScrape + metadata: + name: "{{ include \"neon-proxy.fullname\" . }}" + labels: + helm.sh/chart: neon-proxy-{{ .Chart.Version }} + app.kubernetes.io/name: neon-proxy + app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + app.kubernetes.io/managed-by: Helm + namespace: "{{ .Release.Namespace }}" + spec: + selector: + matchLabels: + app.kubernetes.io/name: "neon-proxy" + endpoints: + - port: http + path: /metrics + interval: 10s + scrapeTimeout: 10s + namespaceSelector: + matchNames: + - "{{ .Release.Namespace }}" diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d8e4e06210..53f33325e4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1186,15 +1186,19 @@ jobs: - target_region: us-east-2 target_cluster: prod-us-east-2-delta deploy_link_proxy: true + deploy_legacy_scram_proxy: false - target_region: us-west-2 target_cluster: prod-us-west-2-eta deploy_link_proxy: false + deploy_legacy_scram_proxy: true - target_region: eu-central-1 target_cluster: prod-eu-central-1-gamma deploy_link_proxy: false + deploy_legacy_scram_proxy: false - target_region: ap-southeast-1 target_cluster: prod-ap-southeast-1-epsilon deploy_link_proxy: false + deploy_legacy_scram_proxy: false environment: name: prod-${{ matrix.target_region }} steps: @@ -1220,6 +1224,12 @@ jobs: DOCKER_TAG=${{needs.tag.outputs.build-tag}} helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + - name: Re-deploy legacy scram proxy + if: matrix.deploy_legacy_scram_proxy + run: | + DOCKER_TAG=${{needs.tag.outputs.build-tag}} + helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + deploy-storage-broker-prod-new: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest From 238898131104b6059a22f11403f30831210b80e3 Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Fri, 27 Jan 2023 11:20:51 +0100 Subject: [PATCH 053/102] Add cleanup tasks for ansible and helm (#3465) To fix: https://github.com/neondatabase/neon/actions/runs/4023027504/jobs/6913421070 https://github.com/neondatabase/neon/actions/runs/4023027504/jobs/6913421268 Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 53f33325e4..0360fc347e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -879,6 +879,10 @@ jobs: ANSIBLE_CONFIG=./ansible.cfg ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} rm -f neon_install.tar.gz .neon_current_version + # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ansible/collections': Permission denied + - name: Cleanup ansible folder + run: rm -rf ~/.ansible + deploy-new: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned @@ -919,6 +923,9 @@ jobs: ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} rm -f neon_install.tar.gz .neon_current_version + - name: Cleanup ansible folder + run: rm -rf ~/.ansible + deploy-pr-test-new: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned @@ -952,6 +959,9 @@ jobs: ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} rm -f neon_install.tar.gz .neon_current_version + - name: Cleanup ansible folder + run: rm -rf ~/.ansible + deploy-prod-new: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest @@ -1037,6 +1047,9 @@ jobs: DOCKER_TAG=${{needs.tag.outputs.build-tag}} helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + - name: Cleanup helm folder + run: rm -rf ~/.cache + deploy-storage-broker: name: deploy storage broker on old staging and old prod runs-on: [ self-hosted, gen3, small ] @@ -1080,6 +1093,9 @@ jobs: run: helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace ${{ matrix.storage_broker_ns }} --create-namespace --install --atomic -f .github/helm-values/${{ matrix.storage_broker_config }}.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s + - name: Cleanup helm folder + run: rm -rf ~/.cache + deploy-proxy-new: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned @@ -1133,6 +1149,9 @@ jobs: DOCKER_TAG=${{needs.tag.outputs.build-tag}} helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + - name: Cleanup helm folder + run: rm -rf ~/.cache + deploy-storage-broker-dev-new: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned @@ -1169,6 +1188,9 @@ jobs: run: helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s + - name: Cleanup helm folder + run: rm -rf ~/.cache + deploy-proxy-prod-new: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest From 99399c112a737c035632410e847fac8fcd737ead Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 24 Jan 2023 19:00:18 +0100 Subject: [PATCH 054/102] move walreceiver module under timeline Walreceiver is a per-timeline abstraction. Move it there to reflect the hierarchy of abstractions and task_mgr tasks. The code that sets up the global storage_broker client is not timeline-scoped. So, break it out into a separate module. The motivation for this change is to prepare the code base for replacing the task_mgr global task registry with a more ownership-oriented approach to manage task lifetimes. I removed TaskStateUpdate::Init because, after doing the changes, rustc warned that it was never constructed. A quick search through the commit history shows that this has always been true since commit fb68d01449edb4be9a0d064d69a442dd3688783e Author: Dmitry Rodionov Date: Mon Sep 26 23:57:02 2022 +0300 Preserve task result in TaskHandle by keeping join handle around (#2521) So, the warning is not an indication of some accidental code removal. This is PR: https://github.com/neondatabase/neon/pull/3456 --- pageserver/src/bin/pageserver.rs | 2 +- pageserver/src/broker_client.rs | 48 +++++++++++++++++++ pageserver/src/lib.rs | 2 +- pageserver/src/tenant/timeline.rs | 5 +- .../src/{ => tenant/timeline}/walreceiver.rs | 44 ----------------- .../walreceiver/connection_manager.rs | 12 ++--- .../walreceiver/walreceiver_connection.rs | 3 +- 7 files changed, 61 insertions(+), 55 deletions(-) create mode 100644 pageserver/src/broker_client.rs rename pageserver/src/{ => tenant/timeline}/walreceiver.rs (83%) rename pageserver/src/{ => tenant/timeline}/walreceiver/connection_manager.rs (99%) rename pageserver/src/{ => tenant/timeline}/walreceiver/walreceiver_connection.rs (99%) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 52b0c7f2be..f2cd93bd3a 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -250,7 +250,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { let signals = signals::install_shutdown_handlers()?; // Launch broker client - WALRECEIVER_RUNTIME.block_on(pageserver::walreceiver::init_broker_client(conf))?; + WALRECEIVER_RUNTIME.block_on(pageserver::broker_client::init_broker_client(conf))?; // Initialize authentication for incoming connections let auth = match &conf.auth_type { diff --git a/pageserver/src/broker_client.rs b/pageserver/src/broker_client.rs new file mode 100644 index 0000000000..6c92967ca3 --- /dev/null +++ b/pageserver/src/broker_client.rs @@ -0,0 +1,48 @@ +//! The broker client instance of the pageserver, created during pageserver startup. +//! Used by each timelines' [`walreceiver`]. + +use crate::config::PageServerConf; + +use anyhow::Context; +use once_cell::sync::OnceCell; +use storage_broker::BrokerClientChannel; +use tracing::*; + +static BROKER_CLIENT: OnceCell = OnceCell::new(); + +/// +/// Initialize the broker client. This must be called once at page server startup. +/// +pub async fn init_broker_client(conf: &'static PageServerConf) -> anyhow::Result<()> { + let broker_endpoint = conf.broker_endpoint.clone(); + + // Note: we do not attempt connecting here (but validate endpoints sanity). + let broker_client = + storage_broker::connect(broker_endpoint.clone(), conf.broker_keepalive_interval).context( + format!( + "Failed to create broker client to {}", + &conf.broker_endpoint + ), + )?; + + if BROKER_CLIENT.set(broker_client).is_err() { + panic!("broker already initialized"); + } + + info!( + "Initialized broker client with endpoints: {}", + broker_endpoint + ); + Ok(()) +} + +/// +/// Get a handle to the broker client +/// +pub fn get_broker_client() -> &'static BrokerClientChannel { + BROKER_CLIENT.get().expect("broker client not initialized") +} + +pub fn is_broker_client_initialized() -> bool { + BROKER_CLIENT.get().is_some() +} diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 3ea2165da9..09e21ae755 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -1,5 +1,6 @@ mod auth; pub mod basebackup; +pub mod broker_client; pub mod config; pub mod consumption_metrics; pub mod context; @@ -16,7 +17,6 @@ pub mod tenant; pub mod trace; pub mod virtual_file; pub mod walingest; -pub mod walreceiver; pub mod walrecord; pub mod walredo; diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 134c8eed3c..c473a80f32 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1,5 +1,7 @@ //! +mod walreceiver; + use anyhow::{anyhow, bail, ensure, Context}; use bytes::Bytes; use fail::fail_point; @@ -23,6 +25,7 @@ use std::sync::atomic::{AtomicI64, Ordering as AtomicOrdering}; use std::sync::{Arc, Mutex, MutexGuard, RwLock, Weak}; use std::time::{Duration, Instant, SystemTime}; +use crate::broker_client::is_broker_client_initialized; use crate::context::{DownloadBehavior, RequestContext}; use crate::tenant::remote_timeline_client::{self, index::LayerFileMetadata}; use crate::tenant::storage_layer::{ @@ -59,11 +62,11 @@ use crate::page_cache; use crate::repository::GcResult; use crate::repository::{Key, Value}; use crate::task_mgr::TaskKind; -use crate::walreceiver::{is_broker_client_initialized, spawn_connection_manager_task}; use crate::walredo::WalRedoManager; use crate::METADATA_FILE_NAME; use crate::ZERO_PAGE; use crate::{is_temporary, task_mgr}; +use walreceiver::spawn_connection_manager_task; use super::remote_timeline_client::index::IndexPart; use super::remote_timeline_client::RemoteTimelineClient; diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/tenant/timeline/walreceiver.rs similarity index 83% rename from pageserver/src/walreceiver.rs rename to pageserver/src/tenant/timeline/walreceiver.rs index fc9daadc5c..f33a12c5cc 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/tenant/timeline/walreceiver.rs @@ -23,58 +23,15 @@ mod connection_manager; mod walreceiver_connection; -use crate::config::PageServerConf; use crate::task_mgr::WALRECEIVER_RUNTIME; -use anyhow::Context; -use once_cell::sync::OnceCell; use std::future::Future; -use storage_broker::BrokerClientChannel; use tokio::sync::watch; use tokio_util::sync::CancellationToken; use tracing::*; pub use connection_manager::spawn_connection_manager_task; -static BROKER_CLIENT: OnceCell = OnceCell::new(); - -/// -/// Initialize the broker client. This must be called once at page server startup. -/// -pub async fn init_broker_client(conf: &'static PageServerConf) -> anyhow::Result<()> { - let broker_endpoint = conf.broker_endpoint.clone(); - - // Note: we do not attempt connecting here (but validate endpoints sanity). - let broker_client = - storage_broker::connect(broker_endpoint.clone(), conf.broker_keepalive_interval).context( - format!( - "Failed to create broker client to {}", - &conf.broker_endpoint - ), - )?; - - if BROKER_CLIENT.set(broker_client).is_err() { - panic!("broker already initialized"); - } - - info!( - "Initialized broker client with endpoints: {}", - broker_endpoint - ); - Ok(()) -} - -/// -/// Get a handle to the broker client -/// -pub fn get_broker_client() -> &'static BrokerClientChannel { - BROKER_CLIENT.get().expect("broker client not initialized") -} - -pub fn is_broker_client_initialized() -> bool { - BROKER_CLIENT.get().is_some() -} - /// A handle of an asynchronous task. /// The task has a channel that it can use to communicate its lifecycle events in a certain form, see [`TaskEvent`] /// and a cancellation token that it can listen to for earlier interrupts. @@ -95,7 +52,6 @@ pub enum TaskEvent { #[derive(Debug, Clone)] pub enum TaskStateUpdate { - Init, Started, Progress(E), } diff --git a/pageserver/src/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs similarity index 99% rename from pageserver/src/walreceiver/connection_manager.rs rename to pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index 3af408c61b..cd7c7c51d2 100644 --- a/pageserver/src/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -11,11 +11,12 @@ use std::{collections::HashMap, num::NonZeroU64, ops::ControlFlow, sync::Arc, time::Duration}; +use super::TaskStateUpdate; +use crate::broker_client::get_broker_client; use crate::context::RequestContext; -use crate::task_mgr::TaskKind; use crate::task_mgr::WALRECEIVER_RUNTIME; +use crate::task_mgr::{self, TaskKind}; use crate::tenant::Timeline; -use crate::{task_mgr, walreceiver::TaskStateUpdate}; use anyhow::Context; use chrono::{NaiveDateTime, Utc}; use pageserver_api::models::TimelineState; @@ -28,10 +29,7 @@ use storage_broker::Streaming; use tokio::{select, sync::watch}; use tracing::*; -use crate::{ - exponential_backoff, walreceiver::get_broker_client, DEFAULT_BASE_BACKOFF_SECONDS, - DEFAULT_MAX_BACKOFF_SECONDS, -}; +use crate::{exponential_backoff, DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS}; use postgres_connection::{parse_host_port, PgConnectionConfig}; use utils::{ id::{NodeId, TenantTimelineId}, @@ -149,7 +147,7 @@ async fn connection_manager_loop_step( let wal_connection = walreceiver_state.wal_connection.as_mut() .expect("Should have a connection, as checked by the corresponding select! guard"); match wal_connection_update { - TaskEvent::Update(TaskStateUpdate::Init | TaskStateUpdate::Started) => {}, + TaskEvent::Update(TaskStateUpdate::Started) => {}, TaskEvent::Update(TaskStateUpdate::Progress(new_status)) => { if new_status.has_processed_wal { // We have advanced last_record_lsn by processing the WAL received diff --git a/pageserver/src/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs similarity index 99% rename from pageserver/src/walreceiver/walreceiver_connection.rs rename to pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 1836195fd4..7e06c398af 100644 --- a/pageserver/src/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -22,8 +22,9 @@ use tokio_postgres::{replication::ReplicationStream, Client}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; +use super::TaskStateUpdate; use crate::context::RequestContext; -use crate::{metrics::LIVE_CONNECTIONS_COUNT, walreceiver::TaskStateUpdate}; +use crate::metrics::LIVE_CONNECTIONS_COUNT; use crate::{ task_mgr, task_mgr::TaskKind, From 8342e9ea6f5ea5023458520f00c8293490205a95 Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Fri, 27 Jan 2023 13:28:26 +0100 Subject: [PATCH 055/102] Update helm job (#3467) As followup from https://github.com/neondatabase/build/pull/47 Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0360fc347e..9be8abcd9d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1006,7 +1006,7 @@ jobs: deploy-proxy: runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] if: | @@ -1029,18 +1029,13 @@ jobs: submodules: true fetch-depth: 0 - - name: Add curl - run: apt update && apt install curl -y - - name: Store kubeconfig file run: | echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG} chmod 0600 ${KUBECONFIG} - - name: Setup helm v3 - run: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts + - name: Add neon helm chart + run: helm repo add neondatabase https://neondatabase.github.io/helm-charts - name: Re-deploy proxy run: | @@ -1053,7 +1048,7 @@ jobs: deploy-storage-broker: name: deploy storage broker on old staging and old prod runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] if: | @@ -1076,18 +1071,13 @@ jobs: submodules: true fetch-depth: 0 - - name: Add curl - run: apt update && apt install curl -y - - name: Store kubeconfig file run: | echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG} chmod 0600 ${KUBECONFIG} - - name: Setup helm v3 - run: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts + - name: Add neon helm chart + run: helm repo add neondatabase https://neondatabase.github.io/helm-charts - name: Deploy storage-broker run: From 0ec84e2f1fa5505091e8d5263fdef418ed4ed7cd Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 27 Jan 2023 15:34:59 +0200 Subject: [PATCH 056/102] Allow creating config for attached tenant (#3446) Currently `attach` doesn't write a tenant config, because we don't back it up in the first place. The current implementation of `Tenant::persist_tenant_config` does not allow changing tenant's configuration through the http api which will fail because the file wasn't created on attach and `OpenOptions::truncate(true).write(true).create_new(false)` is used. I think this patch allows for least controversial middle ground which *enables* changing tenant configuration even for attached tenants (not just created tenants). --- pageserver/src/tenant.rs | 128 +++++++++++------------- pageserver/src/tenant/mgr.rs | 9 +- test_runner/regress/test_tenant_conf.py | 53 +++++++++- 3 files changed, 118 insertions(+), 72 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 59450cf277..feb302b6f4 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1795,69 +1795,70 @@ impl Tenant { } pub(super) fn persist_tenant_config( + tenant_id: &TenantId, target_config_path: &Path, tenant_conf: TenantConfOpt, - first_save: bool, + creating_tenant: bool, ) -> anyhow::Result<()> { let _enter = info_span!("saving tenantconf").entered(); - info!("persisting tenantconf to {}", target_config_path.display()); - // TODO this will prepend comments endlessly ? - let mut conf_content = r#"# This file contains a specific per-tenant's config. -# It is read in case of pageserver restart. - -[tenant_config] -"# - .to_string(); - - // Convert the config to a toml file. - conf_content += &toml_edit::easy::to_string(&tenant_conf)?; - - let mut target_config_file = VirtualFile::open_with_options( - target_config_path, - OpenOptions::new() - .truncate(true) // This needed for overwriting with small config files - .write(true) - .create_new(first_save), - )?; - - target_config_file - .write(conf_content.as_bytes()) - .context("Failed to write toml bytes into file") - .and_then(|_| { - target_config_file - .sync_all() - .context("Faile to fsync config file") - }) - .with_context(|| { + // imitate a try-block with a closure + let do_persist = |target_config_path: &Path| -> anyhow::Result<()> { + let target_config_parent = target_config_path.parent().with_context(|| { format!( - "Failed to write config file into path '{}'", + "Config path does not have a parent: {}", target_config_path.display() ) })?; - // fsync the parent directory to ensure the directory entry is durable - if first_save { - target_config_path - .parent() - .context("Config file does not have a parent") - .and_then(|target_config_parent| { - File::open(target_config_parent).context("Failed to open config parent") - }) - .and_then(|tenant_dir| { - tenant_dir - .sync_all() - .context("Failed to fsync config parent") - }) - .with_context(|| { - format!( - "Failed to fsync on first save for config {}", - target_config_path.display() - ) - })?; - } + info!("persisting tenantconf to {}", target_config_path.display()); - Ok(()) + let mut conf_content = r#"# This file contains a specific per-tenant's config. +# It is read in case of pageserver restart. + +[tenant_config] +"# + .to_string(); + + // Convert the config to a toml file. + conf_content += &toml_edit::easy::to_string(&tenant_conf)?; + + let mut target_config_file = VirtualFile::open_with_options( + target_config_path, + OpenOptions::new() + .truncate(true) // This needed for overwriting with small config files + .write(true) + .create_new(creating_tenant) + // when creating a new tenant, first_save will be true and `.create(true)` will be + // ignored (per rust std docs). + // + // later when updating the config of created tenant, or persisting config for the + // first time for attached tenant, the `.create(true)` is used. + .create(true), + )?; + + target_config_file + .write(conf_content.as_bytes()) + .context("write toml bytes into file") + .and_then(|_| target_config_file.sync_all().context("fsync config file")) + .context("write config file")?; + + // fsync the parent directory to ensure the directory entry is durable. + // before this was done conditionally on creating_tenant, but these management actions are rare + // enough to just fsync it always. + + crashsafe::fsync(target_config_parent)?; + Ok(()) + }; + + // this function is called from creating the tenant and updating the tenant config, which + // would otherwise share this context, so keep it here in one place. + do_persist(target_config_path).with_context(|| { + format!( + "write tenant {tenant_id} config to {}", + target_config_path.display() + ) + }) } // @@ -2512,26 +2513,19 @@ fn try_create_target_tenant_dir( target_tenant_directory, temporary_tenant_dir, ) - .with_context(|| format!("Failed to resolve tenant {tenant_id} temporary timelines dir"))?; + .with_context(|| format!("resolve tenant {tenant_id} temporary timelines dir"))?; let temporary_tenant_config_path = rebase_directory( &conf.tenant_config_path(tenant_id), target_tenant_directory, temporary_tenant_dir, ) - .with_context(|| format!("Failed to resolve tenant {tenant_id} temporary config path"))?; + .with_context(|| format!("resolve tenant {tenant_id} temporary config path"))?; + + Tenant::persist_tenant_config(&tenant_id, &temporary_tenant_config_path, tenant_conf, true)?; - Tenant::persist_tenant_config(&temporary_tenant_config_path, tenant_conf, true).with_context( - || { - format!( - "Failed to write tenant {} config to {}", - tenant_id, - temporary_tenant_config_path.display() - ) - }, - )?; crashsafe::create_dir(&temporary_tenant_timelines_dir).with_context(|| { format!( - "could not create tenant {} temporary timelines directory {}", + "create tenant {} temporary timelines directory {}", tenant_id, temporary_tenant_timelines_dir.display() ) @@ -2542,7 +2536,7 @@ fn try_create_target_tenant_dir( fs::rename(temporary_tenant_dir, target_tenant_directory).with_context(|| { format!( - "failed to move tenant {} temporary directory {} into the permanent one {}", + "move tenant {} temporary directory {} into the permanent one {}", tenant_id, temporary_tenant_dir.display(), target_tenant_directory.display() @@ -2550,14 +2544,14 @@ fn try_create_target_tenant_dir( })?; let target_dir_parent = target_tenant_directory.parent().with_context(|| { format!( - "Failed to get tenant {} dir parent for {}", + "get tenant {} dir parent for {}", tenant_id, target_tenant_directory.display() ) })?; crashsafe::fsync(target_dir_parent).with_context(|| { format!( - "Failed to fsync renamed directory's parent {} for tenant {}", + "fsync renamed directory's parent {} for tenant {}", target_dir_parent.display(), tenant_id, ) diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index e7b71d7dd7..a9edee3794 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -291,10 +291,11 @@ pub async fn update_tenant_config( tenant_id: TenantId, ) -> anyhow::Result<()> { info!("configuring tenant {tenant_id}"); - get_tenant(tenant_id, true) - .await? - .update_tenant_config(tenant_conf); - Tenant::persist_tenant_config(&conf.tenant_config_path(tenant_id), tenant_conf, false)?; + let tenant = get_tenant(tenant_id, true).await?; + + tenant.update_tenant_config(tenant_conf); + let tenant_config_path = conf.tenant_config_path(tenant_id); + Tenant::persist_tenant_config(&tenant.tenant_id(), &tenant_config_path, tenant_conf, false)?; Ok(()) } diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index f6a7977c32..cbbf01a285 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -2,7 +2,15 @@ from contextlib import closing import psycopg2.extras from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnvBuilder +from fixtures.neon_fixtures import ( + LocalFsStorage, + NeonEnvBuilder, + RemoteStorageKind, + assert_tenant_status, + wait_for_upload, +) +from fixtures.types import Lsn +from fixtures.utils import wait_until def test_tenant_config(neon_env_builder: NeonEnvBuilder): @@ -158,3 +166,46 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" "pitr_interval": 60, }.items() ) + + +def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_remote_storage( + remote_storage_kind=RemoteStorageKind.LOCAL_FS, + test_name="test_creating_tenant_conf_after_attach", + ) + + env = neon_env_builder.init_start() + assert isinstance(env.remote_storage, LocalFsStorage) + + # tenant is created with defaults, as in without config file + (tenant_id, timeline_id) = env.neon_cli.create_tenant() + config_path = env.repo_dir / "tenants" / str(tenant_id) / "config" + assert config_path.exists(), "config file is always initially created" + + http_client = env.pageserver.http_client() + + detail = http_client.timeline_detail(tenant_id, timeline_id) + last_record_lsn = Lsn(detail["last_record_lsn"]) + assert last_record_lsn.lsn_int != 0, "initdb must have executed" + + wait_for_upload(http_client, tenant_id, timeline_id, last_record_lsn) + + http_client.tenant_detach(tenant_id) + + assert not config_path.exists(), "detach did not remove config file" + + http_client.tenant_attach(tenant_id) + wait_until( + number_of_iterations=5, + interval=1, + func=lambda: assert_tenant_status(http_client, tenant_id, "Active"), + ) + + env.neon_cli.config_tenant(tenant_id, {"gc_horizon": "1000000"}) + contents_first = config_path.read_text() + env.neon_cli.config_tenant(tenant_id, {"gc_horizon": "0"}) + contents_later = config_path.read_text() + + # dont test applying the setting here, we have that another test case to show it + # we just care about being able to create the file + assert len(contents_first) > len(contents_later) From c5ca7d0c686e5c9286472375db650ccf97224215 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 27 Jan 2023 18:36:24 +0200 Subject: [PATCH 057/102] Implement asynchronous pipe for communication with walredo process (#3368) Co-authored-by: Christian Schwarz --- pageserver/src/walredo.rs | 329 +++++++++++++++++++++++++++----------- 1 file changed, 236 insertions(+), 93 deletions(-) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index fd0524016f..c943bf0a27 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -22,16 +22,18 @@ use byteorder::{ByteOrder, LittleEndian}; use bytes::{BufMut, Bytes, BytesMut}; use nix::poll::*; use serde::Serialize; +use std::collections::VecDeque; use std::fs::OpenOptions; use std::io::prelude::*; use std::io::{Error, ErrorKind}; use std::ops::{Deref, DerefMut}; +use std::os::fd::RawFd; use std::os::unix::io::AsRawFd; use std::os::unix::prelude::CommandExt; use std::path::PathBuf; use std::process::Stdio; use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; -use std::sync::Mutex; +use std::sync::{Mutex, MutexGuard}; use std::time::Duration; use std::time::Instant; use std::{fs, io}; @@ -90,6 +92,20 @@ pub trait WalRedoManager: Send + Sync { ) -> Result; } +struct ProcessInput { + child: NoLeakChild, + stdin: ChildStdin, + stderr_fd: RawFd, + stdout_fd: RawFd, + n_requests: usize, +} + +struct ProcessOutput { + stdout: ChildStdout, + pending_responses: VecDeque>, + n_processed_responses: usize, +} + /// /// This is the real implementation that uses a Postgres process to /// perform WAL replay. Only one thread can use the process at a time, @@ -101,7 +117,9 @@ pub struct PostgresRedoManager { tenant_id: TenantId, conf: &'static PageServerConf, - process: Mutex>, + stdout: Mutex>, + stdin: Mutex>, + stderr: Mutex>, } /// Can this request be served by neon redo functions @@ -209,16 +227,17 @@ impl PostgresRedoManager { PostgresRedoManager { tenant_id, conf, - process: Mutex::new(None), + stdin: Mutex::new(None), + stdout: Mutex::new(None), + stderr: Mutex::new(None), } } /// Launch process pre-emptively. Should not be needed except for benchmarking. - pub fn launch_process(&mut self, pg_version: u32) -> anyhow::Result<()> { - let inner = self.process.get_mut().unwrap(); - if inner.is_none() { - let p = PostgresRedoProcess::launch(self.conf, self.tenant_id, pg_version)?; - *inner = Some(p); + pub fn launch_process(&self, pg_version: u32) -> anyhow::Result<()> { + let mut proc = self.stdin.lock().unwrap(); + if proc.is_none() { + self.launch(&mut proc, pg_version)?; } Ok(()) } @@ -241,22 +260,19 @@ impl PostgresRedoManager { let start_time = Instant::now(); - let mut process_guard = self.process.lock().unwrap(); + let mut proc = self.stdin.lock().unwrap(); let lock_time = Instant::now(); // launch the WAL redo process on first use - if process_guard.is_none() { - let p = PostgresRedoProcess::launch(self.conf, self.tenant_id, pg_version)?; - *process_guard = Some(p); + if proc.is_none() { + self.launch(&mut proc, pg_version)?; } - let process = process_guard.as_mut().unwrap(); - WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64()); // Relational WAL records are applied using wal-redo-postgres let buf_tag = BufferTag { rel, blknum }; - let result = process - .apply_wal_records(buf_tag, base_img, records, wal_redo_timeout) + let result = self + .apply_wal_records(proc, buf_tag, base_img, records, wal_redo_timeout) .map_err(WalRedoError::IoError); let end_time = Instant::now(); @@ -295,8 +311,22 @@ impl PostgresRedoManager { base_img_lsn, lsn ); - let process = process_guard.take().unwrap(); - process.kill(); + // self.stdin only holds stdin & stderr as_raw_fd(). + // Dropping it as part of take() doesn't close them. + // The owning objects (ChildStdout and ChildStderr) are stored in + // self.stdout and self.stderr, respsectively. + // We intentionally keep them open here to avoid a race between + // currently running `apply_wal_records()` and a `launch()` call + // after we return here. + // The currently running `apply_wal_records()` must not read from + // the newly launched process. + // By keeping self.stdout and self.stderr open here, `launch()` will + // get other file descriptors for the new child's stdout and stderr, + // and hence the current `apply_wal_records()` calls will observe + // `output.stdout.as_raw_fd() != stdout_fd` . + if let Some(proc) = self.stdin.lock().unwrap().take() { + proc.child.kill_and_wait(); + } } result } @@ -595,32 +625,23 @@ impl CloseFileDescriptors for C { } } -/// -/// Handle to the Postgres WAL redo process -/// -struct PostgresRedoProcess { - tenant_id: TenantId, - child: NoLeakChild, - stdin: ChildStdin, - stdout: ChildStdout, - stderr: ChildStderr, -} - -impl PostgresRedoProcess { +impl PostgresRedoManager { // // Start postgres binary in special WAL redo mode. // - #[instrument(skip_all,fields(tenant_id=%tenant_id, pg_version=pg_version))] + #[instrument(skip_all,fields(tenant_id=%self.tenant_id, pg_version=pg_version))] fn launch( - conf: &PageServerConf, - tenant_id: TenantId, + &self, + input: &mut MutexGuard>, pg_version: u32, - ) -> Result { + ) -> Result<(), Error> { // FIXME: We need a dummy Postgres cluster to run the process in. Currently, we // just create one with constant name. That fails if you try to launch more than // one WAL redo manager concurrently. let datadir = path_with_suffix_extension( - conf.tenant_path(&tenant_id).join("wal-redo-datadir"), + self.conf + .tenant_path(&self.tenant_id) + .join("wal-redo-datadir"), TEMP_FILE_SUFFIX, ); @@ -634,10 +655,12 @@ impl PostgresRedoProcess { ) })?; } - let pg_bin_dir_path = conf + let pg_bin_dir_path = self + .conf .pg_bin_dir(pg_version) .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_bin_dir path: {e}")))?; - let pg_lib_dir_path = conf + let pg_lib_dir_path = self + .conf .pg_lib_dir(pg_version) .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_lib_dir path: {e}")))?; @@ -723,27 +746,31 @@ impl PostgresRedoProcess { // all fallible operations post-spawn are complete, so get rid of the guard let child = scopeguard::ScopeGuard::into_inner(child); - Ok(PostgresRedoProcess { - tenant_id, + **input = Some(ProcessInput { child, + stdout_fd: stdout.as_raw_fd(), + stderr_fd: stderr.as_raw_fd(), stdin, + n_requests: 0, + }); + + *self.stdout.lock().unwrap() = Some(ProcessOutput { stdout, - stderr, - }) + pending_responses: VecDeque::new(), + n_processed_responses: 0, + }); + *self.stderr.lock().unwrap() = Some(stderr); + + Ok(()) } - #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%self.child.id()))] - fn kill(self) { - self.child.kill_and_wait(); - } - - // // Apply given WAL records ('records') over an old page image. Returns // new page image. // - #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%self.child.id()))] + #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%input.as_ref().unwrap().child.id()))] fn apply_wal_records( - &mut self, + &self, + mut input: MutexGuard>, tag: BufferTag, base_img: Option, records: &[(Lsn, NeonWalRecord)], @@ -780,33 +807,23 @@ impl PostgresRedoProcess { build_get_page_msg(tag, &mut writebuf); WAL_REDO_RECORD_COUNTER.inc_by(records.len() as u64); - // The input is now in 'writebuf'. Do a blind write first, writing as much as - // we can, before calling poll(). That skips one call to poll() if the stdin is - // already available for writing, which it almost certainly is because the - // process is idle. - let mut nwrite = self.stdin.write(&writebuf)?; - - // We expect the WAL redo process to respond with an 8k page image. We read it - // into this buffer. - let mut resultbuf = vec![0; BLCKSZ.into()]; - let mut nresult: usize = 0; // # of bytes read into 'resultbuf' so far + let proc = input.as_mut().unwrap(); + let mut nwrite = 0usize; + let stdout_fd = proc.stdout_fd; // Prepare for calling poll() let mut pollfds = [ - PollFd::new(self.stdout.as_raw_fd(), PollFlags::POLLIN), - PollFd::new(self.stderr.as_raw_fd(), PollFlags::POLLIN), - PollFd::new(self.stdin.as_raw_fd(), PollFlags::POLLOUT), + PollFd::new(proc.stdin.as_raw_fd(), PollFlags::POLLOUT), + PollFd::new(proc.stderr_fd, PollFlags::POLLIN), + PollFd::new(stdout_fd, PollFlags::POLLIN), ]; - // We do three things simultaneously: send the old base image and WAL records to - // the child process's stdin, read the result from child's stdout, and forward any logging + // We do two things simultaneously: send the old base image and WAL records to + // the child process's stdin and forward any logging // information that the child writes to its stderr to the page server's log. - while nresult < BLCKSZ.into() { - // If we have more data to write, wake up if 'stdin' becomes writeable or - // we have data to read. Otherwise only wake up if there's data to read. - let nfds = if nwrite < writebuf.len() { 3 } else { 2 }; + while nwrite < writebuf.len() { let n = loop { - match nix::poll::poll(&mut pollfds[0..nfds], wal_redo_timeout.as_millis() as i32) { + match nix::poll::poll(&mut pollfds[0..2], wal_redo_timeout.as_millis() as i32) { Err(e) if e == nix::errno::Errno::EINTR => continue, res => break res, } @@ -820,14 +837,16 @@ impl PostgresRedoProcess { let err_revents = pollfds[1].revents().unwrap(); if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { let mut errbuf: [u8; 16384] = [0; 16384]; - let n = self.stderr.read(&mut errbuf)?; + let mut stderr_guard = self.stderr.lock().unwrap(); + let stderr = stderr_guard.as_mut().unwrap(); + let len = stderr.read(&mut errbuf)?; // The message might not be split correctly into lines here. But this is // good enough, the important thing is to get the message to the log. - if n > 0 { + if len > 0 { error!( "wal-redo-postgres: {}", - String::from_utf8_lossy(&errbuf[0..n]) + String::from_utf8_lossy(&errbuf[0..len]) ); // To make sure we capture all log from the process if it fails, keep @@ -841,33 +860,157 @@ impl PostgresRedoProcess { )); } - // If we have more data to write and 'stdin' is writeable, do write. - if nwrite < writebuf.len() { - let in_revents = pollfds[2].revents().unwrap(); - if in_revents & (PollFlags::POLLERR | PollFlags::POLLOUT) != PollFlags::empty() { - nwrite += self.stdin.write(&writebuf[nwrite..])?; - } else if in_revents.contains(PollFlags::POLLHUP) { - // We still have more data to write, but the process closed the pipe. - return Err(Error::new( - ErrorKind::BrokenPipe, - "WAL redo process closed its stdin unexpectedly", - )); - } - } - - // If we have some data in stdout, read it to the result buffer. - let out_revents = pollfds[0].revents().unwrap(); - if out_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { - nresult += self.stdout.read(&mut resultbuf[nresult..])?; - } else if out_revents.contains(PollFlags::POLLHUP) { + // If 'stdin' is writeable, do write. + let in_revents = pollfds[0].revents().unwrap(); + if in_revents & (PollFlags::POLLERR | PollFlags::POLLOUT) != PollFlags::empty() { + nwrite += proc.stdin.write(&writebuf[nwrite..])?; + } else if in_revents.contains(PollFlags::POLLHUP) { + // We still have more data to write, but the process closed the pipe. return Err(Error::new( ErrorKind::BrokenPipe, - "WAL redo process closed its stdout unexpectedly", + "WAL redo process closed its stdin unexpectedly", )); } } + let request_no = proc.n_requests; + proc.n_requests += 1; + drop(input); - Ok(Bytes::from(resultbuf)) + // To improve walredo performance we separate sending requests and receiving + // responses. Them are protected by different mutexes (output and input). + // If thread T1, T2, T3 send requests D1, D2, D3 to walredo process + // then there is not warranty that T1 will first granted output mutex lock. + // To address this issue we maintain number of sent requests, number of processed + // responses and ring buffer with pending responses. After sending response + // (under input mutex), threads remembers request number. Then it releases + // input mutex, locks output mutex and fetch in ring buffer all responses until + // its stored request number. The it takes correspondent element from + // pending responses ring buffer and truncate all empty elements from the front, + // advancing processed responses number. + + let mut output_guard = self.stdout.lock().unwrap(); + let output = output_guard.as_mut().unwrap(); + if output.stdout.as_raw_fd() != stdout_fd { + // If stdout file descriptor is changed then it means that walredo process is crashed and restarted. + // As far as ProcessInput and ProcessOutout are protected by different mutexes, + // it can happen that we send request to one process and waiting response from another. + // To prevent such situation we compare stdout file descriptors. + // As far as old stdout pipe is destroyed only after new one is created, + // it can not reuse the same file descriptor, so this check is safe. + // + // Cross-read this with the comment in apply_batch_postgres if result.is_err(). + // That's where we kill the child process. + return Err(Error::new( + ErrorKind::BrokenPipe, + "WAL redo process closed its stdout unexpectedly", + )); + } + let n_processed_responses = output.n_processed_responses; + while n_processed_responses + output.pending_responses.len() <= request_no { + // We expect the WAL redo process to respond with an 8k page image. We read it + // into this buffer. + let mut resultbuf = vec![0; BLCKSZ.into()]; + let mut nresult: usize = 0; // # of bytes read into 'resultbuf' so far + while nresult < BLCKSZ.into() { + // We do two things simultaneously: reading response from stdout + // and forward any logging information that the child writes to its stderr to the page server's log. + let n = loop { + match nix::poll::poll(&mut pollfds[1..3], wal_redo_timeout.as_millis() as i32) { + Err(e) if e == nix::errno::Errno::EINTR => continue, + res => break res, + } + }?; + + if n == 0 { + return Err(Error::new(ErrorKind::Other, "WAL redo timed out")); + } + + // If we have some messages in stderr, forward them to the log. + let err_revents = pollfds[1].revents().unwrap(); + if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { + let mut errbuf: [u8; 16384] = [0; 16384]; + let mut stderr_guard = self.stderr.lock().unwrap(); + let stderr = stderr_guard.as_mut().unwrap(); + let len = stderr.read(&mut errbuf)?; + + // The message might not be split correctly into lines here. But this is + // good enough, the important thing is to get the message to the log. + if len > 0 { + error!( + "wal-redo-postgres: {}", + String::from_utf8_lossy(&errbuf[0..len]) + ); + + // To make sure we capture all log from the process if it fails, keep + // reading from the stderr, before checking the stdout. + continue; + } + } else if err_revents.contains(PollFlags::POLLHUP) { + return Err(Error::new( + ErrorKind::BrokenPipe, + "WAL redo process closed its stderr unexpectedly", + )); + } + + // If we have some data in stdout, read it to the result buffer. + let out_revents = pollfds[2].revents().unwrap(); + if out_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { + nresult += output.stdout.read(&mut resultbuf[nresult..])?; + } else if out_revents.contains(PollFlags::POLLHUP) { + return Err(Error::new( + ErrorKind::BrokenPipe, + "WAL redo process closed its stdout unexpectedly", + )); + } + } + output + .pending_responses + .push_back(Some(Bytes::from(resultbuf))); + } + // Replace our request's response with None in `pending_responses`. + // Then make space in the ring buffer by clearing out any seqence of contiguous + // `None`'s from the front of `pending_responses`. + // NB: We can't pop_front() because other requests' responses because another + // requester might have grabbed the output mutex before us: + // T1: grab input mutex + // T1: send request_no 23 + // T1: release input mutex + // T2: grab input mutex + // T2: send request_no 24 + // T2: release input mutex + // T2: grab output mutex + // T2: n_processed_responses + output.pending_responses.len() <= request_no + // 23 0 24 + // T2: enters poll loop that reads stdout + // T2: put response for 23 into pending_responses + // T2: put response for 24 into pending_resposnes + // pending_responses now looks like this: Front Some(response_23) Some(response_24) Back + // T2: takes its response_24 + // pending_responses now looks like this: Front Some(response_23) None Back + // T2: does the while loop below + // pending_responses now looks like this: Front Some(response_23) None Back + // T2: releases output mutex + // T1: grabs output mutex + // T1: n_processed_responses + output.pending_responses.len() > request_no + // 23 2 23 + // T1: skips poll loop that reads stdout + // T1: takes its response_23 + // pending_responses now looks like this: Front None None Back + // T2: does the while loop below + // pending_responses now looks like this: Front Back + // n_processed_responses now has value 25 + let res = output.pending_responses[request_no - n_processed_responses] + .take() + .expect("we own this request_no, nobody else is supposed to take it"); + while let Some(front) = output.pending_responses.front() { + if front.is_none() { + output.pending_responses.pop_front(); + output.n_processed_responses += 1; + } else { + break; + } + } + Ok(res) } } From 4718c67c17f60402ac2e5a901a6bd9c290ea9fd9 Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Fri, 27 Jan 2023 18:05:49 +0100 Subject: [PATCH 058/102] Update deploy steps (#3470) First one isn't optimal, but as it was requested to run the runner as nonroot -> https://github.com/neondatabase/runner/pull/1#discussion_r1069909593 this job will need more significant refactoring. This should unblock the deployment process. --------- Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9be8abcd9d..641150e157 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -885,7 +885,9 @@ jobs: deploy-new: runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + options: --user root --privileged # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly needs: [ push-docker-hub, tag, regress-tests ] @@ -1117,6 +1119,12 @@ jobs: submodules: true fetch-depth: 0 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::369495373322:role/github-runner + aws-region: eu-central-1 + - name: Configure environment run: | helm repo add neondatabase https://neondatabase.github.io/helm-charts @@ -1169,6 +1177,12 @@ jobs: submodules: true fetch-depth: 0 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::369495373322:role/github-runner + aws-region: eu-central-1 + - name: Configure environment run: | helm repo add neondatabase https://neondatabase.github.io/helm-charts From 4d291d0e90d5d6e8598bb0a1e4fb54845dc9baec Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Fri, 27 Jan 2023 19:27:23 +0100 Subject: [PATCH 059/102] Prevent assume error (#3476) To fix `Error: The requested DurationSeconds exceeds the MaxSessionDuration set for this role.` Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 641150e157..73f8eb9dbf 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1124,6 +1124,8 @@ jobs: with: role-to-assume: arn:aws:iam::369495373322:role/github-runner aws-region: eu-central-1 + role-skip-session-tagging: true + role-duration-seconds: 1800 - name: Configure environment run: | @@ -1182,6 +1184,8 @@ jobs: with: role-to-assume: arn:aws:iam::369495373322:role/github-runner aws-region: eu-central-1 + role-skip-session-tagging: true + role-duration-seconds: 1800 - name: Configure environment run: | From 67d418e91c42a3e0f721797f1d092fd537b5bd8f Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Sun, 29 Jan 2023 12:40:50 +0200 Subject: [PATCH 060/102] Set the last_record_gauge to the value which was persisted metadata (#3460) ## Describe your changes Whenever a tenant is detached or the pageserver is restarted the pageserver_last_record_lsn metric is dropped This fix resurrects the value from the metadata whenever the tenant is attached again ## Issue ticket number and link [3571](https://github.com/neondatabase/cloud/issues/3571) ## Checklist before requesting a review - [X] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. --- pageserver/src/tenant/timeline.rs | 4 +++ test_runner/regress/test_tenant_detach.py | 40 +++++++++++++++++------ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index c473a80f32..ad68770404 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -945,6 +945,10 @@ impl Timeline { }; result.repartition_threshold = result.get_checkpoint_distance() / 10; result + .metrics + .last_record_gauge + .set(disk_consistent_lsn.0 as i64); + result }) } diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 1beca58298..6c3454b79b 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -6,6 +6,7 @@ from threading import Thread import asyncpg import pytest from fixtures.log_helper import log +from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, @@ -59,11 +60,11 @@ def test_tenant_reattach( # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() - pg = env.postgres.create_start("main", tenant_id=tenant_id) - with pg.cursor() as cur: - cur.execute("CREATE TABLE t(key int primary key, value text)") - cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'") - current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()")) + with env.postgres.create_start("main", tenant_id=tenant_id) as pg: + with pg.cursor() as cur: + cur.execute("CREATE TABLE t(key int primary key, value text)") + cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'") + current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()")) # Wait for the all data to be processed by the pageserver and uploaded in remote storage wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, current_lsn) @@ -78,15 +79,34 @@ def test_tenant_reattach( ".*failed to perform remote task UploadMetadata.*, will retry.*" ) + ps_metrics = parse_metrics(pageserver_http.get_metrics(), "pageserver") + tenant_metric_filter = { + "tenant_id": str(tenant_id), + "timeline_id": str(timeline_id), + } + pageserver_last_record_lsn_before_detach = int( + ps_metrics.query_one("pageserver_last_record_lsn", filter=tenant_metric_filter).value + ) + pageserver_http.tenant_detach(tenant_id) pageserver_http.tenant_attach(tenant_id) - with pg.cursor() as cur: - assert query_scalar(cur, "SELECT count(*) FROM t") == 100000 + time.sleep(1) # for metrics propagation - # Check that we had to retry the downloads - assert env.pageserver.log_contains(".*list prefixes.*failed, will retry.*") - assert env.pageserver.log_contains(".*download.*failed, will retry.*") + ps_metrics = parse_metrics(pageserver_http.get_metrics(), "pageserver") + pageserver_last_record_lsn = int( + ps_metrics.query_one("pageserver_last_record_lsn", filter=tenant_metric_filter).value + ) + + assert pageserver_last_record_lsn_before_detach == pageserver_last_record_lsn + + with env.postgres.create_start("main", tenant_id=tenant_id) as pg: + with pg.cursor() as cur: + assert query_scalar(cur, "SELECT count(*) FROM t") == 100000 + + # Check that we had to retry the downloads + assert env.pageserver.log_contains(".*list prefixes.*failed, will retry.*") + assert env.pageserver.log_contains(".*download.*failed, will retry.*") num_connections = 10 From ddb9c2fe940b6c7dd917091ec6686fab1d060e64 Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Sun, 29 Jan 2023 14:04:06 +0200 Subject: [PATCH 061/102] Add metrics for tenants state (#3448) ## Describe your changes Added a metric that allow to monitor tenants state ## Issue ticket number and link https://github.com/neondatabase/neon/issues/3161 ## Checklist before requesting a review - [X] I have performed a self-review of my code. - [X] I have added an e2e test for it. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. --- libs/pageserver_api/src/models.rs | 18 +++++++++++ pageserver/src/metrics.rs | 23 ++++++++++++++ pageserver/src/tenant.rs | 29 +++++++++++++++++- test_runner/fixtures/metrics.py | 1 + test_runner/regress/test_tenants.py | 47 ++++++++++++++++++++++++++++- 5 files changed, 116 insertions(+), 2 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index d81e05aaa2..0d7aa2db55 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -29,6 +29,14 @@ pub enum TenantState { Broken, } +pub mod state { + pub const LOADING: &str = "loading"; + pub const ATTACHING: &str = "attaching"; + pub const ACTIVE: &str = "active"; + pub const STOPPING: &str = "stopping"; + pub const BROKEN: &str = "broken"; +} + impl TenantState { pub fn has_in_progress_downloads(&self) -> bool { match self { @@ -39,6 +47,16 @@ impl TenantState { Self::Broken => false, } } + + pub fn as_str(&self) -> &'static str { + match self { + TenantState::Loading => state::LOADING, + TenantState::Attaching => state::ATTACHING, + TenantState::Active => state::ACTIVE, + TenantState::Stopping => state::STOPPING, + TenantState::Broken => state::BROKEN, + } + } } /// A state of a timeline in pageserver's memory. diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index b61e64048b..7836712ca6 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -5,6 +5,7 @@ use metrics::{ IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec, }; use once_cell::sync::Lazy; +use pageserver_api::models::state; use utils::id::{TenantId, TimelineId}; /// Prometheus histogram buckets (in seconds) that capture the majority of @@ -112,6 +113,24 @@ static CURRENT_LOGICAL_SIZE: Lazy = Lazy::new(|| { .expect("failed to define current logical size metric") }); +// Metrics collected on tenant states. +const TENANT_STATE_OPTIONS: &[&str] = &[ + state::LOADING, + state::ATTACHING, + state::ACTIVE, + state::STOPPING, + state::BROKEN, +]; + +pub static TENANT_STATE_METRIC: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_tenant_states_count", + "Count of tenants per state", + &["tenant_id", "state"] + ) + .expect("Failed to register pageserver_tenant_states_count metric") +}); + // Metrics for cloud upload. These metrics reflect data uploaded to cloud storage, // or in testing they estimate how much we would upload if we did. static NUM_PERSISTENT_FILES_CREATED: Lazy = Lazy::new(|| { @@ -496,6 +515,10 @@ impl Drop for TimelineMetrics { pub fn remove_tenant_metrics(tenant_id: &TenantId) { let _ = STORAGE_TIME.remove_label_values(&["gc", &tenant_id.to_string(), "-"]); + let tid = tenant_id.to_string(); + for state in TENANT_STATE_OPTIONS { + let _ = TENANT_STATE_METRIC.remove_label_values(&[&tid, state]); + } } use futures::Future; diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index feb302b6f4..9fb540cfae 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -51,6 +51,7 @@ use crate::config::PageServerConf; use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir; use crate::is_uninit_mark; +use crate::metrics::TENANT_STATE_METRIC; use crate::metrics::{remove_tenant_metrics, STORAGE_TIME}; use crate::repository::GcResult; use crate::task_mgr; @@ -1736,7 +1737,33 @@ impl Tenant { tenant_id: TenantId, remote_storage: Option, ) -> Tenant { - let (state, _) = watch::channel(state); + let (state, mut rx) = watch::channel(state); + + tokio::spawn(async move { + let current_state = *rx.borrow_and_update(); + let tid = tenant_id.to_string(); + TENANT_STATE_METRIC + .with_label_values(&[&tid, current_state.as_str()]) + .inc(); + loop { + match rx.changed().await { + Ok(()) => { + let new_state = *rx.borrow(); + TENANT_STATE_METRIC + .with_label_values(&[&tid, current_state.as_str()]) + .dec(); + TENANT_STATE_METRIC + .with_label_values(&[&tid, new_state.as_str()]) + .inc(); + } + Err(_sender_dropped_error) => { + info!("Tenant dropped the state updates sender, quitting waiting for tenant state change"); + return; + } + } + } + }); + Tenant { tenant_id, conf, diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index 8b78e06c22..b5e2876074 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -69,5 +69,6 @@ PAGESERVER_PER_TENANT_METRICS: Tuple[str, ...] = ( "pageserver_wait_lsn_seconds_sum", "pageserver_created_persistent_files_total", "pageserver_written_persistent_bytes_total", + "pageserver_tenant_states_count", *PAGESERVER_PER_TENANT_REMOTE_TIMELINE_CLIENT_METRICS, ) diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 9477ae3c25..ce77c3940b 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -1,5 +1,6 @@ import os import shutil +import time from contextlib import closing from datetime import datetime from pathlib import Path @@ -259,7 +260,7 @@ def test_pageserver_with_empty_tenants( files_in_timelines_dir == 0 ), f"Tenant {tenant_with_empty_timelines_dir} should have an empty timelines/ directory" - # Trigger timeline reinitialization after pageserver restart + # Trigger timeline re-initialization after pageserver restart env.postgres.stop_all() env.pageserver.stop() @@ -278,7 +279,51 @@ def test_pageserver_with_empty_tenants( broken_tenant["state"] == "Broken" ), f"Tenant {tenant_without_timelines_dir} without timelines dir should be broken" + broken_tenant_status = client.tenant_status(tenant_without_timelines_dir) + assert ( + broken_tenant_status["state"] == "Broken" + ), f"Tenant {tenant_without_timelines_dir} without timelines dir should be broken" + + assert env.pageserver.log_contains(".*Setting tenant as Broken state, reason:.*") + [loaded_tenant] = [t for t in tenants if t["id"] == str(tenant_with_empty_timelines_dir)] assert ( loaded_tenant["state"] == "Active" ), "Tenant {tenant_with_empty_timelines_dir} with empty timelines dir should be active and ready for timeline creation" + + loaded_tenant_status = client.tenant_status(tenant_with_empty_timelines_dir) + assert ( + loaded_tenant_status["state"] == "Active" + ), f"Tenant {tenant_with_empty_timelines_dir} without timelines dir should be active" + + time.sleep(1) # to allow metrics propagation + + ps_metrics = parse_metrics(client.get_metrics(), "pageserver") + broken_tenants_metric_filter = { + "tenant_id": str(tenant_without_timelines_dir), + "state": "broken", + } + active_tenants_metric_filter = { + "tenant_id": str(tenant_with_empty_timelines_dir), + "state": "active", + } + + tenant_active_count = int( + ps_metrics.query_one( + "pageserver_tenant_states_count", filter=active_tenants_metric_filter + ).value + ) + + assert ( + tenant_active_count == 1 + ), f"Tenant {tenant_with_empty_timelines_dir} should have metric as active" + + tenant_broken_count = int( + ps_metrics.query_one( + "pageserver_tenant_states_count", filter=broken_tenants_metric_filter + ).value + ) + + assert ( + tenant_broken_count == 1 + ), f"Tenant {tenant_without_timelines_dir} should have metric as broken" From 5ee77c0b1fea79dbee69e291f6b084197801a785 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 27 Jan 2023 18:41:07 +0200 Subject: [PATCH 062/102] Fix holding tracing span guard over query execution. I added these spans to trace how long the queries take, but I didn't realize that there's a difference between: let _ = span.entered(); and let _guard = span.entered(); The former drops the guard immediately, while the latter holds it until the end of the scope. As a result, the span was ended immediately, and the query was executed outside the span. --- compute_tools/src/spec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 8e249f3722..bbd0ec21ed 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -387,13 +387,13 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { name.pg_quote(), db.owner.pg_quote() ); - let _ = info_span!("executing", query).entered(); + let _guard = info_span!("executing", query).entered(); client.execute(query.as_str(), &[])?; } DatabaseAction::Create => { let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote()); query.push_str(&db.to_pg_options()); - let _ = info_span!("executing", query).entered(); + let _guard = info_span!("executing", query).entered(); client.execute(query.as_str(), &[])?; } }; From 5fc233964a85f73d15e6e5743327744346094f48 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Mon, 30 Jan 2023 10:36:57 +0100 Subject: [PATCH 063/102] Create release PR --- .github/workflows/release.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..74a2af8e96 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,33 @@ +name: Create Release Branch + +on: + schedule: + - cron: '0 10 * * 2' + +jobs: + create_release_branch: + runs-on: [ubuntu-latest] + + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + ref: main + + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + + - name: Create release branch + run: git checkout -b release/${{ steps.date.outputs.date }} + + - name: Push new branch + run: git push origin release/${{ steps.date.outputs.date }} + + - name: Create pull request into release + uses: thomaseizinger/create-pull-request@1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + head: release/${{ steps.date.outputs.date }} + base: release + title: Release ${{ steps.date.outputs.date }} From 7bb13569b311abcc0889aa7642158addcc98f86b Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Mon, 30 Jan 2023 14:00:44 +0100 Subject: [PATCH 064/102] Switch more jobs to small runner (#3483) As these jobs don't benefit from additional cores Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 73f8eb9dbf..7559b527e3 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -337,7 +337,7 @@ jobs: uses: ./.github/actions/save-coverage-data benchmarks: - runs-on: [ self-hosted, gen3, large ] + runs-on: [ self-hosted, gen3, small ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -405,7 +405,7 @@ jobs: DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json coverage-report: - runs-on: [ self-hosted, gen3, large ] + runs-on: [ self-hosted, gen3, small ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init @@ -481,7 +481,7 @@ jobs: }" trigger-e2e-tests: - runs-on: [ self-hosted, gen3, large ] + runs-on: [ self-hosted, gen3, small ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned options: --init From c61bc25ef9945eb618a1db49d0dc93ce40e3d350 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Mon, 30 Jan 2023 16:08:23 +0200 Subject: [PATCH 065/102] Clean up NeedsDownload error (#3464) --- pageserver/src/http/routes.rs | 4 +--- pageserver/src/tenant/timeline.rs | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 750a242361..a7802f3cbe 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -87,9 +87,7 @@ fn apierror_from_prerror(err: PageReconstructError) -> ApiError { PageReconstructError::NeedsDownload(_, _) => { // This shouldn't happen, because we use a RequestContext that requests to // download any missing layer files on-demand. - ApiError::InternalServerError(anyhow::anyhow!( - "would need to download remote layer file" - )) + ApiError::InternalServerError(anyhow::anyhow!("need to download remote layer file")) } PageReconstructError::Cancelled => { ApiError::InternalServerError(anyhow::anyhow!("request was cancelled")) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index ad68770404..7b796a0749 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -15,6 +15,7 @@ use pageserver_api::models::{ use tokio::sync::{oneshot, watch, Semaphore, TryAcquireError}; use tokio_util::sync::CancellationToken; use tracing::*; +use utils::id::TenantTimelineId; use std::cmp::{max, min, Ordering}; use std::collections::HashMap; @@ -382,7 +383,7 @@ pub enum PageReconstructError { Other(#[from] anyhow::Error), // source and Display delegate to anyhow::Error /// The operation would require downloading a layer that is missing locally. - NeedsDownload(Weak, Weak), + NeedsDownload(TenantTimelineId, LayerFileName), /// The operation was cancelled Cancelled, @@ -396,7 +397,14 @@ impl std::fmt::Debug for PageReconstructError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { match self { Self::Other(err) => err.fmt(f), - Self::NeedsDownload(_tli, _layer) => write!(f, "needs download"), + Self::NeedsDownload(tenant_timeline_id, layer_file_name) => { + write!( + f, + "layer {}/{} needs download", + tenant_timeline_id, + layer_file_name.file_name() + ) + } Self::Cancelled => write!(f, "cancelled"), Self::WalRedo(err) => err.fmt(f), } @@ -407,7 +415,14 @@ impl std::fmt::Display for PageReconstructError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { match self { Self::Other(err) => err.fmt(f), - Self::NeedsDownload(_tli, _layer) => write!(f, "needs download"), + Self::NeedsDownload(tenant_timeline_id, layer_file_name) => { + write!( + f, + "layer {}/{} needs download", + tenant_timeline_id, + layer_file_name.file_name() + ) + } Self::Cancelled => write!(f, "cancelled"), Self::WalRedo(err) => err.fmt(f), } @@ -1879,8 +1894,8 @@ impl Timeline { } (DownloadBehavior::Error, false) => { return Err(PageReconstructError::NeedsDownload( - timeline.myself.clone(), - Arc::downgrade(&remote_layer), + TenantTimelineId::new(self.tenant_id, self.timeline_id), + remote_layer.file_name.clone(), )) } } From 20b38acff0287df29050653327eb12a5959e9298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lassi=20P=C3=B6l=C3=B6nen?= Date: Mon, 30 Jan 2023 17:10:29 +0200 Subject: [PATCH 066/102] Replace per timeline `pageserver_storage_operations_seconds` with a global one (#3409) Related to: https://github.com/neondatabase/neon/issues/2848 `pageserver_storage_operations_seconds` is the most expensive metric we have, as there are a lot of tenants/timelines and the histogram had 42 buckets. These are quite sparse too, so instead of having a histogram per timeline, create a new histogram `pageserver_storage_operations_seconds_global` without tenant and timeline dimensions and replace `pageserver_storage_operations_seconds` with sum and counter. Co-authored-by: Joonas Koivunen --- libs/metrics/src/lib.rs | 1 + pageserver/src/metrics.rs | 143 +++++++++++++++++++++------- pageserver/src/tenant.rs | 17 +--- pageserver/src/tenant/timeline.rs | 13 ++- test_runner/fixtures/metrics.py | 11 ++- test_runner/regress/test_tenants.py | 9 ++ 6 files changed, 143 insertions(+), 51 deletions(-) diff --git a/libs/metrics/src/lib.rs b/libs/metrics/src/lib.rs index 880ab0e83c..07d220195b 100644 --- a/libs/metrics/src/lib.rs +++ b/libs/metrics/src/lib.rs @@ -8,6 +8,7 @@ pub use prometheus::opts; pub use prometheus::register; pub use prometheus::{core, default_registry, proto}; pub use prometheus::{exponential_buckets, linear_buckets}; +pub use prometheus::{register_counter_vec, Counter, CounterVec}; pub use prometheus::{register_gauge, Gauge}; pub use prometheus::{register_gauge_vec, GaugeVec}; pub use prometheus::{register_histogram, Histogram}; diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 7836712ca6..6bd0eddbb5 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1,8 +1,9 @@ use metrics::core::{AtomicU64, GenericCounter}; use metrics::{ - register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec, - register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec, Histogram, HistogramVec, - IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec, + register_counter_vec, register_histogram, register_histogram_vec, register_int_counter, + register_int_counter_vec, register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec, + Counter, CounterVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, + UIntGauge, UIntGaugeVec, }; use once_cell::sync::Lazy; use pageserver_api::models::state; @@ -36,11 +37,29 @@ const STORAGE_TIME_OPERATIONS: &[&str] = &[ "gc", ]; -pub static STORAGE_TIME: Lazy = Lazy::new(|| { - register_histogram_vec!( - "pageserver_storage_operations_seconds", - "Time spent on storage operations", +pub static STORAGE_TIME_SUM_PER_TIMELINE: Lazy = Lazy::new(|| { + register_counter_vec!( + "pageserver_storage_operations_seconds_sum", + "Total time spent on storage operations with operation, tenant and timeline dimensions", &["operation", "tenant_id", "timeline_id"], + ) + .expect("failed to define a metric") +}); + +pub static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "pageserver_storage_operations_seconds_count", + "Count of storage operations with operation, tenant and timeline dimensions", + &["operation", "tenant_id", "timeline_id"], + ) + .expect("failed to define a metric") +}); + +pub static STORAGE_TIME_GLOBAL: Lazy = Lazy::new(|| { + register_histogram_vec!( + "pageserver_storage_operations_seconds_global", + "Time spent on storage operations", + &["operation"], get_buckets_for_critical_operations(), ) .expect("failed to define a metric") @@ -394,18 +413,81 @@ pub static WAL_REDO_RECORD_COUNTER: Lazy = Lazy::new(|| { .unwrap() }); +/// Similar to [`prometheus::HistogramTimer`] but does not record on drop. +pub struct StorageTimeMetricsTimer { + metrics: StorageTimeMetrics, + start: Instant, +} + +impl StorageTimeMetricsTimer { + fn new(metrics: StorageTimeMetrics) -> Self { + Self { + metrics, + start: Instant::now(), + } + } + + /// Record the time from creation to now. + pub fn stop_and_record(self) { + let duration = self.start.elapsed().as_secs_f64(); + self.metrics.timeline_sum.inc_by(duration); + self.metrics.timeline_count.inc(); + self.metrics.global_histogram.observe(duration); + } +} + +/// Timing facilities for an globally histogrammed metric, which is supported by per tenant and +/// timeline total sum and count. +#[derive(Clone, Debug)] +pub struct StorageTimeMetrics { + /// Sum of f64 seconds, per operation, tenant_id and timeline_id + timeline_sum: Counter, + /// Number of oeprations, per operation, tenant_id and timeline_id + timeline_count: IntCounter, + /// Global histogram having only the "operation" label. + global_histogram: Histogram, +} + +impl StorageTimeMetrics { + pub fn new(operation: &str, tenant_id: &str, timeline_id: &str) -> Self { + let timeline_sum = STORAGE_TIME_SUM_PER_TIMELINE + .get_metric_with_label_values(&[operation, tenant_id, timeline_id]) + .unwrap(); + let timeline_count = STORAGE_TIME_COUNT_PER_TIMELINE + .get_metric_with_label_values(&[operation, tenant_id, timeline_id]) + .unwrap(); + let global_histogram = STORAGE_TIME_GLOBAL + .get_metric_with_label_values(&[operation]) + .unwrap(); + + StorageTimeMetrics { + timeline_sum, + timeline_count, + global_histogram, + } + } + + /// Starts timing a new operation. + /// + /// Note: unlike [`prometheus::HistogramTimer`] the returned timer does not record on drop. + pub fn start_timer(&self) -> StorageTimeMetricsTimer { + StorageTimeMetricsTimer::new(self.clone()) + } +} + #[derive(Debug)] pub struct TimelineMetrics { tenant_id: String, timeline_id: String, pub reconstruct_time_histo: Histogram, pub materialized_page_cache_hit_counter: GenericCounter, - pub flush_time_histo: Histogram, - pub compact_time_histo: Histogram, - pub create_images_time_histo: Histogram, - pub init_logical_size_histo: Histogram, - pub logical_size_histo: Histogram, - pub load_layer_map_histo: Histogram, + pub flush_time_histo: StorageTimeMetrics, + pub compact_time_histo: StorageTimeMetrics, + pub create_images_time_histo: StorageTimeMetrics, + pub init_logical_size_histo: StorageTimeMetrics, + pub logical_size_histo: StorageTimeMetrics, + pub load_layer_map_histo: StorageTimeMetrics, + pub garbage_collect_histo: StorageTimeMetrics, pub last_record_gauge: IntGauge, pub wait_lsn_time_histo: Histogram, pub resident_physical_size_gauge: UIntGauge, @@ -425,24 +507,16 @@ impl TimelineMetrics { let materialized_page_cache_hit_counter = MATERIALIZED_PAGE_CACHE_HIT .get_metric_with_label_values(&[&tenant_id, &timeline_id]) .unwrap(); - let flush_time_histo = STORAGE_TIME - .get_metric_with_label_values(&["layer flush", &tenant_id, &timeline_id]) - .unwrap(); - let compact_time_histo = STORAGE_TIME - .get_metric_with_label_values(&["compact", &tenant_id, &timeline_id]) - .unwrap(); - let create_images_time_histo = STORAGE_TIME - .get_metric_with_label_values(&["create images", &tenant_id, &timeline_id]) - .unwrap(); - let init_logical_size_histo = STORAGE_TIME - .get_metric_with_label_values(&["init logical size", &tenant_id, &timeline_id]) - .unwrap(); - let logical_size_histo = STORAGE_TIME - .get_metric_with_label_values(&["logical size", &tenant_id, &timeline_id]) - .unwrap(); - let load_layer_map_histo = STORAGE_TIME - .get_metric_with_label_values(&["load layer map", &tenant_id, &timeline_id]) - .unwrap(); + let flush_time_histo = StorageTimeMetrics::new("layer flush", &tenant_id, &timeline_id); + let compact_time_histo = StorageTimeMetrics::new("compact", &tenant_id, &timeline_id); + let create_images_time_histo = + StorageTimeMetrics::new("create images", &tenant_id, &timeline_id); + let init_logical_size_histo = + StorageTimeMetrics::new("init logical size", &tenant_id, &timeline_id); + let logical_size_histo = StorageTimeMetrics::new("logical size", &tenant_id, &timeline_id); + let load_layer_map_histo = + StorageTimeMetrics::new("load layer map", &tenant_id, &timeline_id); + let garbage_collect_histo = StorageTimeMetrics::new("gc", &tenant_id, &timeline_id); let last_record_gauge = LAST_RECORD_LSN .get_metric_with_label_values(&[&tenant_id, &timeline_id]) .unwrap(); @@ -472,6 +546,7 @@ impl TimelineMetrics { create_images_time_histo, init_logical_size_histo, logical_size_histo, + garbage_collect_histo, load_layer_map_histo, last_record_gauge, wait_lsn_time_histo, @@ -497,7 +572,10 @@ impl Drop for TimelineMetrics { let _ = PERSISTENT_BYTES_WRITTEN.remove_label_values(&[tenant_id, timeline_id]); for op in STORAGE_TIME_OPERATIONS { - let _ = STORAGE_TIME.remove_label_values(&[op, tenant_id, timeline_id]); + let _ = + STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[op, tenant_id, timeline_id]); + let _ = + STORAGE_TIME_COUNT_PER_TIMELINE.remove_label_values(&[op, tenant_id, timeline_id]); } for op in STORAGE_IO_TIME_OPERATIONS { let _ = STORAGE_IO_TIME.remove_label_values(&[op, tenant_id, timeline_id]); @@ -514,7 +592,6 @@ impl Drop for TimelineMetrics { } pub fn remove_tenant_metrics(tenant_id: &TenantId) { - let _ = STORAGE_TIME.remove_label_values(&["gc", &tenant_id.to_string(), "-"]); let tid = tenant_id.to_string(); for state in TENANT_STATE_OPTIONS { let _ = TENANT_STATE_METRIC.remove_label_values(&[&tid, state]); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 9fb540cfae..2f45fe0dfc 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -51,8 +51,7 @@ use crate::config::PageServerConf; use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir; use crate::is_uninit_mark; -use crate::metrics::TENANT_STATE_METRIC; -use crate::metrics::{remove_tenant_metrics, STORAGE_TIME}; +use crate::metrics::{remove_tenant_metrics, TENANT_STATE_METRIC}; use crate::repository::GcResult; use crate::task_mgr; use crate::task_mgr::TaskKind; @@ -1243,17 +1242,11 @@ impl Tenant { "Cannot run GC iteration on inactive tenant" ); - let timeline_str = target_timeline_id - .map(|x| x.to_string()) - .unwrap_or_else(|| "-".to_string()); + let gc_result = self + .gc_iteration_internal(target_timeline_id, horizon, pitr, ctx) + .await; - { - let _timer = STORAGE_TIME - .with_label_values(&["gc", &self.tenant_id.to_string(), &timeline_str]) - .start_timer(); - self.gc_iteration_internal(target_timeline_id, horizon, pitr, ctx) - .await - } + gc_result } /// Perform one compaction iteration. diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 7b796a0749..0ca8a0e491 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -133,7 +133,6 @@ pub struct Timeline { ancestor_timeline: Option>, ancestor_lsn: Lsn, - // Metrics metrics: TimelineMetrics, /// Ensures layers aren't frozen by checkpointer between @@ -2894,6 +2893,8 @@ impl Timeline { /// obsolete. /// pub(super) async fn gc(&self) -> anyhow::Result { + let timer = self.metrics.garbage_collect_histo.start_timer(); + fail_point!("before-timeline-gc"); let _layer_removal_cs = self.layer_removal_cs.lock().await; @@ -2914,11 +2915,17 @@ impl Timeline { let new_gc_cutoff = Lsn::min(horizon_cutoff, pitr_cutoff); - self.gc_timeline(horizon_cutoff, pitr_cutoff, retain_lsns, new_gc_cutoff) + let res = self + .gc_timeline(horizon_cutoff, pitr_cutoff, retain_lsns, new_gc_cutoff) .instrument( info_span!("gc_timeline", timeline = %self.timeline_id, cutoff = %new_gc_cutoff), ) - .await + .await?; + + // only record successes + timer.stop_and_record(); + + Ok(res) } async fn gc_timeline( diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index b5e2876074..bdaaa95216 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -46,6 +46,12 @@ PAGESERVER_PER_TENANT_REMOTE_TIMELINE_CLIENT_METRICS: Tuple[str, ...] = ( "pageserver_remote_physical_size", ) +PAGESERVER_GLOBAL_METRICS: Tuple[str, ...] = ( + "pageserver_storage_operations_seconds_global_count", + "pageserver_storage_operations_seconds_global_sum", + "pageserver_storage_operations_seconds_global_bucket", +) + PAGESERVER_PER_TENANT_METRICS: Tuple[str, ...] = ( "pageserver_current_logical_size", "pageserver_resident_physical_size", @@ -61,9 +67,8 @@ PAGESERVER_PER_TENANT_METRICS: Tuple[str, ...] = ( "pageserver_smgr_query_seconds_bucket", "pageserver_smgr_query_seconds_count", "pageserver_smgr_query_seconds_sum", - "pageserver_storage_operations_seconds_bucket", - "pageserver_storage_operations_seconds_count", - "pageserver_storage_operations_seconds_sum", + "pageserver_storage_operations_seconds_count_total", + "pageserver_storage_operations_seconds_sum_total", "pageserver_wait_lsn_seconds_bucket", "pageserver_wait_lsn_seconds_count", "pageserver_wait_lsn_seconds_sum", diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index ce77c3940b..e56bb1b469 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -9,6 +9,7 @@ from typing import List import pytest from fixtures.log_helper import log from fixtures.metrics import ( + PAGESERVER_GLOBAL_METRICS, PAGESERVER_PER_TENANT_METRICS, PAGESERVER_PER_TENANT_REMOTE_TIMELINE_CLIENT_METRICS, parse_metrics, @@ -161,6 +162,14 @@ def test_metrics_normal_work(neon_env_builder: NeonEnvBuilder): f"process_start_time_seconds (UTC): {datetime.fromtimestamp(metrics.query_one('process_start_time_seconds').value)}" ) + # Test (a subset of) pageserver global metrics + for metric in PAGESERVER_GLOBAL_METRICS: + ps_samples = ps_metrics.query_all(metric, {}) + assert len(ps_samples) > 0 + for sample in ps_samples: + labels = ",".join([f'{key}="{value}"' for key, value in sample.labels.items()]) + log.info(f"{sample.name}{{{labels}}} {sample.value}") + @pytest.mark.parametrize( "remote_storage_kind", From ec0e6415786ade6e80abb9cd1d19448e28f1286a Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Mon, 30 Jan 2023 13:48:34 +0100 Subject: [PATCH 067/102] Create Release PR: review fixes --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74a2af8e96..49e04ee001 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: - name: Get current date id: date - run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + run: echo "date=(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - name: Create release branch run: git checkout -b release/${{ steps.date.outputs.date }} @@ -25,7 +25,7 @@ jobs: run: git push origin release/${{ steps.date.outputs.date }} - name: Create pull request into release - uses: thomaseizinger/create-pull-request@1 + uses: thomaseizinger/create-pull-request@e3972219c86a56550fb70708d96800d8e24ba862 # 1.3.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} head: release/${{ steps.date.outputs.date }} From 82cbcb36abe4f85a74bd921442d83b3101dc9ce2 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Mon, 30 Jan 2023 20:10:54 +0100 Subject: [PATCH 068/102] Extract neon deploy jobs into separate workflows (#3424) Extract deploy jobs from build_and_test.yml to deploy-dev and deploy-prod workflows. Add trigger to run this workflows after Neon is build and tested on main and release branches. This will allow us to redeploy/rollback/patch config without full rebuild. --- .github/workflows/build_and_test.yml | 450 +-------------------------- .github/workflows/deploy-dev.yml | 171 ++++++++++ .github/workflows/deploy-prod.yml | 266 ++++++++++++++++ 3 files changed, 449 insertions(+), 438 deletions(-) create mode 100644 .github/workflows/deploy-dev.yml create mode 100644 .github/workflows/deploy-prod.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7559b527e3..84781b417a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1,4 +1,4 @@ -name: Test and Deploy +name: Build and Test on: push: @@ -813,121 +813,6 @@ jobs: - name: Cleanup ECR folder run: rm -rf ~/.ecr - calculate-deploy-targets: - runs-on: [ self-hosted, gen3, small ] - if: | - github.ref_name == 'release' && - github.event_name != 'workflow_dispatch' - outputs: - matrix-include: ${{ steps.set-matrix.outputs.include }} - steps: - - id: set-matrix - run: | - if [[ "$GITHUB_REF_NAME" == "release" ]]; then - PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "storage_broker_ns": "neon-storage-broker", "storage_broker_config": "production.neon-storage-broker", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA", "console_api_key_secret": "NEON_PRODUCTION_API_KEY"}' - echo "include=[$PRODUCTION]" >> $GITHUB_OUTPUT - else - echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to 'release'" - exit 1 - fi - - deploy: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. - # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly - needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] - if: | - github.ref_name == 'release' && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}} - environment: - name: prod-old - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Redeploy - run: | - export DOCKER_TAG=${{needs.tag.outputs.build-tag}} - cd "$(pwd)/.github/ansible" - - if [[ "$GITHUB_REF_NAME" == "main" ]]; then - ./get_binaries.sh - elif [[ "$GITHUB_REF_NAME" == "release" ]]; then - RELEASE=true ./get_binaries.sh - else - echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" - exit 1 - fi - - eval $(ssh-agent) - echo "${{ secrets.TELEPORT_SSH_KEY }}" | tr -d '\n'| base64 --decode >ssh-key - echo "${{ secrets.TELEPORT_SSH_CERT }}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub - chmod 0600 ssh-key - ssh-add ssh-key - rm -f ssh-key ssh-key-cert.pub - ANSIBLE_CONFIG=./ansible.cfg ansible-galaxy collection install sivel.toiletwater - ANSIBLE_CONFIG=./ansible.cfg ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets[matrix.console_api_key_secret] }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} - rm -f neon_install.tar.gz .neon_current_version - - # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ansible/collections': Permission denied - - name: Cleanup ansible folder - run: rm -rf ~/.ansible - - deploy-new: - runs-on: [ self-hosted, gen3, small ] - container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - options: --user root --privileged - # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. - # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly - needs: [ push-docker-hub, tag, regress-tests ] - if: | - (github.ref_name == 'main') && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - target_region: [ eu-west-1, us-east-2 ] - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Redeploy - run: | - export DOCKER_TAG=${{needs.tag.outputs.build-tag}} - cd "$(pwd)/.github/ansible" - if [[ "$GITHUB_REF_NAME" == "main" ]]; then - ./get_binaries.sh - elif [[ "$GITHUB_REF_NAME" == "release" ]]; then - RELEASE=true ./get_binaries.sh - else - echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" - exit 1 - fi - ansible-galaxy collection install sivel.toiletwater - ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} - rm -f neon_install.tar.gz .neon_current_version - - - name: Cleanup ansible folder - run: rm -rf ~/.ansible - deploy-pr-test-new: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned @@ -964,348 +849,37 @@ jobs: - name: Cleanup ansible folder run: rm -rf ~/.ansible - deploy-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - # We need both storage **and** compute images for deploy, because control plane picks the compute version based on the storage version. - # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly + deploy: + runs-on: [ self-hosted, gen3, small ] + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest needs: [ push-docker-hub, tag, regress-tests ] - if: | - (github.ref_name == 'release') && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1 ] - environment: - name: prod-${{ matrix.target_region }} + if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch' steps: - name: Checkout uses: actions/checkout@v3 with: - submodules: true + submodules: false fetch-depth: 0 - - name: Redeploy + - name: Trigger deploy workflow + env: + GH_TOKEN: ${{ github.token }} run: | - export DOCKER_TAG=${{needs.tag.outputs.build-tag}} - cd "$(pwd)/.github/ansible" - if [[ "$GITHUB_REF_NAME" == "main" ]]; then - ./get_binaries.sh + gh workflow run deploy-dev.yml --ref ${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} elif [[ "$GITHUB_REF_NAME" == "release" ]]; then - RELEASE=true ./get_binaries.sh + gh workflow run deploy-prod.yml --ref ${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} else echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" exit 1 fi - ansible-galaxy collection install sivel.toiletwater - ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} - rm -f neon_install.tar.gz .neon_current_version - - deploy-proxy: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] - if: | - github.ref_name == 'release' && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}} - environment: - name: prod-old - env: - KUBECONFIG: .kubeconfig - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Store kubeconfig file - run: | - echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - - name: Add neon helm chart - run: helm repo add neondatabase https://neondatabase.github.io/helm-charts - - - name: Re-deploy proxy - run: | - DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-storage-broker: - name: deploy storage broker on old staging and old prod - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, calculate-deploy-targets, tag, regress-tests ] - if: | - github.ref_name == 'release' && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}} - environment: - name: prod-old - env: - KUBECONFIG: .kubeconfig - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Store kubeconfig file - run: | - echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - - name: Add neon helm chart - run: helm repo add neondatabase https://neondatabase.github.io/helm-charts - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace ${{ matrix.storage_broker_ns }} --create-namespace --install --atomic -f .github/helm-values/${{ matrix.storage_broker_config }}.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-proxy-new: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, tag, regress-tests ] - if: | - (github.ref_name == 'main') && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - deploy_link_proxy: true - deploy_legacy_scram_proxy: true - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Re-deploy scram proxy - run: | - DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy link proxy - if: matrix.deploy_link_proxy - run: | - DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy legacy scram proxy - if: matrix.deploy_legacy_scram_proxy - run: | - DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-storage-broker-dev-new: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, tag, regress-tests ] - if: | - (github.ref_name == 'main') && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: dev-us-east-2-beta - - target_region: eu-west-1 - target_cluster: dev-eu-west-1-zeta - environment: - name: dev-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - role-to-assume: arn:aws:iam::369495373322:role/github-runner - aws-region: eu-central-1 - role-skip-session-tagging: true - role-duration-seconds: 1800 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - - deploy-proxy-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, tag, regress-tests ] - if: | - (github.ref_name == 'release') && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - deploy_link_proxy: true - deploy_legacy_scram_proxy: false - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - deploy_link_proxy: false - deploy_legacy_scram_proxy: true - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - deploy_link_proxy: false - deploy_legacy_scram_proxy: false - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Re-deploy scram proxy - run: | - DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy link proxy - if: matrix.deploy_link_proxy - run: | - DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Re-deploy legacy scram proxy - if: matrix.deploy_legacy_scram_proxy - run: | - DOCKER_TAG=${{needs.tag.outputs.build-tag}} - helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - deploy-storage-broker-prod-new: - runs-on: prod - container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - # Compute image isn't strictly required for proxy deploy, but let's still wait for it to run all deploy jobs consistently. - needs: [ push-docker-hub, tag, regress-tests ] - if: | - (github.ref_name == 'release') && - github.event_name != 'workflow_dispatch' - defaults: - run: - shell: bash - strategy: - matrix: - include: - - target_region: us-east-2 - target_cluster: prod-us-east-2-delta - - target_region: us-west-2 - target_cluster: prod-us-west-2-eta - - target_region: eu-central-1 - target_cluster: prod-eu-central-1-gamma - - target_region: ap-southeast-1 - target_cluster: prod-ap-southeast-1-epsilon - environment: - name: prod-${{ matrix.target_region }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Configure environment - run: | - helm repo add neondatabase https://neondatabase.github.io/helm-charts - aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} - - - name: Deploy storage-broker - run: - helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ needs.tag.outputs.build-tag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s - promote-compatibility-data: runs-on: [ self-hosted, gen3, small ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init - needs: [ deploy, deploy-proxy ] + needs: [ push-docker-hub, tag, regress-tests ] if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch' steps: - name: Promote compatibility snapshot for the release diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml new file mode 100644 index 0000000000..6433cf5c6e --- /dev/null +++ b/.github/workflows/deploy-dev.yml @@ -0,0 +1,171 @@ +name: Neon Deploy dev + +on: + workflow_dispatch: + inputs: + dockerTag: + description: 'Docker tag to deploy' + required: true + type: string + deployStorage: + description: 'Should storage be deployed' + required: true + type: boolean + default: true + deployProxy: + description: 'Should proxy be deployed' + required: true + type: boolean + default: true + deployStorageBroker: + description: 'Should storage-broker be deployed' + required: true + type: boolean + default: true + +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} + +concurrency: + group: deploy-dev + cancel-in-progress: false + +jobs: + deploy-storage-new: + runs-on: [ self-hosted, gen3, small ] + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + options: --user root --privileged + if: inputs.deployStorage + defaults: + run: + shell: bash + strategy: + matrix: + target_region: [ eu-west-1, us-east-2 ] + environment: + name: dev-${{ matrix.target_region }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Redeploy + run: | + export DOCKER_TAG=${{ inputs.dockerTag }} + cd "$(pwd)/.github/ansible" + + ./get_binaries.sh + + ansible-galaxy collection install sivel.toiletwater + ansible-playbook deploy.yaml -i staging.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_STAGING_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} + rm -f neon_install.tar.gz .neon_current_version + + - name: Cleanup ansible folder + run: rm -rf ~/.ansible + + deploy-proxy-new: + runs-on: [ self-hosted, gen3, small ] + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + if: inputs.deployProxy + defaults: + run: + shell: bash + strategy: + matrix: + include: + - target_region: us-east-2 + target_cluster: dev-us-east-2-beta + deploy_link_proxy: true + deploy_legacy_scram_proxy: true + - target_region: eu-west-1 + target_cluster: dev-eu-west-1-zeta + deploy_link_proxy: false + deploy_legacy_scram_proxy: false + environment: + name: dev-${{ matrix.target_region }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::369495373322:role/github-runner + aws-region: eu-central-1 + role-skip-session-tagging: true + role-duration-seconds: 1800 + + - name: Configure environment + run: | + helm repo add neondatabase https://neondatabase.github.io/helm-charts + aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} + + - name: Re-deploy scram proxy + run: | + DOCKER_TAG=${{ inputs.dockerTag }} + helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + + - name: Re-deploy link proxy + if: matrix.deploy_link_proxy + run: | + DOCKER_TAG=${{ inputs.dockerTag }} + helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + + - name: Re-deploy legacy scram proxy + if: matrix.deploy_legacy_scram_proxy + run: | + DOCKER_TAG=${{ inputs.dockerTag }} + helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + + - name: Cleanup helm folder + run: rm -rf ~/.cache + + deploy-storage-broker-new: + runs-on: [ self-hosted, gen3, small ] + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + if: inputs.deployStorageBroker + defaults: + run: + shell: bash + strategy: + matrix: + include: + - target_region: us-east-2 + target_cluster: dev-us-east-2-beta + - target_region: eu-west-1 + target_cluster: dev-eu-west-1-zeta + environment: + name: dev-${{ matrix.target_region }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: arn:aws:iam::369495373322:role/github-runner + aws-region: eu-central-1 + role-skip-session-tagging: true + role-duration-seconds: 1800 + + - name: Configure environment + run: | + helm repo add neondatabase https://neondatabase.github.io/helm-charts + aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} + + - name: Deploy storage-broker + run: + helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s + + - name: Cleanup helm folder + run: rm -rf ~/.cache diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml new file mode 100644 index 0000000000..0b4ecdeaa4 --- /dev/null +++ b/.github/workflows/deploy-prod.yml @@ -0,0 +1,266 @@ +name: Neon Deploy prod + +on: + workflow_dispatch: + inputs: + dockerTag: + description: 'Docker tag to deploy' + required: true + type: string + deployStorage: + description: 'Should storage be deployed' + required: true + type: boolean + default: true + deployProxy: + description: 'Should proxy be deployed' + required: true + type: boolean + default: true + deployStorageBroker: + description: 'Should storage-broker be deployed' + required: true + type: boolean + default: true + +concurrency: + group: deploy-prod + cancel-in-progress: false + +jobs: + deploy-prod-new: + runs-on: prod + container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest + if: inputs.deployStorage + defaults: + run: + shell: bash + strategy: + matrix: + target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1 ] + environment: + name: prod-${{ matrix.target_region }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Redeploy + run: | + export DOCKER_TAG=${{ inputs.dockerTag }} + cd "$(pwd)/.github/ansible" + + ./get_binaries.sh + + ansible-galaxy collection install sivel.toiletwater + ansible-playbook deploy.yaml -i prod.${{ matrix.target_region }}.hosts.yaml -e @ssm_config -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} + rm -f neon_install.tar.gz .neon_current_version + + deploy-proxy-prod-new: + runs-on: prod + container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest + if: inputs.deployProxy + defaults: + run: + shell: bash + strategy: + matrix: + include: + - target_region: us-east-2 + target_cluster: prod-us-east-2-delta + deploy_link_proxy: true + deploy_legacy_scram_proxy: false + - target_region: us-west-2 + target_cluster: prod-us-west-2-eta + deploy_link_proxy: false + deploy_legacy_scram_proxy: true + - target_region: eu-central-1 + target_cluster: prod-eu-central-1-gamma + deploy_link_proxy: false + deploy_legacy_scram_proxy: false + - target_region: ap-southeast-1 + target_cluster: prod-ap-southeast-1-epsilon + deploy_link_proxy: false + deploy_legacy_scram_proxy: false + environment: + name: prod-${{ matrix.target_region }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Configure environment + run: | + helm repo add neondatabase https://neondatabase.github.io/helm-charts + aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} + + - name: Re-deploy scram proxy + run: | + DOCKER_TAG=${{ inputs.dockerTag }} + helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + + - name: Re-deploy link proxy + if: matrix.deploy_link_proxy + run: | + DOCKER_TAG=${{ inputs.dockerTag }} + helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + + - name: Re-deploy legacy scram proxy + if: matrix.deploy_legacy_scram_proxy + run: | + DOCKER_TAG=${{ inputs.dockerTag }} + helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + + deploy-storage-broker-prod-new: + runs-on: prod + container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest + if: inputs.deployStorageBroker + defaults: + run: + shell: bash + strategy: + matrix: + include: + - target_region: us-east-2 + target_cluster: prod-us-east-2-delta + - target_region: us-west-2 + target_cluster: prod-us-west-2-eta + - target_region: eu-central-1 + target_cluster: prod-eu-central-1-gamma + - target_region: ap-southeast-1 + target_cluster: prod-ap-southeast-1-epsilon + environment: + name: prod-${{ matrix.target_region }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Configure environment + run: | + helm repo add neondatabase https://neondatabase.github.io/helm-charts + aws --region ${{ matrix.target_region }} eks update-kubeconfig --name ${{ matrix.target_cluster }} + + - name: Deploy storage-broker + run: + helm upgrade neon-storage-broker-lb neondatabase/neon-storage-broker --namespace neon-storage-broker-lb --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s + + # Deploy to old account below + + deploy: + runs-on: [ self-hosted, gen3, small ] + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + if: inputs.deployStorage + defaults: + run: + shell: bash + environment: + name: prod-old + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Redeploy + run: | + export DOCKER_TAG=${{ inputs.dockerTag }} + cd "$(pwd)/.github/ansible" + + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + ./get_binaries.sh + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + RELEASE=true ./get_binaries.sh + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + + eval $(ssh-agent) + echo "${{ secrets.TELEPORT_SSH_KEY }}" | tr -d '\n'| base64 --decode >ssh-key + echo "${{ secrets.TELEPORT_SSH_CERT }}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub + chmod 0600 ssh-key + ssh-add ssh-key + rm -f ssh-key ssh-key-cert.pub + ANSIBLE_CONFIG=./ansible.cfg ansible-galaxy collection install sivel.toiletwater + ANSIBLE_CONFIG=./ansible.cfg ansible-playbook deploy.yaml -i production.hosts.yaml -e CONSOLE_API_TOKEN=${{ secrets.NEON_PRODUCTION_API_KEY }} -e SENTRY_URL_PAGESERVER=${{ secrets.SENTRY_URL_PAGESERVER }} -e SENTRY_URL_SAFEKEEPER=${{ secrets.SENTRY_URL_SAFEKEEPER }} + rm -f neon_install.tar.gz .neon_current_version + + # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ansible/collections': Permission denied + - name: Cleanup ansible folder + run: rm -rf ~/.ansible + + deploy-proxy: + runs-on: [ self-hosted, gen3, small ] + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + if: inputs.deployProxy + defaults: + run: + shell: bash + environment: + name: prod-old + env: + KUBECONFIG: .kubeconfig + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Store kubeconfig file + run: | + echo "${{ secrets.PRODUCTION_KUBECONFIG_DATA }}" | base64 --decode > ${KUBECONFIG} + chmod 0600 ${KUBECONFIG} + + - name: Add neon helm chart + run: helm repo add neondatabase https://neondatabase.github.io/helm-charts + + - name: Re-deploy proxy + run: | + DOCKER_TAG=${{ inputs.dockerTag }} + helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/production.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s + + - name: Cleanup helm folder + run: rm -rf ~/.cache + + deploy-storage-broker: + name: deploy storage broker on old staging and old prod + runs-on: [ self-hosted, gen3, small ] + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + if: inputs.deployStorageBroker + defaults: + run: + shell: bash + environment: + name: prod-old + env: + KUBECONFIG: .kubeconfig + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Store kubeconfig file + run: | + echo "${{ secrets.PRODUCTION_KUBECONFIG_DATA }}" | base64 --decode > ${KUBECONFIG} + chmod 0600 ${KUBECONFIG} + + - name: Add neon helm chart + run: helm repo add neondatabase https://neondatabase.github.io/helm-charts + + - name: Deploy storage-broker + run: + helm upgrade neon-storage-broker neondatabase/neon-storage-broker --namespace neon-storage-broker --create-namespace --install --atomic -f .github/helm-values/production.neon-storage-broker.yaml --set image.tag=${{ inputs.dockerTag }} --set settings.sentryUrl=${{ secrets.SENTRY_URL_BROKER }} --wait --timeout 5m0s + + - name: Cleanup helm folder + run: rm -rf ~/.cache From 5e08b35f5388773a875785f31b453efbc8d61a2d Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Mon, 30 Jan 2023 22:08:00 +0100 Subject: [PATCH 069/102] Fix new deploy workflow (#3492) Add 'branch' input to specify commit for deploy scripts/configs. Commit can't be passed to workflow as ref, and we need to pin configs to specific commit for main/release deploys Update deploy input descriptions to match GH interface --- .github/workflows/build_and_test.yml | 4 ++-- .github/workflows/deploy-dev.yml | 14 +++++++++++--- .github/workflows/deploy-prod.yml | 17 ++++++++++++++--- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 84781b417a..89e12360f9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -866,9 +866,9 @@ jobs: GH_TOKEN: ${{ github.token }} run: | if [[ "$GITHUB_REF_NAME" == "main" ]]; then - gh workflow run deploy-dev.yml --ref ${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} + gh workflow run deploy-dev.yml --ref main -f branch=${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} elif [[ "$GITHUB_REF_NAME" == "release" ]]; then - gh workflow run deploy-prod.yml --ref ${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} + gh workflow run deploy-prod.yml --ref release -f branch=${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} else echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" exit 1 diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 6433cf5c6e..409517bf63 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -7,18 +7,23 @@ on: description: 'Docker tag to deploy' required: true type: string + branch: + description: 'Branch or commit used for deploy scripts and configs' + required: true + type: string + default: 'main' deployStorage: - description: 'Should storage be deployed' + description: 'Deploy storage' required: true type: boolean default: true deployProxy: - description: 'Should proxy be deployed' + description: 'Deploy proxy' required: true type: boolean default: true deployStorageBroker: - description: 'Should storage-broker be deployed' + description: 'Deploy storage-broker' required: true type: boolean default: true @@ -52,6 +57,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Redeploy run: | @@ -93,6 +99,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1-node16 @@ -149,6 +156,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1-node16 diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 0b4ecdeaa4..e1954b5540 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -7,18 +7,23 @@ on: description: 'Docker tag to deploy' required: true type: string + branch: + description: 'Branch or commit used for deploy scripts and configs' + required: true + type: string + default: 'main' deployStorage: - description: 'Should storage be deployed' + description: 'Deploy storage' required: true type: boolean default: true deployProxy: - description: 'Should proxy be deployed' + description: 'Deploy proxy' required: true type: boolean default: true deployStorageBroker: - description: 'Should storage-broker be deployed' + description: 'Deploy storage-broker' required: true type: boolean default: true @@ -46,6 +51,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Redeploy run: | @@ -92,6 +98,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Configure environment run: | @@ -141,6 +148,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Configure environment run: | @@ -168,6 +176,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Redeploy run: | @@ -214,6 +223,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Store kubeconfig file run: | @@ -249,6 +259,7 @@ jobs: with: submodules: true fetch-depth: 0 + ref: ${{ inputs.branch }} - name: Store kubeconfig file run: | From 0806a46c0ccb8edf708dc5160081442b39b90d13 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Tue, 31 Jan 2023 13:36:25 +0100 Subject: [PATCH 070/102] Fix production deploy (#3498) `get_binaries.sh` no longer use `RELEASE` environmental variable, it just use `DOCKER_TAG` --- .github/workflows/deploy-prod.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index e1954b5540..5f1c7f2b89 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -183,14 +183,7 @@ jobs: export DOCKER_TAG=${{ inputs.dockerTag }} cd "$(pwd)/.github/ansible" - if [[ "$GITHUB_REF_NAME" == "main" ]]; then - ./get_binaries.sh - elif [[ "$GITHUB_REF_NAME" == "release" ]]; then - RELEASE=true ./get_binaries.sh - else - echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" - exit 1 - fi + ./get_binaries.sh eval $(ssh-agent) echo "${{ secrets.TELEPORT_SSH_KEY }}" | tr -d '\n'| base64 --decode >ssh-key From a7d8bfa6315091a3a88cc446bf92c491a37dfdbb Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Tue, 31 Jan 2023 11:49:22 +0100 Subject: [PATCH 071/102] Fix create release PR --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 49e04ee001..4bce9cdd1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,18 +16,18 @@ jobs: - name: Get current date id: date - run: echo "date=(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - name: Create release branch - run: git checkout -b release/${{ steps.date.outputs.date }} + run: git checkout -b releases/${{ steps.date.outputs.date }} - name: Push new branch - run: git push origin release/${{ steps.date.outputs.date }} + run: git push origin releases/${{ steps.date.outputs.date }} - name: Create pull request into release uses: thomaseizinger/create-pull-request@e3972219c86a56550fb70708d96800d8e24ba862 # 1.3.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - head: release/${{ steps.date.outputs.date }} + head: releases/${{ steps.date.outputs.date }} base: release title: Release ${{ steps.date.outputs.date }} From 895f929bcef6078683073d4088452df1691eeff3 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 31 Jan 2023 15:50:52 +0200 Subject: [PATCH 072/102] Add layer_map_analyzer tool (#3451) See #3348 --- pageserver/src/bin/layer_map_analyzer.rs | 230 ++++++++++++++++++ pageserver/src/tenant.rs | 2 +- pageserver/src/tenant/storage_layer.rs | 2 +- .../src/tenant/storage_layer/delta_layer.rs | 8 +- 4 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 pageserver/src/bin/layer_map_analyzer.rs diff --git a/pageserver/src/bin/layer_map_analyzer.rs b/pageserver/src/bin/layer_map_analyzer.rs new file mode 100644 index 0000000000..e740879458 --- /dev/null +++ b/pageserver/src/bin/layer_map_analyzer.rs @@ -0,0 +1,230 @@ +//! Tool for extracting content-dependent metadata about layers. Useful for scanning real project layer files and evaluating the effectiveness of different heuristics on them. +//! +//! Currently it only analyzes holes, which are regions within the layer range that the layer contains no updates for. In the future it might do more analysis (maybe key quantiles?) but it should never return sensitive data. + +use anyhow::Result; +use std::cmp::Ordering; +use std::collections::BinaryHeap; +use std::ops::Range; +use std::{env, fs, path::Path, path::PathBuf, str, str::FromStr}; + +use pageserver::page_cache::PAGE_SZ; +use pageserver::repository::{Key, KEY_SIZE}; +use pageserver::tenant::block_io::{BlockReader, FileBlockReader}; +use pageserver::tenant::disk_btree::{DiskBtreeReader, VisitDirection}; +use pageserver::tenant::storage_layer::delta_layer::{Summary, DELTA_KEY_SIZE}; +use pageserver::tenant::storage_layer::range_overlaps; +use pageserver::virtual_file::VirtualFile; + +use utils::{bin_ser::BeSer, lsn::Lsn}; + +const MIN_HOLE_LENGTH: i128 = (128 * 1024 * 1024 / PAGE_SZ) as i128; +const DEFAULT_MAX_HOLES: usize = 10; + +/// Wrapper for key range to provide reverse ordering by range length for BinaryHeap +#[derive(PartialEq, Eq)] +struct Hole(Range); + +impl Ord for Hole { + fn cmp(&self, other: &Self) -> Ordering { + let other_len = other.0.end.to_i128() - other.0.start.to_i128(); + let self_len = self.0.end.to_i128() - self.0.start.to_i128(); + other_len.cmp(&self_len) + } +} + +impl PartialOrd for Hole { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +struct LayerFile { + key_range: Range, + lsn_range: Range, + is_delta: bool, + holes: Vec, +} + +impl LayerFile { + fn skips(&self, key_range: &Range) -> bool { + if !range_overlaps(&self.key_range, key_range) { + return false; + } + let start = match self + .holes + .binary_search_by_key(&key_range.start, |hole| hole.0.start) + { + Ok(index) => index, + Err(index) => { + if index == 0 { + return false; + } + index - 1 + } + }; + self.holes[start].0.end >= key_range.end + } +} + +fn parse_filename(name: &str) -> Option { + let split: Vec<&str> = name.split("__").collect(); + if split.len() != 2 { + return None; + } + let keys: Vec<&str> = split[0].split('-').collect(); + let mut lsns: Vec<&str> = split[1].split('-').collect(); + let is_delta = if lsns.len() == 1 { + lsns.push(lsns[0]); + false + } else { + true + }; + + let key_range = Key::from_hex(keys[0]).unwrap()..Key::from_hex(keys[1]).unwrap(); + let lsn_range = Lsn::from_hex(lsns[0]).unwrap()..Lsn::from_hex(lsns[1]).unwrap(); + let holes = Vec::new(); + Some(LayerFile { + key_range, + lsn_range, + is_delta, + holes, + }) +} + +// Finds the max_holes largest holes, ignoring any that are smaller than MIN_HOLE_LENGTH" +fn get_holes(path: &Path, max_holes: usize) -> Result> { + let file = FileBlockReader::new(VirtualFile::open(path)?); + let summary_blk = file.read_blk(0)?; + let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; + let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( + actual_summary.index_start_blk, + actual_summary.index_root_blk, + file, + ); + // min-heap (reserve space for one more element added before eviction) + let mut heap: BinaryHeap = BinaryHeap::with_capacity(max_holes + 1); + let mut prev_key: Option = None; + tree_reader.visit( + &[0u8; DELTA_KEY_SIZE], + VisitDirection::Forwards, + |key, _value| { + let curr = Key::from_slice(&key[..KEY_SIZE]); + if let Some(prev) = prev_key { + if curr.to_i128() - prev.to_i128() >= MIN_HOLE_LENGTH { + heap.push(Hole(prev..curr)); + if heap.len() > max_holes { + heap.pop(); // remove smallest hole + } + } + } + prev_key = Some(curr.next()); + true + }, + )?; + let mut holes = heap.into_vec(); + holes.sort_by_key(|hole| hole.0.start); + Ok(holes) +} + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + if args.len() < 2 { + println!("Usage: layer_map_analyzer PAGESERVER_DATA_DIR [MAX_HOLES]"); + return Ok(()); + } + let storage_path = PathBuf::from_str(&args[1])?; + let max_holes = if args.len() > 2 { + args[2].parse::().unwrap() + } else { + DEFAULT_MAX_HOLES + }; + + // Initialize virtual_file (file desriptor cache) and page cache which are needed to access layer persistent B-Tree. + pageserver::virtual_file::init(10); + pageserver::page_cache::init(100); + + let mut total_delta_layers = 0usize; + let mut total_image_layers = 0usize; + let mut total_excess_layers = 0usize; + for tenant in fs::read_dir(storage_path.join("tenants"))? { + let tenant = tenant?; + if !tenant.file_type()?.is_dir() { + continue; + } + for timeline in fs::read_dir(tenant.path().join("timelines"))? { + let timeline = timeline?; + if !timeline.file_type()?.is_dir() { + continue; + } + // Collect sorted vec of layers and count deltas + let mut layers = Vec::new(); + let mut n_deltas = 0usize; + + for layer in fs::read_dir(timeline.path())? { + let layer = layer?; + if let Some(mut layer_file) = + parse_filename(&layer.file_name().into_string().unwrap()) + { + if layer_file.is_delta { + layer_file.holes = get_holes(&layer.path(), max_holes)?; + n_deltas += 1; + } + layers.push(layer_file); + } + } + layers.sort_by_key(|layer| layer.lsn_range.end); + + // Count the number of holes and number of excess layers. + // Excess layer is image layer generated when holes in delta layers are not considered. + let mut n_excess_layers = 0usize; + let mut n_holes = 0usize; + + for i in 0..layers.len() { + if !layers[i].is_delta { + let mut n_deltas_since_last_image = 0usize; + let mut n_skipped = 0usize; + let img_key_range = &layers[i].key_range; + for j in (0..i).rev() { + if range_overlaps(img_key_range, &layers[j].key_range) { + if layers[j].is_delta { + n_deltas_since_last_image += 1; + if layers[j].skips(img_key_range) { + n_skipped += 1; + } + } else { + // Image layer is always dense, despite to the fact that it doesn't contain all possible + // key values in the specified range: there are may be no keys in the storage belonging + // to the image layer range but not present in the image layer. + break; + } + } + } + if n_deltas_since_last_image >= 3 && n_deltas_since_last_image - n_skipped < 3 { + // It is just approximation: it doesn't take in account all image coverage. + // Moreover the new layer map doesn't count total deltas, but the max stack of overlapping deltas. + n_excess_layers += 1; + } + n_holes += n_skipped; + } + } + println!( + "Tenant {} timeline {} delta layers {} image layers {} excess layers {} holes {}", + tenant.file_name().into_string().unwrap(), + timeline.file_name().into_string().unwrap(), + n_deltas, + layers.len() - n_deltas, + n_excess_layers, + n_holes + ); + total_delta_layers += n_deltas; + total_image_layers += layers.len() - n_deltas; + total_excess_layers += n_excess_layers; + } + } + println!( + "Total delta layers {} image layers {} excess layers {}", + total_delta_layers, total_image_layers, total_excess_layers + ); + Ok(()) +} diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 2f45fe0dfc..1af5eb07a2 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -77,7 +77,7 @@ use utils::{ mod blob_io; pub mod block_io; -mod disk_btree; +pub mod disk_btree; pub(crate) mod ephemeral_file; pub mod layer_map; diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 2149fc7eb7..cb7380be2d 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -1,6 +1,6 @@ //! Common traits and structs for layers -mod delta_layer; +pub mod delta_layer; mod filename; mod image_layer; mod inmemory_layer; diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 302ba2dc78..e9bcb55f9b 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -62,7 +62,7 @@ use super::{DeltaFileName, Layer, LayerFileName, LayerIter, LayerKeyIter, PathOr /// the 'index' starts at the block indicated by 'index_start_blk' /// #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -struct Summary { +pub struct Summary { /// Magic value to identify this as a neon delta file. Always DELTA_FILE_MAGIC. magic: u16, format_version: u16, @@ -73,9 +73,9 @@ struct Summary { lsn_range: Range, /// Block number where the 'index' part of the file begins. - index_start_blk: u32, + pub index_start_blk: u32, /// Block within the 'index', where the B-tree root page is stored - index_root_blk: u32, + pub index_root_blk: u32, } impl From<&DeltaLayer> for Summary { @@ -125,7 +125,7 @@ impl BlobRef { } } -const DELTA_KEY_SIZE: usize = KEY_SIZE + 8; +pub const DELTA_KEY_SIZE: usize = KEY_SIZE + 8; struct DeltaKey([u8; DELTA_KEY_SIZE]); /// From a058bc6de86cee2980149d807d2d6dc8b552e9c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Jan 2023 17:30:45 +0200 Subject: [PATCH 073/102] Bump aiohttp from 3.7.0 to 3.7.4 (#3445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.7.0 to 3.7.4.
Release notes

Sourced from aiohttp's releases.

aiohttp 3.7.3 release

Features

  • Use Brotli instead of brotlipy [#3803](https://github.com/aio-libs/aiohttp/issues/3803) <https://github.com/aio-libs/aiohttp/issues/3803>_
  • Made exceptions pickleable. Also changed the repr of some exceptions. [#4077](https://github.com/aio-libs/aiohttp/issues/4077) <https://github.com/aio-libs/aiohttp/issues/4077>_

Bugfixes

  • Raise a ClientResponseError instead of an AssertionError for a blank HTTP Reason Phrase. [#3532](https://github.com/aio-libs/aiohttp/issues/3532) <https://github.com/aio-libs/aiohttp/issues/3532>_
  • Fix web_middlewares.normalize_path_middleware behavior for patch without slash. [#3669](https://github.com/aio-libs/aiohttp/issues/3669) <https://github.com/aio-libs/aiohttp/issues/3669>_
  • Fix overshadowing of overlapped sub-applications prefixes. [#3701](https://github.com/aio-libs/aiohttp/issues/3701) <https://github.com/aio-libs/aiohttp/issues/3701>_
  • Make BaseConnector.close() a coroutine and wait until the client closes all connections. Drop deprecated "with Connector():" syntax. [#3736](https://github.com/aio-libs/aiohttp/issues/3736) <https://github.com/aio-libs/aiohttp/issues/3736>_
  • Reset the sock_read timeout each time data is received for a aiohttp.client response. [#3808](https://github.com/aio-libs/aiohttp/issues/3808) <https://github.com/aio-libs/aiohttp/issues/3808>_
  • Fixed type annotation for add_view method of UrlDispatcher to accept any subclass of View [#3880](https://github.com/aio-libs/aiohttp/issues/3880) <https://github.com/aio-libs/aiohttp/issues/3880>_
  • Fixed querying the address families from DNS that the current host supports. [#5156](https://github.com/aio-libs/aiohttp/issues/5156) <https://github.com/aio-libs/aiohttp/issues/5156>_
  • Change return type of MultipartReader.aiter() and BodyPartReader.aiter() to AsyncIterator. [#5163](https://github.com/aio-libs/aiohttp/issues/5163) <https://github.com/aio-libs/aiohttp/issues/5163>_
  • Provide x86 Windows wheels. [#5230](https://github.com/aio-libs/aiohttp/issues/5230) <https://github.com/aio-libs/aiohttp/issues/5230>_

Improved Documentation

  • Add documentation for aiohttp.web.FileResponse. [#3958](https://github.com/aio-libs/aiohttp/issues/3958) <https://github.com/aio-libs/aiohttp/issues/3958>_
  • Removed deprecation warning in tracing example docs [#3964](https://github.com/aio-libs/aiohttp/issues/3964) <https://github.com/aio-libs/aiohttp/issues/3964>_
  • Fixed wrong "Usage" docstring of aiohttp.client.request. [#4603](https://github.com/aio-libs/aiohttp/issues/4603) <https://github.com/aio-libs/aiohttp/issues/4603>_
  • Add aiohttp-pydantic to third party libraries [#5228](https://github.com/aio-libs/aiohttp/issues/5228) <https://github.com/aio-libs/aiohttp/issues/5228>_

Misc

... (truncated)

Changelog

Sourced from aiohttp's changelog.

3.7.4 (2021-02-25)

Bugfixes

  • (SECURITY BUG) Started preventing open redirects in the aiohttp.web.normalize_path_middleware middleware. For more details, see https://github.com/aio-libs/aiohttp/security/advisories/GHSA-v6wp-4m6f-gcjg.

    Thanks to Beast Glatisant <https://github.com/g147>__ for finding the first instance of this issue and Jelmer Vernooij <https://jelmer.uk/>__ for reporting and tracking it down in aiohttp. [#5497](https://github.com/aio-libs/aiohttp/issues/5497) <https://github.com/aio-libs/aiohttp/issues/5497>_

  • Fix interpretation difference of the pure-Python and the Cython-based HTTP parsers construct a yarl.URL object for HTTP request-target.

    Before this fix, the Python parser would turn the URI's absolute-path for //some-path into / while the Cython code preserved it as //some-path. Now, both do the latter. [#5498](https://github.com/aio-libs/aiohttp/issues/5498) <https://github.com/aio-libs/aiohttp/issues/5498>_

3.7.3 (2020-11-18)

Features

  • Use Brotli instead of brotlipy [#3803](https://github.com/aio-libs/aiohttp/issues/3803) <https://github.com/aio-libs/aiohttp/issues/3803>_
  • Made exceptions pickleable. Also changed the repr of some exceptions. [#4077](https://github.com/aio-libs/aiohttp/issues/4077) <https://github.com/aio-libs/aiohttp/issues/4077>_

Bugfixes

  • Raise a ClientResponseError instead of an AssertionError for a blank HTTP Reason Phrase. [#3532](https://github.com/aio-libs/aiohttp/issues/3532) <https://github.com/aio-libs/aiohttp/issues/3532>_
  • Fix web_middlewares.normalize_path_middleware behavior for patch without slash. [#3669](https://github.com/aio-libs/aiohttp/issues/3669) <https://github.com/aio-libs/aiohttp/issues/3669>_
  • Fix overshadowing of overlapped sub-applications prefixes. [#3701](https://github.com/aio-libs/aiohttp/issues/3701) <https://github.com/aio-libs/aiohttp/issues/3701>_

... (truncated)

Commits
  • 0a26acc Bump aiohttp to v3.7.4 for a security release
  • 021c416 Merge branch 'ghsa-v6wp-4m6f-gcjg' into master
  • 4ed7c25 Bump chardet from 3.0.4 to 4.0.0 (#5333)
  • b61f0fd Fix how pure-Python HTTP parser interprets //
  • 5c1efbc Bump pre-commit from 2.9.2 to 2.9.3 (#5322)
  • 0075075 Bump pygments from 2.7.2 to 2.7.3 (#5318)
  • 5085173 Bump multidict from 5.0.2 to 5.1.0 (#5308)
  • 5d1a75e Bump pre-commit from 2.9.0 to 2.9.2 (#5290)
  • 6724d0e Bump pre-commit from 2.8.2 to 2.9.0 (#5273)
  • c688451 Removed duplicate timeout parameter in ClientSession reference docs. (#5262) ...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=aiohttp&package-manager=pip&previous-version=3.7.0&new-version=3.7.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) - `@dependabot use these labels` will set the current labels as the default for future PRs for this repo and language - `@dependabot use these reviewers` will set the current reviewers as the default for future PRs for this repo and language - `@dependabot use these assignees` will set the current assignees as the default for future PRs for this repo and language - `@dependabot use this milestone` will set the current milestone as the default for future PRs for this repo and language You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/neondatabase/neon/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Vadim Kharitonov --- poetry.lock | 76 +++++++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/poetry.lock b/poetry.lock index fc37124184..6d7f39ecbd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "aiohttp" -version = "3.7.0" +version = "3.7.4" description = "Async http client/server framework (asyncio)" category = "main" optional = false @@ -11,6 +11,7 @@ async-timeout = ">=3.0,<4.0" attrs = ">=17.3.0" chardet = ">=2.0,<4.0" multidict = ">=4.5,<7.0" +typing-extensions = ">=3.6.5" yarl = ">=1.0,<2.0" [package.extras] @@ -1641,43 +1642,47 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "0f7289ef9439d1d7cd36b07efb53741b773669b0f860189c800270b7def0c241" +content-hash = "7563a38912963d8cf20c99acb06fe55623e65b799c4b88d37dc672e5384c96a3" [metadata.files] aiohttp = [ - {file = "aiohttp-3.7.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:72fe89f7e14939e896d984c4b592580f8cdfa7497feb1c0c24639a9c60be3eb9"}, - {file = "aiohttp-3.7.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:fdf778d4c4bf976e69a37213fe8083613d0851976ddcf485bd7c0650a43d3852"}, - {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fee7b5e68939ffc09f9b29f167ed49c8b50de3eee0a1d8108b439ddd9963af46"}, - {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:dd64634713be409202058f2ea267dfbcdd74b387b8793425f21ef0266d45d0e9"}, - {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:713dd7fd70ddda9dc8d014c49dd0e55b58afe4e0cddb8722c7501f53edf30c3f"}, - {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:d31c43f7c4948ce01957f9a1ceee0784e067778477557ebccdf805398331c1a1"}, - {file = "aiohttp-3.7.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:5e26d6003eb6df304608d9fd9c9437065a8532d869a3ffcbd8113a3d710f8239"}, - {file = "aiohttp-3.7.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bf08462cddd10ddd8ffe5cb5c1638bfa051290909ebedb31c06e46578b9b7529"}, - {file = "aiohttp-3.7.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:07bacf6721db51a4c6160ed3031a2a97910647969dafd7c653f600f3b542f463"}, - {file = "aiohttp-3.7.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:245b58e30bc889d18b783db2f09ef1d814f466e15c84325410827451297003a0"}, - {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b392e5c3e122586c49cd8b9426f577bf4d51958933b839d158d28b69515af74e"}, - {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:5b5c320621a171aa85f96909af28fbb5286bd6842066db3062b083ba92261256"}, - {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:97d2341d1360dbe2c5b1d94922f7d68f9ce2ded1daab88b9bdeb49ce419cdc1b"}, - {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:beda23f292716887532661dc19abb9db2302ccfbd671a080cd8f4be7463d0841"}, - {file = "aiohttp-3.7.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:cbcaae9a6f14f762348d19b2dce8162772c0b0a1739314e18492a308a22caf96"}, - {file = "aiohttp-3.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7a49ef7b691babc83db126db874fbf26ba2f781899b91399f9ff8b235f059245"}, - {file = "aiohttp-3.7.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f56892f57310415cf6a179eec3ea6c7a82a9d37fbc00894943ea3154011a6d2a"}, - {file = "aiohttp-3.7.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:df1274b7620c32d3b15bfb0a8fb3165dd6cdc9c39f4db74d162f051c80826542"}, - {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a04ba359dc5f2e21b96bfc90c4a7665441441ba61b52e992b7799493889a3419"}, - {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:f548d7976d168f0f45ac5909ca5f606ae3f6f7aa1725b22504004a053b29a7d0"}, - {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:deef02e2a9f5095463098c7c22d5566f20a6e4e14fc0996c0c2efc74d461b680"}, - {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:fe44c96bc380588d36729392b602470d88a7c18e646e95dd4348cafe3900d91d"}, - {file = "aiohttp-3.7.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:9210532e6e95b40d22a33415bb84423eef3f633b2d2339b97f3b26438eebc466"}, - {file = "aiohttp-3.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:a586e476a251483d222c73dfb2f27df90bc4ea1b8c7da9396236510e0d4046c8"}, - {file = "aiohttp-3.7.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:900012c5f12ff72b1453229afe288ddc9135176df8b3b3cc5b8f6cfde912aaa4"}, - {file = "aiohttp-3.7.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:064d5f0738bcbab3e0c0ecf85c93b5ee1e07e124f994eaa03bf73687f3ecd9da"}, - {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0a2edf27865e66a33f64fa793cd14d0aae8127ce20a858539e97c25b600556dc"}, - {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:eaa8ae734639d5a0a3b5e33a154b8bfef384cdc090706f95c387cae8b21af764"}, - {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:a8a42f05491d9c04a77806875a68f84fea9af7a59d47b7897cb166632f74606c"}, - {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:b19ded3f6957693b97ba8372aacb5b0021639bbd5e77b1e960796bcef5431969"}, - {file = "aiohttp-3.7.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:cefbd7ce7d1f1db43749a077e4970e29e2b631f367c9eff3862c3c886b4218dd"}, - {file = "aiohttp-3.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:7d64f7dfd4e326d9b0d11b07fcd5ebf78844ba3c8f7699f38b50b0e0db0ae68f"}, - {file = "aiohttp-3.7.0.tar.gz", hash = "sha256:176f1d2b2bc07044f4ed583216578a72a2bd35dffdeb92e0517d0aaa29d29549"}, + {file = "aiohttp-3.7.4-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:6c8200abc9dc5f27203986100579fc19ccad7a832c07d2bc151ce4ff17190076"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:dd7936f2a6daa861143e376b3a1fb56e9b802f4980923594edd9ca5670974895"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:bc3d14bf71a3fb94e5acf5bbf67331ab335467129af6416a437bd6024e4f743d"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:8ec1a38074f68d66ccb467ed9a673a726bb397142c273f90d4ba954666e87d54"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:b84ad94868e1e6a5e30d30ec419956042815dfaea1b1df1cef623e4564c374d9"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:d5d102e945ecca93bcd9801a7bb2fa703e37ad188a2f81b1e65e4abe4b51b00c"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c2a80fd9a8d7e41b4e38ea9fe149deed0d6aaede255c497e66b8213274d6d61b"}, + {file = "aiohttp-3.7.4-cp36-cp36m-win32.whl", hash = "sha256:481d4b96969fbfdcc3ff35eea5305d8565a8300410d3d269ccac69e7256b1329"}, + {file = "aiohttp-3.7.4-cp36-cp36m-win_amd64.whl", hash = "sha256:16d0683ef8a6d803207f02b899c928223eb219111bd52420ef3d7a8aa76227b6"}, + {file = "aiohttp-3.7.4-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:eab51036cac2da8a50d7ff0ea30be47750547c9aa1aa2cf1a1b710a1827e7dbe"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:feb24ff1226beeb056e247cf2e24bba5232519efb5645121c4aea5b6ad74c1f2"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:119feb2bd551e58d83d1b38bfa4cb921af8ddedec9fad7183132db334c3133e0"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:6ca56bdfaf825f4439e9e3673775e1032d8b6ea63b8953d3812c71bd6a8b81de"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:5563ad7fde451b1986d42b9bb9140e2599ecf4f8e42241f6da0d3d624b776f40"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:62bc216eafac3204877241569209d9ba6226185aa6d561c19159f2e1cbb6abfb"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:f4496d8d04da2e98cc9133e238ccebf6a13ef39a93da2e87146c8c8ac9768242"}, + {file = "aiohttp-3.7.4-cp37-cp37m-win32.whl", hash = "sha256:2ffea7904e70350da429568113ae422c88d2234ae776519549513c8f217f58a9"}, + {file = "aiohttp-3.7.4-cp37-cp37m-win_amd64.whl", hash = "sha256:5e91e927003d1ed9283dee9abcb989334fc8e72cf89ebe94dc3e07e3ff0b11e9"}, + {file = "aiohttp-3.7.4-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:4c1bdbfdd231a20eee3e56bd0ac1cd88c4ff41b64ab679ed65b75c9c74b6c5c2"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:71680321a8a7176a58dfbc230789790639db78dad61a6e120b39f314f43f1907"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7dbd087ff2f4046b9b37ba28ed73f15fd0bc9f4fdc8ef6781913da7f808d9536"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:dee68ec462ff10c1d836c0ea2642116aba6151c6880b688e56b4c0246770f297"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:99c5a5bf7135607959441b7d720d96c8e5c46a1f96e9d6d4c9498be8d5f24212"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:5dde6d24bacac480be03f4f864e9a67faac5032e28841b00533cd168ab39cad9"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:418597633b5cd9639e514b1d748f358832c08cd5d9ef0870026535bd5eaefdd0"}, + {file = "aiohttp-3.7.4-cp38-cp38-win32.whl", hash = "sha256:e76e78863a4eaec3aee5722d85d04dcbd9844bc6cd3bfa6aa880ff46ad16bfcb"}, + {file = "aiohttp-3.7.4-cp38-cp38-win_amd64.whl", hash = "sha256:950b7ef08b2afdab2488ee2edaff92a03ca500a48f1e1aaa5900e73d6cf992bc"}, + {file = "aiohttp-3.7.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2eb3efe243e0f4ecbb654b08444ae6ffab37ac0ef8f69d3a2ffb958905379daf"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:822bd4fd21abaa7b28d65fc9871ecabaddc42767884a626317ef5b75c20e8a2d"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:58c62152c4c8731a3152e7e650b29ace18304d086cb5552d317a54ff2749d32a"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:7c7820099e8b3171e54e7eedc33e9450afe7cd08172632d32128bd527f8cb77d"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:5b50e0b9460100fe05d7472264d1975f21ac007b35dcd6fd50279b72925a27f4"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:c44d3c82a933c6cbc21039326767e778eface44fca55c65719921c4b9661a3f7"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:cc31e906be1cc121ee201adbdf844522ea3349600dd0a40366611ca18cd40e81"}, + {file = "aiohttp-3.7.4-cp39-cp39-win32.whl", hash = "sha256:fbd3b5e18d34683decc00d9a360179ac1e7a320a5fee10ab8053ffd6deab76e0"}, + {file = "aiohttp-3.7.4-cp39-cp39-win_amd64.whl", hash = "sha256:40bd1b101b71a18a528ffce812cc14ff77d4a2a1272dfb8b11b200967489ef3e"}, + {file = "aiohttp-3.7.4.tar.gz", hash = "sha256:5d84ecc73141d0a0d61ece0742bb7ff5751b0657dab8405f899d3ceb104cc7de"}, ] aiopg = [ {file = "aiopg-1.3.4-py3-none-any.whl", hash = "sha256:b5b74a124831aad71608c3c203479db90bac4a7eb3f8982bc48c3d3e6f1e57bf"}, @@ -1993,6 +1998,7 @@ jsonschema = [ {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, ] junit-xml = [ + {file = "junit-xml-1.9.tar.gz", hash = "sha256:de16a051990d4e25a3982b2dd9e89d671067548718866416faec14d9de56db9f"}, {file = "junit_xml-1.9-py2.py3-none-any.whl", hash = "sha256:ec5ca1a55aefdd76d28fcc0b135251d156c7106fa979686a4b48d62b761b4732"}, ] markupsafe = [ diff --git a/pyproject.toml b/pyproject.toml index a817e9dda5..d3d3948b9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ psutil = "^5.9.4" types-psutil = "^5.9.5.4" types-toml = "^0.10.8" pytest-httpserver = "^1.0.6" -aiohttp = "3.7" +aiohttp = "3.7.4" [tool.poetry.dev-dependencies] flake8 = "^5.0.4" From 847fc566fd78471e2d3968fd3c43e4ebd2e204b5 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Tue, 31 Jan 2023 16:36:05 +0100 Subject: [PATCH 074/102] Use the same runners/container for old prod deployments as for new prod --- .github/workflows/deploy-prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 5f1c7f2b89..fbd6bdfd93 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -162,8 +162,8 @@ jobs: # Deploy to old account below deploy: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned + runs-on: prod + container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest if: inputs.deployStorage defaults: run: From 2309dd564656fb691390de6e25bd87db1e29ea70 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Wed, 25 Jan 2023 15:38:53 +0100 Subject: [PATCH 075/102] Install postgis_sfcgal --- Dockerfile.compute-node | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 936f368833..7c40470ae2 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -44,20 +44,30 @@ RUN cd postgres && \ FROM build-deps AS postgis-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ - apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc + apt install -y cmake gdal-bin libboost-dev libboost-thread-dev libboost-filesystem-dev \ + libboost-system-dev libboost-iostreams-dev libboost-program-options-dev libboost-timer-dev \ + libcgal-dev libgdal-dev libgmp-dev libmpfr-dev libopenscenegraph-dev libprotobuf-c-dev \ + protobuf-c-compiler xsltproc + +RUN wget https://gitlab.com/Oslandia/SFCGAL/-/archive/v1.3.10/SFCGAL-v1.3.10.tar.gz && \ + tar zxvf SFCGAL-v1.3.10.tar.gz && \ + cd SFCGAL-v1.3.10 && cmake . && make -j $(getconf _NPROCESSORS_ONLN) && \ + DESTDIR=/sfcgal make install -j $(getconf _NPROCESSORS_ONLN) && \ + make clean && cp -R /sfcgal/* / RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \ tar xvzf postgis-3.3.1.tar.gz && \ cd postgis-3.3.1 && \ ./autogen.sh && \ export PATH="/usr/local/pgsql/bin:$PATH" && \ - ./configure && \ + ./configure --with-sfcgal=/usr/local/bin/sfcgal-config && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ cd extensions/postgis && \ make clean && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_raster.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_sfcgal.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer.control && \ @@ -162,6 +172,7 @@ RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz & ######################################################################################### FROM build-deps AS neon-pg-ext-build COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=postgis-build /sfcgal/* / COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=h3-pg-build /h3/usr / @@ -224,7 +235,7 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb # Install: # libreadline8 for psql # libossp-uuid16 for extension ossp-uuid -# libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS +# libgeos, libgdal, libsfcgal1, libproj and libprotobuf-c1 for PostGIS RUN apt update && \ apt install --no-install-recommends -y \ libreadline8 \ @@ -233,6 +244,7 @@ RUN apt update && \ libgdal28 \ libproj19 \ libprotobuf-c1 \ + libsfcgal1 \ gdb && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From 9bb6a6c77c19569b24ee9a456b6dfbeee4338cb7 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Wed, 1 Feb 2023 14:07:23 +0200 Subject: [PATCH 076/102] pysync: override PYTHON_KEYRING_BACKEND (#3480) This bothers me everytime I have to call `pysync`. --- scripts/pysync | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/pysync b/scripts/pysync index 12fa08beca..29c51b8e20 100755 --- a/scripts/pysync +++ b/scripts/pysync @@ -4,6 +4,10 @@ # It is intended to be a primary endpoint for all the people who want to # just setup test environment without going into details of python package management +# on ubuntu 22.04 based `six` fails to install without this, because something imports keyring. +# null keyring is fine, which means no secrets should be accessed. +export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring + poetry config --list if [ -z "${CI}" ]; then From 590695e845a1d33e36df81c37244d281b1c03268 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 1 Feb 2023 14:11:12 +0100 Subject: [PATCH 077/102] improve query param parsing - add parse_query_param() - use Cow<> where possible - move param parsing code to utils::http::request This was originally PR https://github.com/neondatabase/neon/pull/3502 which targeted a different branch. closes #3510 --- Cargo.lock | 1 + libs/utils/Cargo.toml | 1 + libs/utils/src/http/request.rs | 47 +++++++++++- pageserver/src/http/routes.rs | 76 +++++++------------- scripts/export_import_between_pageservers.py | 2 +- test_runner/fixtures/neon_fixtures.py | 8 +-- 6 files changed, 78 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2985a654f3..794ec25bf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4337,6 +4337,7 @@ dependencies = [ "tokio-rustls", "tracing", "tracing-subscriber", + "url", "workspace_hack", ] diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 1f6c96bdbe..1ce3155083 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -37,6 +37,7 @@ metrics.workspace = true pq_proto.workspace = true workspace_hack.workspace = true +url.workspace = true [dev-dependencies] byteorder.workspace = true diff --git a/libs/utils/src/http/request.rs b/libs/utils/src/http/request.rs index 7b96ccd584..766bbfc9df 100644 --- a/libs/utils/src/http/request.rs +++ b/libs/utils/src/http/request.rs @@ -1,4 +1,5 @@ -use std::str::FromStr; +use core::fmt; +use std::{borrow::Cow, str::FromStr}; use super::error::ApiError; use anyhow::anyhow; @@ -29,6 +30,50 @@ pub fn parse_request_param( } } +fn get_query_param<'a>( + request: &'a Request, + param_name: &str, +) -> Result>, ApiError> { + let query = match request.uri().query() { + Some(q) => q, + None => return Ok(None), + }; + let mut values = url::form_urlencoded::parse(query.as_bytes()) + .filter_map(|(k, v)| if k == param_name { Some(v) } else { None }) + // we call .next() twice below. If it's None the first time, .fuse() ensures it's None afterwards + .fuse(); + + let value1 = values.next(); + if values.next().is_some() { + return Err(ApiError::BadRequest(anyhow!( + "param {param_name} specified more than once" + ))); + } + Ok(value1) +} + +pub fn must_get_query_param<'a>( + request: &'a Request, + param_name: &str, +) -> Result, ApiError> { + get_query_param(request, param_name)?.ok_or_else(|| { + ApiError::BadRequest(anyhow!("no {param_name} specified in query parameters")) + }) +} + +pub fn parse_query_param>( + request: &Request, + param_name: &str, +) -> Result, ApiError> { + get_query_param(request, param_name)? + .map(|v| { + v.parse().map_err(|e| { + ApiError::BadRequest(anyhow!("cannot parse query param {param_name}: {e}")) + }) + }) + .transpose() +} + pub async fn ensure_no_body(request: &mut Request) -> Result<(), ApiError> { match request.body_mut().data().await { Some(_) => Err(ApiError::BadRequest(anyhow!("Unexpected request body"))), diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index a7802f3cbe..da31bd9bc0 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -7,6 +7,7 @@ use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest; use remote_storage::GenericRemoteStorage; use tokio_util::sync::CancellationToken; use tracing::*; +use utils::http::request::{must_get_query_param, parse_query_param}; use super::models::{ StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo, @@ -235,8 +236,8 @@ async fn timeline_create_handler(mut request: Request) -> Result) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - let include_non_incremental_logical_size = - query_param_present(&request, "include-non-incremental-logical-size"); + let include_non_incremental_logical_size: Option = + parse_query_param(&request, "include-non-incremental-logical-size")?; check_permission(&request, Some(tenant_id))?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); @@ -249,13 +250,14 @@ async fn timeline_list_handler(request: Request) -> Result, let mut response_data = Vec::with_capacity(timelines.len()); for timeline in timelines { - let timeline_info = - build_timeline_info(&timeline, include_non_incremental_logical_size, &ctx) - .await - .context( - "Failed to convert tenant timeline {timeline_id} into the local one: {e:?}", - ) - .map_err(ApiError::InternalServerError)?; + let timeline_info = build_timeline_info( + &timeline, + include_non_incremental_logical_size.unwrap_or(false), + &ctx, + ) + .await + .context("Failed to convert tenant timeline {timeline_id} into the local one: {e:?}") + .map_err(ApiError::InternalServerError)?; response_data.push(timeline_info); } @@ -267,36 +269,11 @@ async fn timeline_list_handler(request: Request) -> Result, json_response(StatusCode::OK, response_data) } -/// Checks if a query param is present in the request's URL -fn query_param_present(request: &Request, param: &str) -> bool { - request - .uri() - .query() - .map(|v| url::form_urlencoded::parse(v.as_bytes()).any(|(p, _)| p == param)) - .unwrap_or(false) -} - -fn get_query_param(request: &Request, param_name: &str) -> Result { - request.uri().query().map_or( - Err(ApiError::BadRequest(anyhow!("empty query in request"))), - |v| { - url::form_urlencoded::parse(v.as_bytes()) - .find(|(k, _)| k == param_name) - .map_or( - Err(ApiError::BadRequest(anyhow!( - "no {param_name} specified in query parameters" - ))), - |(_, v)| Ok(v.into_owned()), - ) - }, - ) -} - async fn timeline_detail_handler(request: Request) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; - let include_non_incremental_logical_size = - query_param_present(&request, "include-non-incremental-logical-size"); + let include_non_incremental_logical_size: Option = + parse_query_param(&request, "include-non-incremental-logical-size")?; check_permission(&request, Some(tenant_id))?; // Logical size calculation needs downloading. @@ -311,11 +288,14 @@ async fn timeline_detail_handler(request: Request) -> Result(timeline_info) } @@ -330,8 +310,8 @@ async fn get_lsn_by_timestamp_handler(request: Request) -> Result) -> Result, A let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; - let inputs_only = if query_param_present(&request, "inputs_only") { - get_query_param(&request, "inputs_only")? - .parse() - .map_err(|_| ApiError::BadRequest(anyhow!("failed to parse inputs_only")))? - } else { - false - }; + let inputs_only: Option = parse_query_param(&request, "inputs_only")?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let tenant = mgr::get_tenant(tenant_id, true) @@ -522,7 +496,7 @@ async fn tenant_size_handler(request: Request) -> Result, A .await .map_err(ApiError::InternalServerError)?; - let size = if !inputs_only { + let size = if !inputs_only.unwrap_or(false) { Some(inputs.calculate().map_err(ApiError::InternalServerError)?) } else { None diff --git a/scripts/export_import_between_pageservers.py b/scripts/export_import_between_pageservers.py index d83a74ae14..db2b5e81ab 100755 --- a/scripts/export_import_between_pageservers.py +++ b/scripts/export_import_between_pageservers.py @@ -293,7 +293,7 @@ class NeonPageserverHttpClient(requests.Session): def timeline_detail(self, tenant_id: uuid.UUID, timeline_id: uuid.UUID) -> Dict[Any, Any]: res = self.get( - f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline/{timeline_id.hex}?include-non-incremental-logical-size=1" + f"http://localhost:{self.port}/v1/tenant/{tenant_id.hex}/timeline/{timeline_id.hex}?include-non-incremental-logical-size=true" ) self.verbose_error(res) res_json = res.json() diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 8476066691..bd92130124 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1232,9 +1232,9 @@ class PageserverHttpClient(requests.Session): params = {} if include_non_incremental_logical_size: - params["include-non-incremental-logical-size"] = "yes" + params["include-non-incremental-logical-size"] = "true" if include_timeline_dir_layer_file_size_sum: - params["include-timeline-dir-layer-file-size-sum"] = "yes" + params["include-timeline-dir-layer-file-size-sum"] = "true" res = self.get( f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline", params=params @@ -1276,9 +1276,9 @@ class PageserverHttpClient(requests.Session): ) -> Dict[Any, Any]: params = {} if include_non_incremental_logical_size: - params["include-non-incremental-logical-size"] = "yes" + params["include-non-incremental-logical-size"] = "true" if include_timeline_dir_layer_file_size_sum: - params["include-timeline-dir-layer-file-size-sum"] = "yes" + params["include-timeline-dir-layer-file-size-sum"] = "true" res = self.get( f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}", From f1aece1ba09d36cbccddd8a2d78139d82ecdacda Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 1 Feb 2023 14:29:01 +0100 Subject: [PATCH 078/102] add RequestContext plumbing for layer access stats In preparation for #3496 plumb through RequestContext to the data access methods of `PersistentLayer`. This is PR https://github.com/neondatabase/neon/pull/3504 --- pageserver/src/bin/pageserver_binutils.rs | 5 +++- pageserver/src/task_mgr.rs | 2 ++ pageserver/src/tenant.rs | 10 +++++--- pageserver/src/tenant/layer_map.rs | 9 +++---- pageserver/src/tenant/storage_layer.rs | 11 +++++---- .../src/tenant/storage_layer/delta_layer.rs | 18 +++++++------- .../src/tenant/storage_layer/image_layer.rs | 12 ++++++---- .../tenant/storage_layer/inmemory_layer.rs | 4 +++- .../src/tenant/storage_layer/remote_layer.rs | 8 ++++--- pageserver/src/tenant/timeline.rs | 24 +++++++++++++------ 10 files changed, 67 insertions(+), 36 deletions(-) diff --git a/pageserver/src/bin/pageserver_binutils.rs b/pageserver/src/bin/pageserver_binutils.rs index 9da173c873..94877b34b2 100644 --- a/pageserver/src/bin/pageserver_binutils.rs +++ b/pageserver/src/bin/pageserver_binutils.rs @@ -12,7 +12,9 @@ use anyhow::Context; use clap::{value_parser, Arg, Command}; use pageserver::{ + context::{DownloadBehavior, RequestContext}, page_cache, + task_mgr::TaskKind, tenant::{dump_layerfile_from_path, metadata::TimelineMetadata}, virtual_file, }; @@ -75,7 +77,8 @@ fn print_layerfile(path: &Path) -> anyhow::Result<()> { // Basic initialization of things that don't change after startup virtual_file::init(10); page_cache::init(100); - dump_layerfile_from_path(path, true) + let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error); + dump_layerfile_from_path(path, true, &ctx) } fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), anyhow::Error> { diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 09716ba0e0..8e0149046d 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -255,6 +255,8 @@ pub enum TaskKind { // A request that comes in via the pageserver HTTP API. MgmtRequest, + DebugTool, + #[cfg(test)] UnitTest, } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 1af5eb07a2..055c26c8bd 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2643,7 +2643,11 @@ impl Drop for Tenant { } } /// Dump contents of a layer file to stdout. -pub fn dump_layerfile_from_path(path: &Path, verbose: bool) -> anyhow::Result<()> { +pub fn dump_layerfile_from_path( + path: &Path, + verbose: bool, + ctx: &RequestContext, +) -> anyhow::Result<()> { use std::os::unix::fs::FileExt; // All layer files start with a two-byte "magic" value, to identify the kind of @@ -2653,8 +2657,8 @@ pub fn dump_layerfile_from_path(path: &Path, verbose: bool) -> anyhow::Result<() file.read_exact_at(&mut header_buf, 0)?; match u16::from_be_bytes(header_buf) { - crate::IMAGE_FILE_MAGIC => ImageLayer::new_for_path(path, file)?.dump(verbose)?, - crate::DELTA_FILE_MAGIC => DeltaLayer::new_for_path(path, file)?.dump(verbose)?, + crate::IMAGE_FILE_MAGIC => ImageLayer::new_for_path(path, file)?.dump(verbose, ctx)?, + crate::DELTA_FILE_MAGIC => DeltaLayer::new_for_path(path, file)?.dump(verbose, ctx)?, magic => bail!("unrecognized magic identifier: {:?}", magic), } diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index ed1a32c8fd..bb386b436b 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -46,6 +46,7 @@ mod historic_layer_coverage; mod layer_coverage; +use crate::context::RequestContext; use crate::keyspace::KeyPartitioning; use crate::metrics::NUM_ONDISK_LAYERS; use crate::repository::Key; @@ -654,22 +655,22 @@ where /// debugging function to print out the contents of the layer map #[allow(unused)] - pub fn dump(&self, verbose: bool) -> Result<()> { + pub fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { println!("Begin dump LayerMap"); println!("open_layer:"); if let Some(open_layer) = &self.open_layer { - open_layer.dump(verbose)?; + open_layer.dump(verbose, ctx)?; } println!("frozen_layers:"); for frozen_layer in self.frozen_layers.iter() { - frozen_layer.dump(verbose)?; + frozen_layer.dump(verbose, ctx)?; } println!("historic_layers:"); for layer in self.iter_historic_layers() { - layer.dump(verbose)?; + layer.dump(verbose, ctx)?; } println!("End dump LayerMap"); Ok(()) diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index cb7380be2d..8d8aef4a68 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -6,6 +6,7 @@ mod image_layer; mod inmemory_layer; mod remote_layer; +use crate::context::RequestContext; use crate::repository::{Key, Value}; use crate::walrecord::NeonWalRecord; use anyhow::Result; @@ -117,13 +118,14 @@ pub trait Layer: Send + Sync { key: Key, lsn_range: Range, reconstruct_data: &mut ValueReconstructState, + ctx: &RequestContext, ) -> Result; /// A short ID string that uniquely identifies the given layer within a [`LayerMap`]. fn short_id(&self) -> String; /// Dump summary of the contents of the layer to stdout - fn dump(&self, verbose: bool) -> Result<()>; + fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()>; } /// Returned by [`Layer::iter`] @@ -161,11 +163,11 @@ pub trait PersistentLayer: Layer { fn local_path(&self) -> Option; /// Iterate through all keys and values stored in the layer - fn iter(&self) -> Result>; + fn iter(&self, ctx: &RequestContext) -> Result>; /// Iterate through all keys stored in the layer. Returns key, lsn and value size /// It is used only for compaction and so is currently implemented only for DeltaLayer - fn key_iter(&self) -> Result> { + fn key_iter(&self, _ctx: &RequestContext) -> Result> { panic!("Not implemented") } @@ -231,6 +233,7 @@ impl Layer for LayerDescriptor { _key: Key, _lsn_range: Range, _reconstruct_data: &mut ValueReconstructState, + _ctx: &RequestContext, ) -> Result { todo!("This method shouldn't be part of the Layer trait") } @@ -239,7 +242,7 @@ impl Layer for LayerDescriptor { self.short_id.clone() } - fn dump(&self, _verbose: bool) -> Result<()> { + fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { todo!() } } diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index e9bcb55f9b..987b266c76 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -24,6 +24,7 @@ //! "values" part. //! use crate::config::PageServerConf; +use crate::context::RequestContext; use crate::page_cache::{PageReadGuard, PAGE_SZ}; use crate::repository::{Key, Value, KEY_SIZE}; use crate::tenant::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter}; @@ -214,7 +215,7 @@ impl Layer for DeltaLayer { self.filename().file_name() } /// debugging function to print out the contents of the layer - fn dump(&self, verbose: bool) -> Result<()> { + fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { println!( "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} ----", self.tenant_id, @@ -229,7 +230,7 @@ impl Layer for DeltaLayer { return Ok(()); } - let inner = self.load()?; + let inner = self.load(ctx)?; println!( "index_start_blk: {}, root {}", @@ -293,6 +294,7 @@ impl Layer for DeltaLayer { key: Key, lsn_range: Range, reconstruct_state: &mut ValueReconstructState, + ctx: &RequestContext, ) -> anyhow::Result { ensure!(lsn_range.start >= self.lsn_range.start); let mut need_image = true; @@ -301,7 +303,7 @@ impl Layer for DeltaLayer { { // Open the file and lock the metadata in memory - let inner = self.load()?; + let inner = self.load(ctx)?; // Scan the page versions backwards, starting from `lsn`. let file = inner.file.as_ref().unwrap(); @@ -391,16 +393,16 @@ impl PersistentLayer for DeltaLayer { Some(self.path()) } - fn iter(&self) -> Result> { - let inner = self.load().context("load delta layer")?; + fn iter(&self, ctx: &RequestContext) -> Result> { + let inner = self.load(ctx).context("load delta layer")?; Ok(match DeltaValueIter::new(inner) { Ok(iter) => Box::new(iter), Err(err) => Box::new(std::iter::once(Err(err))), }) } - fn key_iter(&self) -> Result> { - let inner = self.load()?; + fn key_iter(&self, ctx: &RequestContext) -> Result> { + let inner = self.load(ctx)?; Ok(Box::new( DeltaKeyIter::new(inner).context("Layer index is corrupted")?, )) @@ -459,7 +461,7 @@ impl DeltaLayer { /// Open the underlying file and read the metadata into memory, if it's /// not loaded already. /// - fn load(&self) -> Result> { + fn load(&self, _ctx: &RequestContext) -> Result> { loop { // Quick exit if already loaded let inner = self.inner.read().unwrap(); diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 9a26fce73b..564f80d258 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -20,6 +20,7 @@ //! mapping from Key to an offset in the "values" part. The //! actual page images are stored in the "values" part. use crate::config::PageServerConf; +use crate::context::RequestContext; use crate::page_cache::PAGE_SZ; use crate::repository::{Key, KEY_SIZE}; use crate::tenant::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter}; @@ -143,7 +144,7 @@ impl Layer for ImageLayer { } /// debugging function to print out the contents of the layer - fn dump(&self, verbose: bool) -> Result<()> { + fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { println!( "----- image layer for ten {} tli {} key {}-{} at {} ----", self.tenant_id, self.timeline_id, self.key_range.start, self.key_range.end, self.lsn @@ -153,7 +154,7 @@ impl Layer for ImageLayer { return Ok(()); } - let inner = self.load()?; + let inner = self.load(ctx)?; let file = inner.file.as_ref().unwrap(); let tree_reader = DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file); @@ -174,12 +175,13 @@ impl Layer for ImageLayer { key: Key, lsn_range: Range, reconstruct_state: &mut ValueReconstructState, + ctx: &RequestContext, ) -> anyhow::Result { assert!(self.key_range.contains(&key)); assert!(lsn_range.start >= self.lsn); assert!(lsn_range.end >= self.lsn); - let inner = self.load()?; + let inner = self.load(ctx)?; let file = inner.file.as_ref().unwrap(); let tree_reader = DiskBtreeReader::new(inner.index_start_blk, inner.index_root_blk, file); @@ -220,7 +222,7 @@ impl PersistentLayer for ImageLayer { fn get_timeline_id(&self) -> TimelineId { self.timeline_id } - fn iter(&self) -> Result> { + fn iter(&self, _ctx: &RequestContext) -> Result> { unimplemented!(); } @@ -270,7 +272,7 @@ impl ImageLayer { /// Open the underlying file and read the metadata into memory, if it's /// not loaded already. /// - fn load(&self) -> Result> { + fn load(&self, _ctx: &RequestContext) -> Result> { loop { // Quick exit if already loaded let inner = self.inner.read().unwrap(); diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 93356a9d8c..e0b252189b 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -5,6 +5,7 @@ //! its position in the file, is kept in memory, though. //! use crate::config::PageServerConf; +use crate::context::RequestContext; use crate::repository::{Key, Value}; use crate::tenant::blob_io::{BlobCursor, BlobWriter}; use crate::tenant::block_io::BlockReader; @@ -110,7 +111,7 @@ impl Layer for InMemoryLayer { } /// debugging function to print out the contents of the layer - fn dump(&self, verbose: bool) -> Result<()> { + fn dump(&self, verbose: bool, _ctx: &RequestContext) -> Result<()> { let inner = self.inner.read().unwrap(); let end_str = inner @@ -166,6 +167,7 @@ impl Layer for InMemoryLayer { key: Key, lsn_range: Range, reconstruct_state: &mut ValueReconstructState, + _ctx: &RequestContext, ) -> anyhow::Result { ensure!(lsn_range.start >= self.start_lsn); let mut need_image = true; diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 33474bb4a2..0478cb82fb 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -2,6 +2,7 @@ //! in remote storage. //! use crate::config::PageServerConf; +use crate::context::RequestContext; use crate::repository::Key; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState}; @@ -51,6 +52,7 @@ impl Layer for RemoteLayer { _key: Key, _lsn_range: Range, _reconstruct_state: &mut ValueReconstructState, + _ctx: &RequestContext, ) -> Result { bail!( "layer {} needs to be downloaded", @@ -63,7 +65,7 @@ impl Layer for RemoteLayer { } /// debugging function to print out the contents of the layer - fn dump(&self, _verbose: bool) -> Result<()> { + fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { println!( "----- remote layer for ten {} tli {} keys {}-{} lsn {}-{} ----", self.tenantid, @@ -111,11 +113,11 @@ impl PersistentLayer for RemoteLayer { None } - fn iter(&self) -> Result> { + fn iter(&self, _ctx: &RequestContext) -> Result> { bail!("cannot iterate a remote layer"); } - fn key_iter(&self) -> Result> { + fn key_iter(&self, _ctx: &RequestContext) -> Result> { bail!("cannot iterate a remote layer"); } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 0ca8a0e491..8134d2a13a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -696,7 +696,7 @@ impl Timeline { // 3. Compact let timer = self.metrics.compact_time_histo.start_timer(); - self.compact_level0(target_file_size).await?; + self.compact_level0(target_file_size, ctx).await?; timer.stop_and_record(); // If `create_image_layers' or `compact_level0` scheduled any @@ -1767,6 +1767,7 @@ impl Timeline { key, lsn_floor..cont_lsn, reconstruct_state, + ctx, ) { Ok(result) => result, Err(e) => return Err(PageReconstructError::from(e)), @@ -1792,6 +1793,7 @@ impl Timeline { key, lsn_floor..cont_lsn, reconstruct_state, + ctx, ) { Ok(result) => result, Err(e) => return Err(PageReconstructError::from(e)), @@ -1825,6 +1827,7 @@ impl Timeline { key, lsn_floor..cont_lsn, reconstruct_state, + ctx, ) { Ok(result) => result, Err(e) => return Err(PageReconstructError::from(e)), @@ -2463,6 +2466,7 @@ impl Timeline { async fn compact_level0_phase1( &self, target_file_size: u64, + ctx: &RequestContext, ) -> anyhow::Result { let layers = self.layers.read().unwrap(); let mut level0_deltas = layers.get_level0_deltas()?; @@ -2522,8 +2526,9 @@ impl Timeline { // This iterator walks through all key-value pairs from all the layers // we're compacting, in key, LSN order. - let all_values_iter = - itertools::process_results(deltas_to_compact.iter().map(|l| l.iter()), |iter_iter| { + let all_values_iter = itertools::process_results( + deltas_to_compact.iter().map(|l| l.iter(ctx)), + |iter_iter| { iter_iter.kmerge_by(|a, b| { if let Ok((a_key, a_lsn, _)) = a { if let Ok((b_key, b_lsn, _)) = b { @@ -2539,11 +2544,12 @@ impl Timeline { true } }) - })?; + }, + )?; // This iterator walks through all keys and is needed to calculate size used by each key let mut all_keys_iter = itertools::process_results( - deltas_to_compact.iter().map(|l| l.key_iter()), + deltas_to_compact.iter().map(|l| l.key_iter(ctx)), |iter_iter| { iter_iter.kmerge_by(|a, b| { let (a_key, a_lsn, _) = a; @@ -2719,11 +2725,15 @@ impl Timeline { /// Collect a bunch of Level 0 layer files, and compact and reshuffle them as /// as Level 1 files. /// - async fn compact_level0(&self, target_file_size: u64) -> anyhow::Result<()> { + async fn compact_level0( + &self, + target_file_size: u64, + ctx: &RequestContext, + ) -> anyhow::Result<()> { let CompactLevel0Phase1Result { new_layers, deltas_to_compact, - } = self.compact_level0_phase1(target_file_size).await?; + } = self.compact_level0_phase1(target_file_size, ctx).await?; if new_layers.is_empty() && deltas_to_compact.is_empty() { // nothing to do From ea0278cf27a3bd2c058f4553fa7435d68455589c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 1 Feb 2023 17:11:41 +0300 Subject: [PATCH 079/102] [proxy] Implement compute node info cache (#3331) This patch adds a timed LRU cache implementation and a compute node info cache on top of that. Cache entries might expire on their own (default ttl=5mins) or become invalid due to real-world events, e.g. compute node scale-to-zero event, so we add a connection retry loop with a wake-up call. Solved problems: - [x] Find a decent LRU implementation. - [x] Implement timed LRU on top of that. - [x] Cache results of `proxy_wake_compute` API call. - [x] Don't invalidate newer cache entries for the same key. - [x] Add cmdline configuration knobs (requires some refactoring). - [x] Add failed connection estab metric. - [x] Refactor auth backends to make things simpler (retries, cache placement, etc). - [x] Address review comments (add code comments + cleanup). - [x] Retry `/proxy_wake_compute` if we couldn't connect to a compute (e.g. stalled cache entry). - [x] Add high-level description for `TimedLru`. TODOs (will be addressed later): - [ ] Add cache metrics (hit, spurious hit, miss). - [ ] Synchronize http requests across concurrent per-client tasks (https://github.com/neondatabase/neon/pull/3331#issuecomment-1399216069). - [ ] Cache results of `proxy_get_role_secret` API call. --- Cargo.lock | 27 +- Cargo.toml | 1 + proxy/Cargo.toml | 31 +- proxy/src/auth.rs | 8 +- proxy/src/auth/backend.rs | 164 ++++---- proxy/src/auth/backend/classic.rs | 61 +++ proxy/src/auth/backend/console.rs | 365 ------------------ proxy/src/auth/backend/link.rs | 18 +- proxy/src/auth/credentials.rs | 1 + proxy/src/cache.rs | 304 +++++++++++++++ proxy/src/cancellation.rs | 16 +- proxy/src/compute.rs | 105 ++--- proxy/src/config.rs | 86 ++++- proxy/src/console.rs | 12 + proxy/src/console/messages.rs | 8 +- proxy/src/{ => console}/mgmt.rs | 24 +- proxy/src/console/provider.rs | 194 ++++++++++ .../postgres.rs => console/provider/mock.rs} | 105 ++--- proxy/src/console/provider/neon.rs | 196 ++++++++++ proxy/src/http/server.rs | 3 +- proxy/src/http/websocket.rs | 4 +- proxy/src/main.rs | 182 +++++---- proxy/src/proxy.rs | 220 ++++++++--- workspace_hack/Cargo.toml | 2 + 24 files changed, 1401 insertions(+), 736 deletions(-) create mode 100644 proxy/src/auth/backend/classic.rs delete mode 100644 proxy/src/auth/backend/console.rs create mode 100644 proxy/src/cache.rs rename proxy/src/{ => console}/mgmt.rs (79%) create mode 100644 proxy/src/console/provider.rs rename proxy/src/{auth/backend/postgres.rs => console/provider/mock.rs} (53%) create mode 100644 proxy/src/console/provider/neon.rs diff --git a/Cargo.lock b/Cargo.lock index 794ec25bf7..9be9af35e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.2" @@ -1515,6 +1526,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] [[package]] name = "hashbrown" @@ -1522,7 +1536,16 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash", + "ahash 0.8.2", +] + +[[package]] +name = "hashlink" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" +dependencies = [ + "hashbrown 0.12.3", ] [[package]] @@ -2764,6 +2787,7 @@ dependencies = [ "futures", "git-version", "hashbrown 0.13.2", + "hashlink", "hex", "hmac", "hostname", @@ -4624,6 +4648,7 @@ dependencies = [ "futures-executor", "futures-task", "futures-util", + "hashbrown 0.12.3", "indexmap", "itertools", "libc", diff --git a/Cargo.toml b/Cargo.toml index e6695c4246..9dcf1a265a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ futures-core = "0.3" futures-util = "0.3" git-version = "0.3" hashbrown = "0.13" +hashlink = "0.8.1" hex = "0.4" hex-literal = "0.3" hmac = "0.12.1" diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 03a6ddac5d..1ff7eebd98 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -6,58 +6,59 @@ license.workspace = true [dependencies] anyhow.workspace = true +async-trait.workspace = true atty.workspace = true base64.workspace = true bstr.workspace = true -bytes = {workspace = true, features = ['serde'] } -clap.workspace = true +bytes = { workspace = true, features = ["serde"] } chrono.workspace = true +clap.workspace = true consumption_metrics.workspace = true futures.workspace = true git-version.workspace = true hashbrown.workspace = true +hashlink.workspace = true hex.workspace = true hmac.workspace = true -hyper.workspace = true +hostname.workspace = true +humantime.workspace = true hyper-tungstenite.workspace = true +hyper.workspace = true itertools.workspace = true md5.workspace = true +metrics.workspace = true once_cell.workspace = true parking_lot.workspace = true pin-project-lite.workspace = true +pq_proto.workspace = true +prometheus.workspace = true rand.workspace = true regex.workspace = true -reqwest = { workspace = true, features = [ "json" ] } +reqwest = { workspace = true, features = ["json"] } routerify.workspace = true -rustls.workspace = true rustls-pemfile.workspace = true +rustls.workspace = true scopeguard.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true socket2.workspace = true thiserror.workspace = true -tokio.workspace = true +tls-listener.workspace = true tokio-postgres.workspace = true tokio-rustls.workspace = true -tls-listener.workspace = true -tracing.workspace = true +tokio.workspace = true tracing-subscriber.workspace = true +tracing.workspace = true url.workspace = true +utils.workspace = true uuid.workspace = true webpki-roots.workspace = true x509-parser.workspace = true -metrics.workspace = true -pq_proto.workspace = true -utils.workspace = true -prometheus.workspace = true -humantime.workspace = true -hostname.workspace = true workspace_hack.workspace = true [dev-dependencies] -async-trait.workspace = true rcgen.workspace = true rstest.workspace = true tokio-postgres-rustls.workspace = true diff --git a/proxy/src/auth.rs b/proxy/src/auth.rs index 0446b53603..dfea84953b 100644 --- a/proxy/src/auth.rs +++ b/proxy/src/auth.rs @@ -1,7 +1,7 @@ //! Client authentication mechanisms. pub mod backend; -pub use backend::{BackendType, ConsoleReqExtra}; +pub use backend::BackendType; mod credentials; pub use credentials::ClientCredentials; @@ -12,7 +12,7 @@ use password_hack::PasswordHackPayload; mod flow; pub use flow::*; -use crate::error::UserFacingError; +use crate::{console, error::UserFacingError}; use std::io; use thiserror::Error; @@ -26,10 +26,10 @@ pub enum AuthErrorImpl { Link(#[from] backend::LinkAuthError), #[error(transparent)] - GetAuthInfo(#[from] backend::GetAuthInfoError), + GetAuthInfo(#[from] console::errors::GetAuthInfoError), #[error(transparent)] - WakeCompute(#[from] backend::WakeComputeError), + WakeCompute(#[from] console::errors::WakeComputeError), /// SASL protocol errors (includes [SCRAM](crate::scram)). #[error(transparent)] diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index e6a179a040..60460e6722 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -1,48 +1,40 @@ -mod postgres; +mod classic; mod link; +use futures::TryFutureExt; pub use link::LinkAuthError; -mod console; -pub use console::{GetAuthInfoError, WakeComputeError}; - use crate::{ auth::{self, AuthFlow, ClientCredentials}, - compute, - console::messages::MetricsAuxInfo, - http, mgmt, stream, url, - waiters::{self, Waiter, Waiters}, + console::{ + self, + provider::{CachedNodeInfo, ConsoleReqExtra}, + Api, + }, + stream, url, }; -use once_cell::sync::Lazy; use std::borrow::Cow; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{info, warn}; -static CPLANE_WAITERS: Lazy> = Lazy::new(Default::default); - -/// Give caller an opportunity to wait for the cloud's reply. -pub async fn with_waiter( - psql_session_id: impl Into, - action: impl FnOnce(Waiter<'static, mgmt::ComputeReady>) -> R, -) -> Result -where - R: std::future::Future>, - E: From, -{ - let waiter = CPLANE_WAITERS.register(psql_session_id.into())?; - action(waiter).await +/// A product of successful authentication. +pub struct AuthSuccess { + /// Did we send [`pq_proto::BeMessage::AuthenticationOk`] to client? + pub reported_auth_ok: bool, + /// Something to be considered a positive result. + pub value: T, } -pub fn notify(psql_session_id: &str, msg: mgmt::ComputeReady) -> Result<(), waiters::NotifyError> { - CPLANE_WAITERS.notify(psql_session_id, msg) -} - -/// Extra query params we'd like to pass to the console. -pub struct ConsoleReqExtra<'a> { - /// A unique identifier for a connection. - pub session_id: uuid::Uuid, - /// Name of client application, if set. - pub application_name: Option<&'a str>, +impl AuthSuccess { + /// Very similar to [`std::option::Option::map`]. + /// Maps [`AuthSuccess`] to [`AuthSuccess`] by applying + /// a function to a contained value. + pub fn map(self, f: impl FnOnce(T) -> R) -> AuthSuccess { + AuthSuccess { + reported_auth_ok: self.reported_auth_ok, + value: f(self.value), + } + } } /// This type serves two purposes: @@ -53,12 +45,11 @@ pub struct ConsoleReqExtra<'a> { /// * However, when we substitute `T` with [`ClientCredentials`], /// this helps us provide the credentials only to those auth /// backends which require them for the authentication process. -#[derive(Debug)] pub enum BackendType<'a, T> { /// Current Cloud API (V2). - Console(Cow<'a, http::Endpoint>, T), + Console(Cow<'a, console::provider::neon::Api>, T), /// Local mock of Cloud API (V2). - Postgres(Cow<'a, url::ApiUrl>, T), + Postgres(Cow<'a, console::provider::mock::Api>, T), /// Authentication via a web browser. Link(Cow<'a, url::ApiUrl>), } @@ -67,14 +58,8 @@ impl std::fmt::Display for BackendType<'_, ()> { fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { use BackendType::*; match self { - Console(endpoint, _) => fmt - .debug_tuple("Console") - .field(&endpoint.url().as_str()) - .finish(), - Postgres(endpoint, _) => fmt - .debug_tuple("Postgres") - .field(&endpoint.as_str()) - .finish(), + Console(endpoint, _) => fmt.debug_tuple("Console").field(&endpoint.url()).finish(), + Postgres(endpoint, _) => fmt.debug_tuple("Postgres").field(&endpoint.url()).finish(), Link(url) => fmt.debug_tuple("Link").field(&url.as_str()).finish(), } } @@ -120,30 +105,16 @@ impl<'a, T, E> BackendType<'a, Result> { } } -/// A product of successful authentication. -pub struct AuthSuccess { - /// Did we send [`pq_proto::BeMessage::AuthenticationOk`] to client? - pub reported_auth_ok: bool, - /// Something to be considered a positive result. - pub value: T, -} - -/// Info for establishing a connection to a compute node. -/// This is what we get after auth succeeded, but not before! -pub struct NodeInfo { - /// Compute node connection params. - pub config: compute::ConnCfg, - /// Labels for proxy's metrics. - pub aux: MetricsAuxInfo, -} - -impl BackendType<'_, ClientCredentials<'_>> { +// TODO: get rid of explicit lifetimes in this block (there's a bug in rustc). +// Read more: https://github.com/rust-lang/rust/issues/99190 +// Alleged fix: https://github.com/rust-lang/rust/pull/89056 +impl<'l> BackendType<'l, ClientCredentials<'_>> { /// Do something special if user didn't provide the `project` parameter. - async fn try_password_hack( - &mut self, - extra: &ConsoleReqExtra<'_>, - client: &mut stream::PqStream, - ) -> auth::Result>> { + async fn try_password_hack<'a>( + &'a mut self, + extra: &'a ConsoleReqExtra<'a>, + client: &'a mut stream::PqStream, + ) -> auth::Result>> { use BackendType::*; // If there's no project so far, that entails that client doesn't @@ -179,33 +150,28 @@ impl BackendType<'_, ClientCredentials<'_>> { // TODO: find a proper way to merge those very similar blocks. let (mut node, payload) = match self { - Console(endpoint, creds) if creds.project.is_none() => { + Console(api, creds) if creds.project.is_none() => { let payload = fetch_magic_payload(client).await?; let mut creds = creds.as_ref(); creds.project = Some(payload.project.as_str().into()); - let node = console::Api::new(endpoint, extra, &creds) - .wake_compute() - .await?; + let node = api.wake_compute(extra, &creds).await?; (node, payload) } - Console(endpoint, creds) if creds.use_cleartext_password_flow => { - // This is a hack to allow cleartext password in secure connections (wss). + // This is a hack to allow cleartext password in secure connections (wss). + Console(api, creds) if creds.use_cleartext_password_flow => { let payload = fetch_plaintext_password(client).await?; - let creds = creds.as_ref(); - let node = console::Api::new(endpoint, extra, &creds) - .wake_compute() - .await?; + let node = api.wake_compute(extra, creds).await?; (node, payload) } - Postgres(endpoint, creds) if creds.project.is_none() => { + Postgres(api, creds) if creds.project.is_none() => { let payload = fetch_magic_payload(client).await?; let mut creds = creds.as_ref(); creds.project = Some(payload.project.as_str().into()); - let node = postgres::Api::new(endpoint, &creds).wake_compute().await?; + let node = api.wake_compute(extra, &creds).await?; (node, payload) } @@ -220,11 +186,11 @@ impl BackendType<'_, ClientCredentials<'_>> { } /// Authenticate the client via the requested backend, possibly using credentials. - pub async fn authenticate( - mut self, - extra: &ConsoleReqExtra<'_>, - client: &mut stream::PqStream, - ) -> auth::Result> { + pub async fn authenticate<'a>( + &mut self, + extra: &'a ConsoleReqExtra<'a>, + client: &'a mut stream::PqStream, + ) -> auth::Result> { use BackendType::*; // Handle cases when `project` is missing in `creds`. @@ -235,7 +201,7 @@ impl BackendType<'_, ClientCredentials<'_>> { } let res = match self { - Console(endpoint, creds) => { + Console(api, creds) => { info!( user = creds.user, project = creds.project(), @@ -243,26 +209,40 @@ impl BackendType<'_, ClientCredentials<'_>> { ); assert!(creds.project.is_some()); - console::Api::new(&endpoint, extra, &creds) - .handle_user(client) - .await? + classic::handle_user(api.as_ref(), extra, creds, client).await? } - Postgres(endpoint, creds) => { + Postgres(api, creds) => { info!("performing mock authentication using a local postgres instance"); assert!(creds.project.is_some()); - postgres::Api::new(&endpoint, &creds) - .handle_user(client) - .await? + classic::handle_user(api.as_ref(), extra, creds, client).await? } // NOTE: this auth backend doesn't use client credentials. Link(url) => { info!("performing link authentication"); - link::handle_user(&url, client).await? + + link::handle_user(url, client) + .await? + .map(CachedNodeInfo::new_uncached) } }; info!("user successfully authenticated"); Ok(res) } + + /// When applicable, wake the compute node, gaining its connection info in the process. + /// The link auth flow doesn't support this, so we return [`None`] in that case. + pub async fn wake_compute<'a>( + &self, + extra: &'a ConsoleReqExtra<'a>, + ) -> Result, console::errors::WakeComputeError> { + use BackendType::*; + + match self { + Console(api, creds) => api.wake_compute(extra, creds).map_ok(Some).await, + Postgres(api, creds) => api.wake_compute(extra, creds).map_ok(Some).await, + Link(_) => Ok(None), + } + } } diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs new file mode 100644 index 0000000000..eefef6e9b4 --- /dev/null +++ b/proxy/src/auth/backend/classic.rs @@ -0,0 +1,61 @@ +use super::AuthSuccess; +use crate::{ + auth::{self, AuthFlow, ClientCredentials}, + compute, + console::{self, AuthInfo, CachedNodeInfo, ConsoleReqExtra}, + sasl, scram, + stream::PqStream, +}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tracing::info; + +pub(super) async fn handle_user( + api: &impl console::Api, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + client: &mut PqStream, +) -> auth::Result> { + info!("fetching user's authentication info"); + let info = api.get_auth_info(extra, creds).await?.unwrap_or_else(|| { + // If we don't have an authentication secret, we mock one to + // prevent malicious probing (possible due to missing protocol steps). + // This mocked secret will never lead to successful authentication. + info!("authentication info not found, mocking it"); + AuthInfo::Scram(scram::ServerSecret::mock(creds.user, rand::random())) + }); + + let flow = AuthFlow::new(client); + let scram_keys = match info { + AuthInfo::Md5(_) => { + info!("auth endpoint chooses MD5"); + return Err(auth::AuthError::bad_auth_method("MD5")); + } + AuthInfo::Scram(secret) => { + info!("auth endpoint chooses SCRAM"); + let scram = auth::Scram(&secret); + let client_key = match flow.begin(scram).await?.authenticate().await? { + sasl::Outcome::Success(key) => key, + sasl::Outcome::Failure(reason) => { + info!("auth backend failed with an error: {reason}"); + return Err(auth::AuthError::auth_failed(creds.user)); + } + }; + + Some(compute::ScramKeys { + client_key: client_key.as_bytes(), + server_key: secret.server_key.as_bytes(), + }) + } + }; + + let mut node = api.wake_compute(extra, creds).await?; + if let Some(keys) = scram_keys { + use tokio_postgres::config::AuthKeys; + node.config.auth_keys(AuthKeys::ScramSha256(keys)); + } + + Ok(AuthSuccess { + reported_auth_ok: false, + value: node, + }) +} diff --git a/proxy/src/auth/backend/console.rs b/proxy/src/auth/backend/console.rs deleted file mode 100644 index b3e3fd0c10..0000000000 --- a/proxy/src/auth/backend/console.rs +++ /dev/null @@ -1,365 +0,0 @@ -//! Cloud API V2. - -use super::{AuthSuccess, ConsoleReqExtra, NodeInfo}; -use crate::{ - auth::{self, AuthFlow, ClientCredentials}, - compute, - console::messages::{ConsoleError, GetRoleSecret, WakeCompute}, - error::{io_error, UserFacingError}, - http, sasl, scram, - stream::PqStream, -}; -use futures::TryFutureExt; -use reqwest::StatusCode as HttpStatusCode; -use std::future::Future; -use thiserror::Error; -use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::{error, info, info_span, warn, Instrument}; - -/// A go-to error message which doesn't leak any detail. -const REQUEST_FAILED: &str = "Console request failed"; - -/// Common console API error. -#[derive(Debug, Error)] -pub enum ApiError { - /// Error returned by the console itself. - #[error("{REQUEST_FAILED} with {}: {}", .status, .text)] - Console { - status: HttpStatusCode, - text: Box, - }, - - /// Various IO errors like broken pipe or malformed payload. - #[error("{REQUEST_FAILED}: {0}")] - Transport(#[from] std::io::Error), -} - -impl ApiError { - /// Returns HTTP status code if it's the reason for failure. - fn http_status_code(&self) -> Option { - use ApiError::*; - match self { - Console { status, .. } => Some(*status), - _ => None, - } - } -} - -impl UserFacingError for ApiError { - fn to_string_client(&self) -> String { - use ApiError::*; - match self { - // To minimize risks, only select errors are forwarded to users. - // Ask @neondatabase/control-plane for review before adding more. - Console { status, .. } => match *status { - HttpStatusCode::NOT_FOUND => { - // Status 404: failed to get a project-related resource. - format!("{REQUEST_FAILED}: endpoint cannot be found") - } - HttpStatusCode::NOT_ACCEPTABLE => { - // Status 406: endpoint is disabled (we don't allow connections). - format!("{REQUEST_FAILED}: endpoint is disabled") - } - HttpStatusCode::LOCKED => { - // Status 423: project might be in maintenance mode (or bad state). - format!("{REQUEST_FAILED}: endpoint is temporary unavailable") - } - _ => REQUEST_FAILED.to_owned(), - }, - _ => REQUEST_FAILED.to_owned(), - } - } -} - -// Helps eliminate graceless `.map_err` calls without introducing another ctor. -impl From for ApiError { - fn from(e: reqwest::Error) -> Self { - io_error(e).into() - } -} - -#[derive(Debug, Error)] -pub enum GetAuthInfoError { - // We shouldn't include the actual secret here. - #[error("Console responded with a malformed auth secret")] - BadSecret, - - #[error(transparent)] - ApiError(ApiError), -} - -// This allows more useful interactions than `#[from]`. -impl> From for GetAuthInfoError { - fn from(e: E) -> Self { - Self::ApiError(e.into()) - } -} - -impl UserFacingError for GetAuthInfoError { - fn to_string_client(&self) -> String { - use GetAuthInfoError::*; - match self { - // We absolutely should not leak any secrets! - BadSecret => REQUEST_FAILED.to_owned(), - // However, API might return a meaningful error. - ApiError(e) => e.to_string_client(), - } - } -} - -#[derive(Debug, Error)] -pub enum WakeComputeError { - #[error("Console responded with a malformed compute address: {0}")] - BadComputeAddress(Box), - - #[error(transparent)] - ApiError(ApiError), -} - -// This allows more useful interactions than `#[from]`. -impl> From for WakeComputeError { - fn from(e: E) -> Self { - Self::ApiError(e.into()) - } -} - -impl UserFacingError for WakeComputeError { - fn to_string_client(&self) -> String { - use WakeComputeError::*; - match self { - // We shouldn't show user the address even if it's broken. - // Besides, user is unlikely to care about this detail. - BadComputeAddress(_) => REQUEST_FAILED.to_owned(), - // However, API might return a meaningful error. - ApiError(e) => e.to_string_client(), - } - } -} - -/// Auth secret which is managed by the cloud. -pub enum AuthInfo { - /// Md5 hash of user's password. - Md5([u8; 16]), - - /// [SCRAM](crate::scram) authentication info. - Scram(scram::ServerSecret), -} - -#[must_use] -pub(super) struct Api<'a> { - endpoint: &'a http::Endpoint, - extra: &'a ConsoleReqExtra<'a>, - creds: &'a ClientCredentials<'a>, -} - -impl<'a> AsRef> for Api<'a> { - fn as_ref(&self) -> &ClientCredentials<'a> { - self.creds - } -} - -impl<'a> Api<'a> { - /// Construct an API object containing the auth parameters. - pub(super) fn new( - endpoint: &'a http::Endpoint, - extra: &'a ConsoleReqExtra<'a>, - creds: &'a ClientCredentials, - ) -> Self { - Self { - endpoint, - extra, - creds, - } - } - - /// Authenticate the existing user or throw an error. - pub(super) async fn handle_user( - &'a self, - client: &mut PqStream, - ) -> auth::Result> { - handle_user(client, self, Self::get_auth_info, Self::wake_compute).await - } -} - -impl Api<'_> { - async fn get_auth_info(&self) -> Result, GetAuthInfoError> { - let request_id = uuid::Uuid::new_v4().to_string(); - async { - let request = self - .endpoint - .get("proxy_get_role_secret") - .header("X-Request-ID", &request_id) - .query(&[("session_id", self.extra.session_id)]) - .query(&[ - ("application_name", self.extra.application_name), - ("project", Some(self.creds.project().expect("impossible"))), - ("role", Some(self.creds.user)), - ]) - .build()?; - - info!(url = request.url().as_str(), "sending http request"); - let response = self.endpoint.execute(request).await?; - let body = match parse_body::(response).await { - Ok(body) => body, - // Error 404 is special: it's ok not to have a secret. - Err(e) => match e.http_status_code() { - Some(HttpStatusCode::NOT_FOUND) => return Ok(None), - _otherwise => return Err(e.into()), - }, - }; - - let secret = scram::ServerSecret::parse(&body.role_secret) - .map(AuthInfo::Scram) - .ok_or(GetAuthInfoError::BadSecret)?; - - Ok(Some(secret)) - } - .map_err(crate::error::log_error) - .instrument(info_span!("get_auth_info", id = request_id)) - .await - } - - /// Wake up the compute node and return the corresponding connection info. - pub async fn wake_compute(&self) -> Result { - let request_id = uuid::Uuid::new_v4().to_string(); - async { - let request = self - .endpoint - .get("proxy_wake_compute") - .header("X-Request-ID", &request_id) - .query(&[("session_id", self.extra.session_id)]) - .query(&[ - ("application_name", self.extra.application_name), - ("project", Some(self.creds.project().expect("impossible"))), - ]) - .build()?; - - info!(url = request.url().as_str(), "sending http request"); - let response = self.endpoint.execute(request).await?; - let body = parse_body::(response).await?; - - // Unfortunately, ownership won't let us use `Option::ok_or` here. - let (host, port) = match parse_host_port(&body.address) { - None => return Err(WakeComputeError::BadComputeAddress(body.address)), - Some(x) => x, - }; - - let mut config = compute::ConnCfg::new(); - config - .host(host) - .port(port) - .dbname(self.creds.dbname) - .user(self.creds.user); - - Ok(NodeInfo { - config, - aux: body.aux, - }) - } - .map_err(crate::error::log_error) - .instrument(info_span!("wake_compute", id = request_id)) - .await - } -} - -/// Common logic for user handling in API V2. -/// We reuse this for a mock API implementation in [`super::postgres`]. -pub(super) async fn handle_user<'a, Endpoint, GetAuthInfo, WakeCompute>( - client: &mut PqStream, - endpoint: &'a Endpoint, - get_auth_info: impl FnOnce(&'a Endpoint) -> GetAuthInfo, - wake_compute: impl FnOnce(&'a Endpoint) -> WakeCompute, -) -> auth::Result> -where - Endpoint: AsRef>, - GetAuthInfo: Future, GetAuthInfoError>>, - WakeCompute: Future>, -{ - let creds = endpoint.as_ref(); - - info!("fetching user's authentication info"); - let info = get_auth_info(endpoint).await?.unwrap_or_else(|| { - // If we don't have an authentication secret, we mock one to - // prevent malicious probing (possible due to missing protocol steps). - // This mocked secret will never lead to successful authentication. - info!("authentication info not found, mocking it"); - AuthInfo::Scram(scram::ServerSecret::mock(creds.user, rand::random())) - }); - - let flow = AuthFlow::new(client); - let scram_keys = match info { - AuthInfo::Md5(_) => { - info!("auth endpoint chooses MD5"); - return Err(auth::AuthError::bad_auth_method("MD5")); - } - AuthInfo::Scram(secret) => { - info!("auth endpoint chooses SCRAM"); - let scram = auth::Scram(&secret); - let client_key = match flow.begin(scram).await?.authenticate().await? { - sasl::Outcome::Success(key) => key, - sasl::Outcome::Failure(reason) => { - info!("auth backend failed with an error: {reason}"); - return Err(auth::AuthError::auth_failed(creds.user)); - } - }; - - Some(compute::ScramKeys { - client_key: client_key.as_bytes(), - server_key: secret.server_key.as_bytes(), - }) - } - }; - - let mut node = wake_compute(endpoint).await?; - if let Some(keys) = scram_keys { - use tokio_postgres::config::AuthKeys; - node.config.auth_keys(AuthKeys::ScramSha256(keys)); - } - - Ok(AuthSuccess { - reported_auth_ok: false, - value: node, - }) -} - -/// Parse http response body, taking status code into account. -async fn parse_body serde::Deserialize<'a>>( - response: reqwest::Response, -) -> Result { - let status = response.status(); - if status.is_success() { - // We shouldn't log raw body because it may contain secrets. - info!("request succeeded, processing the body"); - return Ok(response.json().await?); - } - - // Don't throw an error here because it's not as important - // as the fact that the request itself has failed. - let body = response.json().await.unwrap_or_else(|e| { - warn!("failed to parse error body: {e}"); - ConsoleError { - error: "reason unclear (malformed error message)".into(), - } - }); - - let text = body.error; - error!("console responded with an error ({status}): {text}"); - Err(ApiError::Console { status, text }) -} - -fn parse_host_port(input: &str) -> Option<(&str, u16)> { - let (host, port) = input.split_once(':')?; - Some((host, port.parse().ok()?)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_host_port() { - let (host, port) = parse_host_port("127.0.0.1:5432").expect("failed to parse"); - assert_eq!(host, "127.0.0.1"); - assert_eq!(port, 5432); - } -} diff --git a/proxy/src/auth/backend/link.rs b/proxy/src/auth/backend/link.rs index e16bbc70e4..ef92b1a444 100644 --- a/proxy/src/auth/backend/link.rs +++ b/proxy/src/auth/backend/link.rs @@ -1,5 +1,11 @@ -use super::{AuthSuccess, NodeInfo}; -use crate::{auth, compute, error::UserFacingError, stream::PqStream, waiters}; +use super::AuthSuccess; +use crate::{ + auth, compute, + console::{self, provider::NodeInfo}, + error::UserFacingError, + stream::PqStream, + waiters, +}; use pq_proto::BeMessage as Be; use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite}; @@ -47,7 +53,7 @@ pub fn new_psql_session_id() -> String { hex::encode(rand::random::<[u8; 8]>()) } -pub async fn handle_user( +pub(super) async fn handle_user( link_uri: &reqwest::Url, client: &mut PqStream, ) -> auth::Result> { @@ -55,7 +61,7 @@ pub async fn handle_user( let span = info_span!("link", psql_session_id = &psql_session_id); let greeting = hello_message(link_uri, &psql_session_id); - let db_info = super::with_waiter(psql_session_id, |waiter| async { + let db_info = console::mgmt::with_waiter(psql_session_id, |waiter| async { // Give user a URL to spawn a new database. info!(parent: &span, "sending the auth URL to the user"); client @@ -80,14 +86,14 @@ pub async fn handle_user( .user(&db_info.user); if let Some(password) = db_info.password { - config.password(password); + config.password(password.as_ref()); } Ok(AuthSuccess { reported_auth_ok: true, value: NodeInfo { config, - aux: db_info.aux, + aux: db_info.aux.into(), }, }) } diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index 3b71bef9aa..e1943fe44c 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -33,6 +33,7 @@ impl UserFacingError for ClientCredsParseError {} pub struct ClientCredentials<'a> { pub user: &'a str, pub dbname: &'a str, + // TODO: this is a severe misnomer! We should think of a new name ASAP. pub project: Option>, /// If `True`, we'll use the old cleartext password flow. This is used for /// websocket connections, which want to minimize the number of round trips. diff --git a/proxy/src/cache.rs b/proxy/src/cache.rs new file mode 100644 index 0000000000..4e16cc39ec --- /dev/null +++ b/proxy/src/cache.rs @@ -0,0 +1,304 @@ +use std::{ + borrow::Borrow, + hash::Hash, + ops::{Deref, DerefMut}, + time::{Duration, Instant}, +}; +use tracing::debug; + +// This seems to make more sense than `lru` or `cached`: +// +// * `near/nearcore` ditched `cached` in favor of `lru` +// (https://github.com/near/nearcore/issues?q=is%3Aissue+lru+is%3Aclosed). +// +// * `lru` methods use an obscure `KeyRef` type in their contraints (which is deliberately excluded from docs). +// This severely hinders its usage both in terms of creating wrappers and supported key types. +// +// On the other hand, `hashlink` has good download stats and appears to be maintained. +use hashlink::{linked_hash_map::RawEntryMut, LruCache}; + +/// A generic trait which exposes types of cache's key and value, +/// as well as the notion of cache entry invalidation. +/// This is useful for [`timed_lru::Cached`]. +pub trait Cache { + /// Entry's key. + type Key; + + /// Entry's value. + type Value; + + /// Used for entry invalidation. + type LookupInfo; + + /// Invalidate an entry using a lookup info. + /// We don't have an empty default impl because it's error-prone. + fn invalidate(&self, _: &Self::LookupInfo); +} + +impl Cache for &C { + type Key = C::Key; + type Value = C::Value; + type LookupInfo = C::LookupInfo; + + fn invalidate(&self, info: &Self::LookupInfo) { + C::invalidate(self, info) + } +} + +pub use timed_lru::TimedLru; +pub mod timed_lru { + use super::*; + + /// An implementation of timed LRU cache with fixed capacity. + /// Key properties: + /// + /// * Whenever a new entry is inserted, the least recently accessed one is evicted. + /// The cache also keeps track of entry's insertion time (`created_at`) and TTL (`expires_at`). + /// + /// * When the entry is about to be retrieved, we check its expiration timestamp. + /// If the entry has expired, we remove it from the cache; Otherwise we bump the + /// expiration timestamp (e.g. +5mins) and change its place in LRU list to prolong + /// its existence. + /// + /// * There's an API for immediate invalidation (removal) of a cache entry; + /// It's useful in case we know for sure that the entry is no longer correct. + /// See [`timed_lru::LookupInfo`] & [`timed_lru::Cached`] for more information. + /// + /// * Expired entries are kept in the cache, until they are evicted by the LRU policy, + /// or by a successful lookup (i.e. the entry hasn't expired yet). + /// There is no background job to reap the expired records. + /// + /// * It's possible for an entry that has not yet expired entry to be evicted + /// before expired items. That's a bit wasteful, but probably fine in practice. + pub struct TimedLru { + /// Cache's name for tracing. + name: &'static str, + + /// The underlying cache implementation. + cache: parking_lot::Mutex>>, + + /// Default time-to-live of a single entry. + ttl: Duration, + } + + impl Cache for TimedLru { + type Key = K; + type Value = V; + type LookupInfo = LookupInfo; + + fn invalidate(&self, info: &Self::LookupInfo) { + self.invalidate_raw(info) + } + } + + struct Entry { + created_at: Instant, + expires_at: Instant, + value: T, + } + + impl TimedLru { + /// Construct a new LRU cache with timed entries. + pub fn new(name: &'static str, capacity: usize, ttl: Duration) -> Self { + Self { + name, + cache: LruCache::new(capacity).into(), + ttl, + } + } + + /// Drop an entry from the cache if it's outdated. + #[tracing::instrument(level = "debug", fields(cache = self.name), skip_all)] + fn invalidate_raw(&self, info: &LookupInfo) { + let now = Instant::now(); + + // Do costly things before taking the lock. + let mut cache = self.cache.lock(); + let raw_entry = match cache.raw_entry_mut().from_key(&info.key) { + RawEntryMut::Vacant(_) => return, + RawEntryMut::Occupied(x) => x, + }; + + // Remove the entry if it was created prior to lookup timestamp. + let entry = raw_entry.get(); + let (created_at, expires_at) = (entry.created_at, entry.expires_at); + let should_remove = created_at <= info.created_at || expires_at <= now; + + if should_remove { + raw_entry.remove(); + } + + drop(cache); // drop lock before logging + debug!( + created_at = format_args!("{created_at:?}"), + expires_at = format_args!("{expires_at:?}"), + entry_removed = should_remove, + "processed a cache entry invalidation event" + ); + } + + /// Try retrieving an entry by its key, then execute `extract` if it exists. + #[tracing::instrument(level = "debug", fields(cache = self.name), skip_all)] + fn get_raw(&self, key: &Q, extract: impl FnOnce(&K, &Entry) -> R) -> Option + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let now = Instant::now(); + let deadline = now.checked_add(self.ttl).expect("time overflow"); + + // Do costly things before taking the lock. + let mut cache = self.cache.lock(); + let mut raw_entry = match cache.raw_entry_mut().from_key(key) { + RawEntryMut::Vacant(_) => return None, + RawEntryMut::Occupied(x) => x, + }; + + // Immeditely drop the entry if it has expired. + let entry = raw_entry.get(); + if entry.expires_at <= now { + raw_entry.remove(); + return None; + } + + let value = extract(raw_entry.key(), entry); + let (created_at, expires_at) = (entry.created_at, entry.expires_at); + + // Update the deadline and the entry's position in the LRU list. + raw_entry.get_mut().expires_at = deadline; + raw_entry.to_back(); + + drop(cache); // drop lock before logging + debug!( + created_at = format_args!("{created_at:?}"), + old_expires_at = format_args!("{expires_at:?}"), + new_expires_at = format_args!("{deadline:?}"), + "accessed a cache entry" + ); + + Some(value) + } + + /// Insert an entry to the cache. If an entry with the same key already + /// existed, return the previous value and its creation timestamp. + #[tracing::instrument(level = "debug", fields(cache = self.name), skip_all)] + fn insert_raw(&self, key: K, value: V) -> (Instant, Option) { + let created_at = Instant::now(); + let expires_at = created_at.checked_add(self.ttl).expect("time overflow"); + + let entry = Entry { + created_at, + expires_at, + value, + }; + + // Do costly things before taking the lock. + let old = self + .cache + .lock() + .insert(key, entry) + .map(|entry| entry.value); + + debug!( + created_at = format_args!("{created_at:?}"), + expires_at = format_args!("{expires_at:?}"), + replaced = old.is_some(), + "created a cache entry" + ); + + (created_at, old) + } + } + + impl TimedLru { + pub fn insert(&self, key: K, value: V) -> (Option, Cached<&Self>) { + let (created_at, old) = self.insert_raw(key.clone(), value.clone()); + + let cached = Cached { + token: Some((self, LookupInfo { created_at, key })), + value, + }; + + (old, cached) + } + } + + impl TimedLru { + /// Retrieve a cached entry in convenient wrapper. + pub fn get(&self, key: &Q) -> Option> + where + K: Borrow + Clone, + Q: Hash + Eq + ?Sized, + { + self.get_raw(key, |key, entry| { + let info = LookupInfo { + created_at: entry.created_at, + key: key.clone(), + }; + + Cached { + token: Some((self, info)), + value: entry.value.clone(), + } + }) + } + } + + /// Lookup information for key invalidation. + pub struct LookupInfo { + /// Time of creation of a cache [`Entry`]. + /// We use this during invalidation lookups to prevent eviction of a newer + /// entry sharing the same key (it might've been inserted by a different + /// task after we got the entry we're trying to invalidate now). + created_at: Instant, + + /// Search by this key. + key: K, + } + + /// Wrapper for convenient entry invalidation. + pub struct Cached { + /// Cache + lookup info. + token: Option<(C, C::LookupInfo)>, + + /// The value itself. + pub value: C::Value, + } + + impl Cached { + /// Place any entry into this wrapper; invalidation will be a no-op. + /// Unfortunately, rust doesn't let us implement [`From`] or [`Into`]. + pub fn new_uncached(value: impl Into) -> Self { + Self { + token: None, + value: value.into(), + } + } + + /// Drop this entry from a cache if it's still there. + pub fn invalidate(&self) { + if let Some((cache, info)) = &self.token { + cache.invalidate(info); + } + } + + /// Tell if this entry is actually cached. + pub fn cached(&self) -> bool { + self.token.is_some() + } + } + + impl Deref for Cached { + type Target = C::Value; + + fn deref(&self) -> &Self::Target { + &self.value + } + } + + impl DerefMut for Cached { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.value + } + } +} diff --git a/proxy/src/cancellation.rs b/proxy/src/cancellation.rs index b219cd0fa2..c8c0727471 100644 --- a/proxy/src/cancellation.rs +++ b/proxy/src/cancellation.rs @@ -1,6 +1,5 @@ use anyhow::{anyhow, Context}; use hashbrown::HashMap; -use parking_lot::Mutex; use pq_proto::CancelKeyData; use std::net::SocketAddr; use tokio::net::TcpStream; @@ -9,14 +8,15 @@ use tracing::info; /// Enables serving `CancelRequest`s. #[derive(Default)] -pub struct CancelMap(Mutex>>); +pub struct CancelMap(parking_lot::RwLock>>); impl CancelMap { /// Cancel a running query for the corresponding connection. pub async fn cancel_session(&self, key: CancelKeyData) -> anyhow::Result<()> { + // NB: we should immediately release the lock after cloning the token. let cancel_closure = self .0 - .lock() + .read() .get(&key) .and_then(|x| x.clone()) .with_context(|| format!("query cancellation key not found: {key}"))?; @@ -41,14 +41,14 @@ impl CancelMap { // Random key collisions are unlikely to happen here, but they're still possible, // which is why we have to take care not to rewrite an existing key. self.0 - .lock() + .write() .try_insert(key, None) .map_err(|_| anyhow!("query cancellation key already exists: {key}"))?; // This will guarantee that the session gets dropped // as soon as the future is finished. scopeguard::defer! { - self.0.lock().remove(&key); + self.0.write().remove(&key); info!("dropped query cancellation key {key}"); } @@ -59,12 +59,12 @@ impl CancelMap { #[cfg(test)] fn contains(&self, session: &Session) -> bool { - self.0.lock().contains_key(&session.key) + self.0.read().contains_key(&session.key) } #[cfg(test)] fn is_empty(&self) -> bool { - self.0.lock().is_empty() + self.0.read().is_empty() } } @@ -115,7 +115,7 @@ impl Session<'_> { info!("enabling query cancellation for this session"); self.cancel_map .0 - .lock() + .write() .insert(self.key, Some(cancel_closure)); self.key diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 094db73061..0c0cbcde20 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -42,14 +42,65 @@ pub type ScramKeys = tokio_postgres::config::ScramKeys<32>; /// A config for establishing a connection to compute node. /// Eventually, `tokio_postgres` will be replaced with something better. /// Newtype allows us to implement methods on top of it. +#[derive(Clone)] #[repr(transparent)] pub struct ConnCfg(Box); +/// Creation and initialization routines. impl ConnCfg { - /// Construct a new connection config. pub fn new() -> Self { Self(Default::default()) } + + /// Reuse password or auth keys from the other config. + pub fn reuse_password(&mut self, other: &Self) { + if let Some(password) = other.get_password() { + self.password(password); + } + + if let Some(keys) = other.get_auth_keys() { + self.auth_keys(keys); + } + } + + /// Apply startup message params to the connection config. + pub fn set_startup_params(&mut self, params: &StartupMessageParams) { + if let Some(options) = params.options_raw() { + // We must drop all proxy-specific parameters. + #[allow(unstable_name_collisions)] + let options: String = options + .filter(|opt| !opt.starts_with("project=")) + .intersperse(" ") // TODO: use impl from std once it's stabilized + .collect(); + + self.options(&options); + } + + if let Some(app_name) = params.get("application_name") { + self.application_name(app_name); + } + + // TODO: This is especially ugly... + if let Some(replication) = params.get("replication") { + use tokio_postgres::config::ReplicationMode; + match replication { + "true" | "on" | "yes" | "1" => { + self.replication_mode(ReplicationMode::Physical); + } + "database" => { + self.replication_mode(ReplicationMode::Logical); + } + _other => {} + } + } + + // TODO: extend the list of the forwarded startup parameters. + // Currently, tokio-postgres doesn't allow us to pass + // arbitrary parameters, but the ones above are a good start. + // + // This and the reverse params problem can be better addressed + // in a bespoke connection machinery (a new library for that sake). + } } impl std::ops::Deref for ConnCfg { @@ -132,50 +183,13 @@ pub struct PostgresConnection { pub stream: TcpStream, /// PostgreSQL connection parameters. pub params: std::collections::HashMap, + /// Query cancellation token. + pub cancel_closure: CancelClosure, } impl ConnCfg { /// Connect to a corresponding compute node. - pub async fn connect( - mut self, - params: &StartupMessageParams, - ) -> Result<(PostgresConnection, CancelClosure), ConnectionError> { - if let Some(options) = params.options_raw() { - // We must drop all proxy-specific parameters. - #[allow(unstable_name_collisions)] - let options: String = options - .filter(|opt| !opt.starts_with("project=")) - .intersperse(" ") // TODO: use impl from std once it's stabilized - .collect(); - - self.0.options(&options); - } - - if let Some(app_name) = params.get("application_name") { - self.0.application_name(app_name); - } - - // TODO: This is especially ugly... - if let Some(replication) = params.get("replication") { - use tokio_postgres::config::ReplicationMode; - match replication { - "true" | "on" | "yes" | "1" => { - self.0.replication_mode(ReplicationMode::Physical); - } - "database" => { - self.0.replication_mode(ReplicationMode::Logical); - } - _other => {} - } - } - - // TODO: extend the list of the forwarded startup parameters. - // Currently, tokio-postgres doesn't allow us to pass - // arbitrary parameters, but the ones above are a good start. - // - // This and the reverse params problem can be better addressed - // in a bespoke connection machinery (a new library for that sake). - + pub async fn connect(&self) -> Result { // TODO: establish a secure connection to the DB. let (socket_addr, mut stream) = self.connect_raw().await?; let (client, connection) = self.0.connect_raw(&mut stream, NoTls).await?; @@ -189,8 +203,13 @@ impl ConnCfg { // NB: CancelToken is supposed to hold socket_addr, but we use connect_raw. // Yet another reason to rework the connection establishing code. let cancel_closure = CancelClosure::new(socket_addr, client.cancel_token()); - let db = PostgresConnection { stream, params }; - Ok((db, cancel_closure)) + let connection = PostgresConnection { + stream, + params, + cancel_closure, + }; + + Ok(connection) } } diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 33a8fff847..5e285f3625 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -1,16 +1,16 @@ use crate::auth; -use anyhow::{ensure, Context}; -use std::sync::Arc; +use anyhow::{bail, ensure, Context}; +use std::{str::FromStr, sync::Arc, time::Duration}; pub struct ProxyConfig { pub tls_config: Option, pub auth_backend: auth::BackendType<'static, ()>, - pub metric_collection_config: Option, + pub metric_collection: Option, } pub struct MetricCollectionConfig { pub endpoint: reqwest::Url, - pub interval: std::time::Duration, + pub interval: Duration, } pub struct TlsConfig { @@ -37,6 +37,7 @@ pub fn configure_tls(key_path: &str, cert_path: &str) -> anyhow::Result anyhow::Result anyhow::Result { + let mut size = None; + let mut ttl = None; + + for option in options.split(',') { + let (key, value) = option + .split_once('=') + .with_context(|| format!("bad key-value pair: {option}"))?; + + match key { + "size" => size = Some(value.parse()?), + "ttl" => ttl = Some(humantime::parse_duration(value)?), + unknown => bail!("unknown key: {unknown}"), + } + } + + // TTL doesn't matter if cache is always empty. + if let Some(0) = size { + ttl.get_or_insert(Duration::default()); + } + + Ok(Self { + size: size.context("missing `size`")?, + ttl: ttl.context("missing `ttl`")?, + }) + } +} + +impl FromStr for CacheOptions { + type Err = anyhow::Error; + + fn from_str(options: &str) -> Result { + let error = || format!("failed to parse cache options '{options}'"); + Self::parse(options).with_context(error) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_cache_options() -> anyhow::Result<()> { + let CacheOptions { size, ttl } = "size=4096,ttl=5min".parse()?; + assert_eq!(size, 4096); + assert_eq!(ttl, Duration::from_secs(5 * 60)); + + let CacheOptions { size, ttl } = "ttl=4m,size=2".parse()?; + assert_eq!(size, 2); + assert_eq!(ttl, Duration::from_secs(4 * 60)); + + let CacheOptions { size, ttl } = "size=0,ttl=1s".parse()?; + assert_eq!(size, 0); + assert_eq!(ttl, Duration::from_secs(1)); + + let CacheOptions { size, ttl } = "size=0".parse()?; + assert_eq!(size, 0); + assert_eq!(ttl, Duration::default()); + + Ok(()) + } +} diff --git a/proxy/src/console.rs b/proxy/src/console.rs index 78f09ac9e1..1f3ef99555 100644 --- a/proxy/src/console.rs +++ b/proxy/src/console.rs @@ -3,3 +3,15 @@ /// Payloads used in the console's APIs. pub mod messages; + +/// Wrappers for console APIs and their mocks. +pub mod provider; +pub use provider::{errors, Api, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo}; + +/// Various cache-related types. +pub mod caches { + pub use super::provider::{ApiCaches, NodeInfoCache}; +} + +/// Console's management API. +pub mod mgmt; diff --git a/proxy/src/console/messages.rs b/proxy/src/console/messages.rs index 63a97069b8..0d321c077a 100644 --- a/proxy/src/console/messages.rs +++ b/proxy/src/console/messages.rs @@ -63,13 +63,13 @@ impl KickSession<'_> { /// Compute node connection params. #[derive(Deserialize)] pub struct DatabaseInfo { - pub host: String, + pub host: Box, pub port: u16, - pub dbname: String, - pub user: String, + pub dbname: Box, + pub user: Box, /// Console always provides a password, but it might /// be inconvenient for debug with local PG instance. - pub password: Option, + pub password: Option>, pub aux: MetricsAuxInfo, } diff --git a/proxy/src/mgmt.rs b/proxy/src/console/mgmt.rs similarity index 79% rename from proxy/src/mgmt.rs rename to proxy/src/console/mgmt.rs index cf83b48ae0..51a117d3b7 100644 --- a/proxy/src/mgmt.rs +++ b/proxy/src/console/mgmt.rs @@ -1,8 +1,9 @@ use crate::{ - auth, console::messages::{DatabaseInfo, KickSession}, + waiters::{self, Waiter, Waiters}, }; use anyhow::Context; +use once_cell::sync::Lazy; use pq_proto::{BeMessage, SINGLE_COL_ROWDESC}; use std::{ net::{TcpListener, TcpStream}, @@ -14,6 +15,25 @@ use utils::{ postgres_backend_async::QueryError, }; +static CPLANE_WAITERS: Lazy> = Lazy::new(Default::default); + +/// Give caller an opportunity to wait for the cloud's reply. +pub async fn with_waiter( + psql_session_id: impl Into, + action: impl FnOnce(Waiter<'static, ComputeReady>) -> R, +) -> Result +where + R: std::future::Future>, + E: From, +{ + let waiter = CPLANE_WAITERS.register(psql_session_id.into())?; + action(waiter).await +} + +pub fn notify(psql_session_id: &str, msg: ComputeReady) -> Result<(), waiters::NotifyError> { + CPLANE_WAITERS.notify(psql_session_id, msg) +} + /// Console management API listener thread. /// It spawns console response handlers needed for the link auth. pub fn thread_main(listener: TcpListener) -> anyhow::Result<()> { @@ -76,7 +96,7 @@ fn try_process_query(pgb: &mut PostgresBackend, query: &str) -> Result<(), Query let _enter = span.enter(); info!("got response: {:?}", resp.result); - match auth::backend::notify(resp.session_id, Ok(resp.result)) { + match notify(resp.session_id, Ok(resp.result)) { Ok(()) => { pgb.write_message_noflush(&SINGLE_COL_ROWDESC)? .write_message_noflush(&BeMessage::DataRow(&[Some(b"ok")]))? diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs new file mode 100644 index 0000000000..7621aba19b --- /dev/null +++ b/proxy/src/console/provider.rs @@ -0,0 +1,194 @@ +pub mod mock; +pub mod neon; + +use super::messages::MetricsAuxInfo; +use crate::{ + auth::ClientCredentials, + cache::{timed_lru, TimedLru}, + compute, scram, +}; +use async_trait::async_trait; +use std::sync::Arc; + +pub mod errors { + use crate::error::{io_error, UserFacingError}; + use reqwest::StatusCode as HttpStatusCode; + use thiserror::Error; + + /// A go-to error message which doesn't leak any detail. + const REQUEST_FAILED: &str = "Console request failed"; + + /// Common console API error. + #[derive(Debug, Error)] + pub enum ApiError { + /// Error returned by the console itself. + #[error("{REQUEST_FAILED} with {}: {}", .status, .text)] + Console { + status: HttpStatusCode, + text: Box, + }, + + /// Various IO errors like broken pipe or malformed payload. + #[error("{REQUEST_FAILED}: {0}")] + Transport(#[from] std::io::Error), + } + + impl ApiError { + /// Returns HTTP status code if it's the reason for failure. + pub fn http_status_code(&self) -> Option { + use ApiError::*; + match self { + Console { status, .. } => Some(*status), + _ => None, + } + } + } + + impl UserFacingError for ApiError { + fn to_string_client(&self) -> String { + use ApiError::*; + match self { + // To minimize risks, only select errors are forwarded to users. + // Ask @neondatabase/control-plane for review before adding more. + Console { status, .. } => match *status { + HttpStatusCode::NOT_FOUND => { + // Status 404: failed to get a project-related resource. + format!("{REQUEST_FAILED}: endpoint cannot be found") + } + HttpStatusCode::NOT_ACCEPTABLE => { + // Status 406: endpoint is disabled (we don't allow connections). + format!("{REQUEST_FAILED}: endpoint is disabled") + } + HttpStatusCode::LOCKED => { + // Status 423: project might be in maintenance mode (or bad state). + format!("{REQUEST_FAILED}: endpoint is temporary unavailable") + } + _ => REQUEST_FAILED.to_owned(), + }, + _ => REQUEST_FAILED.to_owned(), + } + } + } + + // Helps eliminate graceless `.map_err` calls without introducing another ctor. + impl From for ApiError { + fn from(e: reqwest::Error) -> Self { + io_error(e).into() + } + } + + #[derive(Debug, Error)] + pub enum GetAuthInfoError { + // We shouldn't include the actual secret here. + #[error("Console responded with a malformed auth secret")] + BadSecret, + + #[error(transparent)] + ApiError(ApiError), + } + + // This allows more useful interactions than `#[from]`. + impl> From for GetAuthInfoError { + fn from(e: E) -> Self { + Self::ApiError(e.into()) + } + } + + impl UserFacingError for GetAuthInfoError { + fn to_string_client(&self) -> String { + use GetAuthInfoError::*; + match self { + // We absolutely should not leak any secrets! + BadSecret => REQUEST_FAILED.to_owned(), + // However, API might return a meaningful error. + ApiError(e) => e.to_string_client(), + } + } + } + #[derive(Debug, Error)] + pub enum WakeComputeError { + #[error("Console responded with a malformed compute address: {0}")] + BadComputeAddress(Box), + + #[error(transparent)] + ApiError(ApiError), + } + + // This allows more useful interactions than `#[from]`. + impl> From for WakeComputeError { + fn from(e: E) -> Self { + Self::ApiError(e.into()) + } + } + + impl UserFacingError for WakeComputeError { + fn to_string_client(&self) -> String { + use WakeComputeError::*; + match self { + // We shouldn't show user the address even if it's broken. + // Besides, user is unlikely to care about this detail. + BadComputeAddress(_) => REQUEST_FAILED.to_owned(), + // However, API might return a meaningful error. + ApiError(e) => e.to_string_client(), + } + } + } +} + +/// Extra query params we'd like to pass to the console. +pub struct ConsoleReqExtra<'a> { + /// A unique identifier for a connection. + pub session_id: uuid::Uuid, + /// Name of client application, if set. + pub application_name: Option<&'a str>, +} + +/// Auth secret which is managed by the cloud. +pub enum AuthInfo { + /// Md5 hash of user's password. + Md5([u8; 16]), + + /// [SCRAM](crate::scram) authentication info. + Scram(scram::ServerSecret), +} + +/// Info for establishing a connection to a compute node. +/// This is what we get after auth succeeded, but not before! +#[derive(Clone)] +pub struct NodeInfo { + /// Compute node connection params. + /// It's sad that we have to clone this, but this will improve + /// once we migrate to a bespoke connection logic. + pub config: compute::ConnCfg, + + /// Labels for proxy's metrics. + pub aux: Arc, +} + +pub type NodeInfoCache = TimedLru, NodeInfo>; +pub type CachedNodeInfo = timed_lru::Cached<&'static NodeInfoCache>; + +/// This will allocate per each call, but the http requests alone +/// already require a few allocations, so it should be fine. +#[async_trait] +pub trait Api { + /// Get the client's auth secret for authentication. + async fn get_auth_info( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result, errors::GetAuthInfoError>; + + /// Wake up the compute node and return the corresponding connection info. + async fn wake_compute( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result; +} + +/// Various caches for [`console`]. +pub struct ApiCaches { + /// Cache for the `wake_compute` API method. + pub node_info: NodeInfoCache, +} diff --git a/proxy/src/auth/backend/postgres.rs b/proxy/src/console/provider/mock.rs similarity index 53% rename from proxy/src/auth/backend/postgres.rs rename to proxy/src/console/provider/mock.rs index 260342f103..301c3be516 100644 --- a/proxy/src/auth/backend/postgres.rs +++ b/proxy/src/console/provider/mock.rs @@ -1,21 +1,14 @@ -//! Local mock of Cloud API V2. +//! Mock console backend which relies on a user-provided postgres instance. use super::{ - console::{self, AuthInfo, GetAuthInfoError, WakeComputeError}, - AuthSuccess, NodeInfo, -}; -use crate::{ - auth::{self, ClientCredentials}, - compute, - error::io_error, - scram, - stream::PqStream, - url::ApiUrl, + errors::{ApiError, GetAuthInfoError, WakeComputeError}, + AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, }; +use crate::{auth::ClientCredentials, compute, error::io_error, scram, url::ApiUrl}; +use async_trait::async_trait; use futures::TryFutureExt; use thiserror::Error; -use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::{info, info_span, warn, Instrument}; +use tracing::{error, info, info_span, warn, Instrument}; #[derive(Debug, Error)] enum MockApiError { @@ -23,49 +16,36 @@ enum MockApiError { PasswordNotSet(tokio_postgres::Error), } -impl From for console::ApiError { +impl From for ApiError { fn from(e: MockApiError) -> Self { io_error(e).into() } } -impl From for console::ApiError { +impl From for ApiError { fn from(e: tokio_postgres::Error) -> Self { io_error(e).into() } } -#[must_use] -pub(super) struct Api<'a> { - endpoint: &'a ApiUrl, - creds: &'a ClientCredentials<'a>, +#[derive(Clone)] +pub struct Api { + endpoint: ApiUrl, } -impl<'a> AsRef> for Api<'a> { - fn as_ref(&self) -> &ClientCredentials<'a> { - self.creds - } -} - -impl<'a> Api<'a> { - /// Construct an API object containing the auth parameters. - pub(super) fn new(endpoint: &'a ApiUrl, creds: &'a ClientCredentials) -> Self { - Self { endpoint, creds } +impl Api { + pub fn new(endpoint: ApiUrl) -> Self { + Self { endpoint } } - /// Authenticate the existing user or throw an error. - pub(super) async fn handle_user( - &'a self, - client: &mut PqStream, - ) -> auth::Result> { - // We reuse user handling logic from a production module. - console::handle_user(client, self, Self::get_auth_info, Self::wake_compute).await + pub fn url(&self) -> &str { + self.endpoint.as_str() } -} -impl Api<'_> { - /// This implementation fetches the auth info from a local postgres instance. - async fn get_auth_info(&self) -> Result, GetAuthInfoError> { + async fn do_get_auth_info( + &self, + creds: &ClientCredentials<'_>, + ) -> Result, GetAuthInfoError> { async { // Perhaps we could persist this connection, but then we'd have to // write more code for reopening it if it got closed, which doesn't @@ -75,7 +55,7 @@ impl Api<'_> { tokio::spawn(connection); let query = "select rolpassword from pg_catalog.pg_authid where rolname = $1"; - let rows = client.query(query, &[&self.creds.user]).await?; + let rows = client.query(query, &[&creds.user]).await?; // We can get at most one row, because `rolname` is unique. let row = match rows.get(0) { @@ -84,7 +64,7 @@ impl Api<'_> { // However, this is still a *valid* outcome which is very similar // to getting `404 Not found` from the Neon console. None => { - warn!("user '{}' does not exist", self.creds.user); + warn!("user '{}' does not exist", creds.user); return Ok(None); } }; @@ -98,23 +78,50 @@ impl Api<'_> { Ok(secret.or_else(|| parse_md5(entry).map(AuthInfo::Md5))) } .map_err(crate::error::log_error) - .instrument(info_span!("get_auth_info", mock = self.endpoint.as_str())) + .instrument(info_span!("postgres", url = self.endpoint.as_str())) .await } - /// We don't need to wake anything locally, so we just return the connection info. - pub async fn wake_compute(&self) -> Result { + async fn do_wake_compute( + &self, + creds: &ClientCredentials<'_>, + ) -> Result { let mut config = compute::ConnCfg::new(); config .host(self.endpoint.host_str().unwrap_or("localhost")) .port(self.endpoint.port().unwrap_or(5432)) - .dbname(self.creds.dbname) - .user(self.creds.user); + .dbname(creds.dbname) + .user(creds.user); - Ok(NodeInfo { + let node = NodeInfo { config, aux: Default::default(), - }) + }; + + Ok(node) + } +} + +#[async_trait] +impl super::Api for Api { + #[tracing::instrument(skip_all)] + async fn get_auth_info( + &self, + _extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result, GetAuthInfoError> { + self.do_get_auth_info(creds).await + } + + #[tracing::instrument(skip_all)] + async fn wake_compute( + &self, + _extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result { + self.do_wake_compute(creds) + .map_ok(CachedNodeInfo::new_uncached) + .await } } diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs new file mode 100644 index 0000000000..00d3ca8352 --- /dev/null +++ b/proxy/src/console/provider/neon.rs @@ -0,0 +1,196 @@ +//! Production console backend. + +use super::{ + super::messages::{ConsoleError, GetRoleSecret, WakeCompute}, + errors::{ApiError, GetAuthInfoError, WakeComputeError}, + ApiCaches, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, +}; +use crate::{auth::ClientCredentials, compute, http, scram}; +use async_trait::async_trait; +use futures::TryFutureExt; +use reqwest::StatusCode as HttpStatusCode; +use tracing::{error, info, info_span, warn, Instrument}; + +#[derive(Clone)] +pub struct Api { + endpoint: http::Endpoint, + caches: &'static ApiCaches, +} + +impl Api { + /// Construct an API object containing the auth parameters. + pub fn new(endpoint: http::Endpoint, caches: &'static ApiCaches) -> Self { + Self { endpoint, caches } + } + + pub fn url(&self) -> &str { + self.endpoint.url().as_str() + } + + async fn do_get_auth_info( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result, GetAuthInfoError> { + let request_id = uuid::Uuid::new_v4().to_string(); + async { + let request = self + .endpoint + .get("proxy_get_role_secret") + .header("X-Request-ID", &request_id) + .query(&[("session_id", extra.session_id)]) + .query(&[ + ("application_name", extra.application_name), + ("project", Some(creds.project().expect("impossible"))), + ("role", Some(creds.user)), + ]) + .build()?; + + info!(url = request.url().as_str(), "sending http request"); + let response = self.endpoint.execute(request).await?; + let body = match parse_body::(response).await { + Ok(body) => body, + // Error 404 is special: it's ok not to have a secret. + Err(e) => match e.http_status_code() { + Some(HttpStatusCode::NOT_FOUND) => return Ok(None), + _otherwise => return Err(e.into()), + }, + }; + + let secret = scram::ServerSecret::parse(&body.role_secret) + .map(AuthInfo::Scram) + .ok_or(GetAuthInfoError::BadSecret)?; + + Ok(Some(secret)) + } + .map_err(crate::error::log_error) + .instrument(info_span!("http", id = request_id)) + .await + } + + async fn do_wake_compute( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result { + let project = creds.project().expect("impossible"); + let request_id = uuid::Uuid::new_v4().to_string(); + async { + let request = self + .endpoint + .get("proxy_wake_compute") + .header("X-Request-ID", &request_id) + .query(&[("session_id", extra.session_id)]) + .query(&[ + ("application_name", extra.application_name), + ("project", Some(project)), + ]) + .build()?; + + info!(url = request.url().as_str(), "sending http request"); + let response = self.endpoint.execute(request).await?; + let body = parse_body::(response).await?; + + // Unfortunately, ownership won't let us use `Option::ok_or` here. + let (host, port) = match parse_host_port(&body.address) { + None => return Err(WakeComputeError::BadComputeAddress(body.address)), + Some(x) => x, + }; + + let mut config = compute::ConnCfg::new(); + config + .host(host) + .port(port) + .dbname(creds.dbname) + .user(creds.user); + + let node = NodeInfo { + config, + aux: body.aux.into(), + }; + + Ok(node) + } + .map_err(crate::error::log_error) + .instrument(info_span!("http", id = request_id)) + .await + } +} + +#[async_trait] +impl super::Api for Api { + #[tracing::instrument(skip_all)] + async fn get_auth_info( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result, GetAuthInfoError> { + self.do_get_auth_info(extra, creds).await + } + + #[tracing::instrument(skip_all)] + async fn wake_compute( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials<'_>, + ) -> Result { + let key = creds.project().expect("impossible"); + + // Every time we do a wakeup http request, the compute node will stay up + // for some time (highly depends on the console's scale-to-zero policy); + // The connection info remains the same during that period of time, + // which means that we might cache it to reduce the load and latency. + if let Some(cached) = self.caches.node_info.get(key) { + info!(key = key, "found cached compute node info"); + return Ok(cached); + } + + let node = self.do_wake_compute(extra, creds).await?; + let (_, cached) = self.caches.node_info.insert(key.into(), node); + info!(key = key, "created a cache entry for compute node info"); + + Ok(cached) + } +} + +/// Parse http response body, taking status code into account. +async fn parse_body serde::Deserialize<'a>>( + response: reqwest::Response, +) -> Result { + let status = response.status(); + if status.is_success() { + // We shouldn't log raw body because it may contain secrets. + info!("request succeeded, processing the body"); + return Ok(response.json().await?); + } + + // Don't throw an error here because it's not as important + // as the fact that the request itself has failed. + let body = response.json().await.unwrap_or_else(|e| { + warn!("failed to parse error body: {e}"); + ConsoleError { + error: "reason unclear (malformed error message)".into(), + } + }); + + let text = body.error; + error!("console responded with an error ({status}): {text}"); + Err(ApiError::Console { status, text }) +} + +fn parse_host_port(input: &str) -> Option<(&str, u16)> { + let (host, port) = input.split_once(':')?; + Some((host, port.parse().ok()?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_host_port() { + let (host, port) = parse_host_port("127.0.0.1:5432").expect("failed to parse"); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 5432); + } +} diff --git a/proxy/src/http/server.rs b/proxy/src/http/server.rs index 05f6feb307..f35f4f9a62 100644 --- a/proxy/src/http/server.rs +++ b/proxy/src/http/server.rs @@ -9,8 +9,7 @@ async fn status_handler(_: Request) -> Result, ApiError> { } fn make_router() -> RouterBuilder { - let router = endpoint::make_router(); - router.get("/v1/status", status_handler) + endpoint::make_router().get("/v1/status", status_handler) } pub async fn task_main(http_listener: TcpListener) -> anyhow::Result<()> { diff --git a/proxy/src/http/websocket.rs b/proxy/src/http/websocket.rs index d43602e833..bedded7567 100644 --- a/proxy/src/http/websocket.rs +++ b/proxy/src/http/websocket.rs @@ -1,6 +1,6 @@ use bytes::{Buf, Bytes}; use futures::{Sink, Stream, StreamExt}; -use hyper::server::accept::{self}; +use hyper::server::accept; use hyper::server::conn::AddrIncoming; use hyper::upgrade::Upgraded; use hyper::{Body, Request, Response, StatusCode}; @@ -161,7 +161,7 @@ impl AsyncBufRead for WebSocketRW { async fn serve_websocket( websocket: HyperWebsocket, - config: &ProxyConfig, + config: &'static ProxyConfig, cancel_map: &CancelMap, session_id: uuid::Uuid, hostname: Option, diff --git a/proxy/src/main.rs b/proxy/src/main.rs index 1b61ab108f..c96ca2a171 100644 --- a/proxy/src/main.rs +++ b/proxy/src/main.rs @@ -5,6 +5,7 @@ //! in somewhat transparent manner (again via communication with control plane API). mod auth; +mod cache; mod cancellation; mod compute; mod config; @@ -12,7 +13,6 @@ mod console; mod error; mod http; mod metrics; -mod mgmt; mod parse; mod proxy; mod sasl; @@ -21,7 +21,6 @@ mod stream; mod url; mod waiters; -use ::metrics::set_build_info_metric; use anyhow::{bail, Context}; use clap::{self, Arg}; use config::ProxyConfig; @@ -29,8 +28,7 @@ use futures::FutureExt; use std::{borrow::Cow, future::Future, net::SocketAddr}; use tokio::{net::TcpListener, task::JoinError}; use tracing::{info, info_span, Instrument}; -use utils::project_git_version; -use utils::sentry_init::init_sentry; +use utils::{project_git_version, sentry_init::init_sentry}; project_git_version!(GIT_VERSION); @@ -51,124 +49,133 @@ async fn main() -> anyhow::Result<()> { // initialize sentry if SENTRY_DSN is provided let _sentry_guard = init_sentry(Some(GIT_VERSION.into()), &[]); - let arg_matches = cli().get_matches(); - - let tls_config = match ( - arg_matches.get_one::("tls-key"), - arg_matches.get_one::("tls-cert"), - ) { - (Some(key_path), Some(cert_path)) => Some(config::configure_tls(key_path, cert_path)?), - (None, None) => None, - _ => bail!("either both or neither tls-key and tls-cert must be specified"), - }; - - let proxy_address: SocketAddr = arg_matches.get_one::("proxy").unwrap().parse()?; - let mgmt_address: SocketAddr = arg_matches.get_one::("mgmt").unwrap().parse()?; - let http_address: SocketAddr = arg_matches.get_one::("http").unwrap().parse()?; - - let metric_collection_config = match - ( - arg_matches.get_one::("metric-collection-endpoint"), - arg_matches.get_one::("metric-collection-interval"), - ) { - - (Some(endpoint), Some(interval)) => { - Some(config::MetricCollectionConfig { - endpoint: endpoint.parse()?, - interval: humantime::parse_duration(interval)?, - }) - } - (None, None) => None, - _ => bail!("either both or neither metric-collection-endpoint and metric-collection-interval must be specified"), - }; - - let auth_backend = match arg_matches - .get_one::("auth-backend") - .unwrap() - .as_str() - { - "console" => { - let url = arg_matches - .get_one::("auth-endpoint") - .unwrap() - .parse()?; - let endpoint = http::Endpoint::new(url, reqwest::Client::new()); - auth::BackendType::Console(Cow::Owned(endpoint), ()) - } - "postgres" => { - let url = arg_matches - .get_one::("auth-endpoint") - .unwrap() - .parse()?; - auth::BackendType::Postgres(Cow::Owned(url), ()) - } - "link" => { - let url = arg_matches.get_one::("uri").unwrap().parse()?; - auth::BackendType::Link(Cow::Owned(url)) - } - other => bail!("unsupported auth backend: {other}"), - }; - - let config: &ProxyConfig = Box::leak(Box::new(ProxyConfig { - tls_config, - auth_backend, - metric_collection_config, - })); - info!("Version: {GIT_VERSION}"); + ::metrics::set_build_info_metric(GIT_VERSION); + + let args = cli().get_matches(); + let config = build_config(&args)?; + info!("Authentication backend: {}", config.auth_backend); // Check that we can bind to address before further initialization + let http_address: SocketAddr = args.get_one::("http").unwrap().parse()?; info!("Starting http on {http_address}"); let http_listener = TcpListener::bind(http_address).await?.into_std()?; + let mgmt_address: SocketAddr = args.get_one::("mgmt").unwrap().parse()?; info!("Starting mgmt on {mgmt_address}"); let mgmt_listener = TcpListener::bind(mgmt_address).await?.into_std()?; + let proxy_address: SocketAddr = args.get_one::("proxy").unwrap().parse()?; info!("Starting proxy on {proxy_address}"); let proxy_listener = TcpListener::bind(proxy_address).await?; let mut tasks = vec![ tokio::spawn(http::server::task_main(http_listener)), tokio::spawn(proxy::task_main(config, proxy_listener)), - tokio::task::spawn_blocking(move || mgmt::thread_main(mgmt_listener)), + tokio::task::spawn_blocking(move || console::mgmt::thread_main(mgmt_listener)), ]; - if let Some(wss_address) = arg_matches.get_one::("wss") { + if let Some(wss_address) = args.get_one::("wss") { let wss_address: SocketAddr = wss_address.parse()?; - info!("Starting wss on {}", wss_address); + info!("Starting wss on {wss_address}"); let wss_listener = TcpListener::bind(wss_address).await?; + tasks.push(tokio::spawn(http::websocket::task_main( wss_listener, config, ))); } - if let Some(metric_collection_config) = &config.metric_collection_config { + // TODO: refactor. + if let Some(metric_collection) = &config.metric_collection { let hostname = hostname::get()? .into_string() .map_err(|e| anyhow::anyhow!("failed to get hostname {e:?}"))?; tasks.push(tokio::spawn( metrics::collect_metrics( - &metric_collection_config.endpoint, - metric_collection_config.interval, + &metric_collection.endpoint, + metric_collection.interval, hostname, ) .instrument(info_span!("collect_metrics")), )); } - let tasks = tasks.into_iter().map(flatten_err); - - set_build_info_metric(GIT_VERSION); // This will block until all tasks have completed. // Furthermore, the first one to fail will cancel the rest. + let tasks = tasks.into_iter().map(flatten_err); let _: Vec<()> = futures::future::try_join_all(tasks).await?; Ok(()) } +/// ProxyConfig is created at proxy startup, and lives forever. +fn build_config(args: &clap::ArgMatches) -> anyhow::Result<&'static ProxyConfig> { + let tls_config = match ( + args.get_one::("tls-key"), + args.get_one::("tls-cert"), + ) { + (Some(key_path), Some(cert_path)) => Some(config::configure_tls(key_path, cert_path)?), + (None, None) => None, + _ => bail!("either both or neither tls-key and tls-cert must be specified"), + }; + + let metric_collection = match ( + args.get_one::("metric-collection-endpoint"), + args.get_one::("metric-collection-interval"), + ) { + (Some(endpoint), Some(interval)) => Some(config::MetricCollectionConfig { + endpoint: endpoint.parse()?, + interval: humantime::parse_duration(interval)?, + }), + (None, None) => None, + _ => bail!( + "either both or neither metric-collection-endpoint \ + and metric-collection-interval must be specified" + ), + }; + + let auth_backend = match args.get_one::("auth-backend").unwrap().as_str() { + "console" => { + let config::CacheOptions { size, ttl } = args + .get_one::("wake-compute-cache") + .unwrap() + .parse()?; + + info!("Using NodeInfoCache (wake_compute) with size={size} ttl={ttl:?}"); + let caches = Box::leak(Box::new(console::caches::ApiCaches { + node_info: console::caches::NodeInfoCache::new("node_info_cache", size, ttl), + })); + + let url = args.get_one::("auth-endpoint").unwrap().parse()?; + let endpoint = http::Endpoint::new(url, reqwest::Client::new()); + + let api = console::provider::neon::Api::new(endpoint, caches); + auth::BackendType::Console(Cow::Owned(api), ()) + } + "postgres" => { + let url = args.get_one::("auth-endpoint").unwrap().parse()?; + let api = console::provider::mock::Api::new(url); + auth::BackendType::Postgres(Cow::Owned(api), ()) + } + "link" => { + let url = args.get_one::("uri").unwrap().parse()?; + auth::BackendType::Link(Cow::Owned(url)) + } + other => bail!("unsupported auth backend: {other}"), + }; + + let config = Box::leak(Box::new(ProxyConfig { + tls_config, + auth_backend, + metric_collection, + })); + + Ok(config) +} + fn cli() -> clap::Command { clap::Command::new("Neon proxy/router") .disable_help_flag(true) @@ -235,16 +242,27 @@ fn cli() -> clap::Command { .arg( Arg::new("metric-collection-endpoint") .long("metric-collection-endpoint") - .help("metric collection HTTP endpoint"), + .help("http endpoint to receive periodic metric updates"), ) .arg( Arg::new("metric-collection-interval") .long("metric-collection-interval") - .help("metric collection interval"), + .help("how often metrics should be sent to a collection endpoint"), + ) + .arg( + Arg::new("wake-compute-cache") + .long("wake-compute-cache") + .help("cache for `wake_compute` api method (use `size=0` to disable)") + .default_value(config::CacheOptions::DEFAULT_OPTIONS_NODE_INFO), ) } -#[test] -fn verify_cli() { - cli().debug_assert(); +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn verify_cli() { + cli().debug_assert(); + } } diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 63573d49c0..a622a35e6d 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -2,9 +2,12 @@ mod tests; use crate::{ - auth, + auth::{self, backend::AuthSuccess}, cancellation::{self, CancelMap}, + compute::{self, PostgresConnection}, config::{ProxyConfig, TlsConfig}, + console::{self, messages::MetricsAuxInfo}, + error::io_error, stream::{MeasuredStream, PqStream, Stream}, }; use anyhow::{bail, Context}; @@ -14,7 +17,10 @@ use once_cell::sync::Lazy; use pq_proto::{BeMessage as Be, FeStartupPacket, StartupMessageParams}; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::{error, info, info_span, Instrument}; +use tracing::{error, info, info_span, warn, Instrument}; + +/// Number of times we should retry the `/proxy_wake_compute` http request. +const NUM_RETRIES_WAKE_COMPUTE: usize = 1; const ERR_INSECURE_CONNECTION: &str = "connection is insecure (try using `sslmode=require`)"; const ERR_PROTO_VIOLATION: &str = "protocol violation"; @@ -35,6 +41,15 @@ static NUM_CONNECTIONS_CLOSED_COUNTER: Lazy = Lazy::new(|| { .unwrap() }); +static NUM_CONNECTION_FAILURES: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "proxy_connection_failures_total", + "Number of connection failures (per kind).", + &["kind"], + ) + .unwrap() +}); + static NUM_BYTES_PROXIED_COUNTER: Lazy = Lazy::new(|| { register_int_counter_vec!( "proxy_io_bytes_per_client", @@ -82,11 +97,12 @@ pub async fn task_main( } } +// TODO(tech debt): unite this with its twin below. pub async fn handle_ws_client( - config: &ProxyConfig, + config: &'static ProxyConfig, cancel_map: &CancelMap, session_id: uuid::Uuid, - stream: impl AsyncRead + AsyncWrite + Unpin + Send, + stream: impl AsyncRead + AsyncWrite + Unpin, hostname: Option, ) -> anyhow::Result<()> { // The `closed` counter will increase when this future is destroyed. @@ -99,7 +115,7 @@ pub async fn handle_ws_client( let hostname = hostname.as_deref(); // TLS is None here, because the connection is already encrypted. - let do_handshake = handshake(stream, None, cancel_map).instrument(info_span!("handshake")); + let do_handshake = handshake(stream, None, cancel_map); let (mut stream, params) = match do_handshake.await? { Some(x) => x, None => return Ok(()), // it's a cancellation request @@ -124,10 +140,10 @@ pub async fn handle_ws_client( } async fn handle_client( - config: &ProxyConfig, + config: &'static ProxyConfig, cancel_map: &CancelMap, session_id: uuid::Uuid, - stream: impl AsyncRead + AsyncWrite + Unpin + Send, + stream: impl AsyncRead + AsyncWrite + Unpin, ) -> anyhow::Result<()> { // The `closed` counter will increase when this future is destroyed. NUM_CONNECTIONS_ACCEPTED_COUNTER.inc(); @@ -136,7 +152,7 @@ async fn handle_client( } let tls = config.tls_config.as_ref(); - let do_handshake = handshake(stream, tls, cancel_map).instrument(info_span!("handshake")); + let do_handshake = handshake(stream, tls, cancel_map); let (mut stream, params) = match do_handshake.await? { Some(x) => x, None => return Ok(()), // it's a cancellation request @@ -165,6 +181,7 @@ async fn handle_client( /// For better testing experience, `stream` can be any object satisfying the traits. /// It's easier to work with owned `stream` here as we need to upgrade it to TLS; /// we also take an extra care of propagating only the select handshake errors to client. +#[tracing::instrument(skip_all)] async fn handshake( stream: S, mut tls: Option<&TlsConfig>, @@ -226,6 +243,133 @@ async fn handshake( } } +/// Try to connect to the compute node once. +#[tracing::instrument(name = "connect_once", skip_all)] +async fn connect_to_compute_once( + node_info: &console::CachedNodeInfo, +) -> Result { + // If we couldn't connect, a cached connection info might be to blame + // (e.g. the compute node's address might've changed at the wrong time). + // Invalidate the cache entry (if any) to prevent subsequent errors. + let invalidate_cache = |_: &compute::ConnectionError| { + let is_cached = node_info.cached(); + if is_cached { + warn!("invalidating stalled compute node info cache entry"); + node_info.invalidate(); + } + + let label = match is_cached { + true => "compute_cached", + false => "compute_uncached", + }; + NUM_CONNECTION_FAILURES.with_label_values(&[label]).inc(); + }; + + node_info + .config + .connect() + .inspect_err(invalidate_cache) + .await +} + +/// Try to connect to the compute node, retrying if necessary. +/// This function might update `node_info`, so we take it by `&mut`. +#[tracing::instrument(skip_all)] +async fn connect_to_compute( + node_info: &mut console::CachedNodeInfo, + params: &StartupMessageParams, + extra: &console::ConsoleReqExtra<'_>, + creds: &auth::BackendType<'_, auth::ClientCredentials<'_>>, +) -> Result { + let mut num_retries: usize = NUM_RETRIES_WAKE_COMPUTE; + loop { + // Apply startup params to the (possibly, cached) compute node info. + node_info.config.set_startup_params(params); + match connect_to_compute_once(node_info).await { + Err(e) if num_retries > 0 => { + info!("compute node's state has changed; requesting a wake-up"); + match creds.wake_compute(extra).map_err(io_error).await? { + // Update `node_info` and try one more time. + Some(mut new) => { + new.config.reuse_password(&node_info.config); + *node_info = new; + } + // Link auth doesn't work that way, so we just exit. + None => return Err(e), + } + } + other => return other, + } + + num_retries -= 1; + info!("retrying after wake-up ({num_retries} attempts left)"); + } +} + +/// Finish client connection initialization: confirm auth success, send params, etc. +#[tracing::instrument(skip_all)] +async fn prepare_client_connection( + node: &compute::PostgresConnection, + reported_auth_ok: bool, + session: cancellation::Session<'_>, + stream: &mut PqStream, +) -> anyhow::Result<()> { + // Register compute's query cancellation token and produce a new, unique one. + // The new token (cancel_key_data) will be sent to the client. + let cancel_key_data = session.enable_query_cancellation(node.cancel_closure.clone()); + + // Report authentication success if we haven't done this already. + // Note that we do this only (for the most part) after we've connected + // to a compute (see above) which performs its own authentication. + if !reported_auth_ok { + stream.write_message_noflush(&Be::AuthenticationOk)?; + } + + // Forward all postgres connection params to the client. + // Right now the implementation is very hacky and inefficent (ideally, + // we don't need an intermediate hashmap), but at least it should be correct. + for (name, value) in &node.params { + // TODO: Theoretically, this could result in a big pile of params... + stream.write_message_noflush(&Be::ParameterStatus { + name: name.as_bytes(), + value: value.as_bytes(), + })?; + } + + stream + .write_message_noflush(&Be::BackendKeyData(cancel_key_data))? + .write_message(&Be::ReadyForQuery) + .await?; + + Ok(()) +} + +/// Forward bytes in both directions (client <-> compute). +#[tracing::instrument(skip_all)] +async fn proxy_pass( + client: impl AsyncRead + AsyncWrite + Unpin, + compute: impl AsyncRead + AsyncWrite + Unpin, + aux: &MetricsAuxInfo, +) -> anyhow::Result<()> { + let m_sent = NUM_BYTES_PROXIED_COUNTER.with_label_values(&aux.traffic_labels("tx")); + let mut client = MeasuredStream::new(client, |cnt| { + // Number of bytes we sent to the client (outbound). + m_sent.inc_by(cnt as u64); + }); + + let m_recv = NUM_BYTES_PROXIED_COUNTER.with_label_values(&aux.traffic_labels("rx")); + let mut compute = MeasuredStream::new(compute, |cnt| { + // Number of bytes the client sent to the compute node (inbound). + m_recv.inc_by(cnt as u64); + }); + + // Starting from here we only proxy the client's traffic. + info!("performing the proxy pass..."); + let _ = tokio::io::copy_bidirectional(&mut client, &mut compute).await?; + + Ok(()) +} + /// Thin connection context. struct Client<'a, S> { /// The underlying libpq protocol stream. @@ -255,17 +399,17 @@ impl<'a, S> Client<'a, S> { } } -impl Client<'_, S> { +impl Client<'_, S> { /// Let the client authenticate and connect to the designated compute node. async fn connect_to_db(self, session: cancellation::Session<'_>) -> anyhow::Result<()> { let Self { mut stream, - creds, + mut creds, params, session_id, } = self; - let extra = auth::ConsoleReqExtra { + let extra = console::ConsoleReqExtra { session_id, // aka this connection's id application_name: params.get("application_name"), }; @@ -278,54 +422,16 @@ impl Client<'_, S> { .instrument(info_span!("auth")) .await?; - let node = auth_result.value; - let (db, cancel_closure) = node - .config - .connect(params) + let AuthSuccess { + reported_auth_ok, + value: mut node_info, + } = auth_result; + + let node = connect_to_compute(&mut node_info, params, &extra, &creds) .or_else(|e| stream.throw_error(e)) .await?; - let cancel_key_data = session.enable_query_cancellation(cancel_closure); - - // Report authentication success if we haven't done this already. - // Note that we do this only (for the most part) after we've connected - // to a compute (see above) which performs its own authentication. - if !auth_result.reported_auth_ok { - stream.write_message_noflush(&Be::AuthenticationOk)?; - } - - // Forward all postgres connection params to the client. - // Right now the implementation is very hacky and inefficent (ideally, - // we don't need an intermediate hashmap), but at least it should be correct. - for (name, value) in &db.params { - // TODO: Theoretically, this could result in a big pile of params... - stream.write_message_noflush(&Be::ParameterStatus { - name: name.as_bytes(), - value: value.as_bytes(), - })?; - } - - stream - .write_message_noflush(&Be::BackendKeyData(cancel_key_data))? - .write_message(&Be::ReadyForQuery) - .await?; - - let m_sent = NUM_BYTES_PROXIED_COUNTER.with_label_values(&node.aux.traffic_labels("tx")); - let mut client = MeasuredStream::new(stream.into_inner(), |cnt| { - // Number of bytes we sent to the client (outbound). - m_sent.inc_by(cnt as u64); - }); - - let m_recv = NUM_BYTES_PROXIED_COUNTER.with_label_values(&node.aux.traffic_labels("rx")); - let mut db = MeasuredStream::new(db.stream, |cnt| { - // Number of bytes the client sent to the compute node (inbound). - m_recv.inc_by(cnt as u64); - }); - - // Starting from here we only proxy the client's traffic. - info!("performing the proxy pass..."); - let _ = tokio::io::copy_bidirectional(&mut client, &mut db).await?; - - Ok(()) + prepare_client_connection(&node, reported_auth_ok, session, &mut stream).await?; + proxy_pass(stream.into_inner(), node.stream, &node_info.aux).await } } diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 3a852b2207..8bfc9e2770 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -25,6 +25,7 @@ futures-channel = { version = "0.3", features = ["sink"] } futures-executor = { version = "0.3" } futures-task = { version = "0.3", default-features = false, features = ["std"] } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +hashbrown = { version = "0.12", features = ["raw"] } indexmap = { version = "1", default-features = false, features = ["std"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } @@ -58,6 +59,7 @@ url = { version = "2", features = ["serde"] } anyhow = { version = "1", features = ["backtrace"] } bytes = { version = "1", features = ["serde"] } either = { version = "1" } +hashbrown = { version = "0.12", features = ["raw"] } indexmap = { version = "1", default-features = false, features = ["std"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } From f3dadfb3d0ffc399744be2bc6e44bbd6dc101e0c Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Wed, 1 Feb 2023 16:01:27 +0100 Subject: [PATCH 080/102] Confirm that there is an emergency before manual execution of prod deploy workflow (#3507) ![image](https://user-images.githubusercontent.com/7127190/215840037-69eda3ee-920e-4b90-bf7d-aa58f0bdfb50.png) --- .github/workflows/build_and_test.yml | 2 +- .github/workflows/deploy-prod.yml | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 89e12360f9..de5bd7efd2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -868,7 +868,7 @@ jobs: if [[ "$GITHUB_REF_NAME" == "main" ]]; then gh workflow run deploy-dev.yml --ref main -f branch=${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} elif [[ "$GITHUB_REF_NAME" == "release" ]]; then - gh workflow run deploy-prod.yml --ref release -f branch=${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} + gh workflow run deploy-prod.yml --ref release -f branch=${{ github.sha }} -f dockerTag=${{needs.tag.outputs.build-tag}} -f disclamerAcknowledged=true else echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" exit 1 diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index fbd6bdfd93..16041d0fe8 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -11,7 +11,7 @@ on: description: 'Branch or commit used for deploy scripts and configs' required: true type: string - default: 'main' + default: 'release' deployStorage: description: 'Deploy storage' required: true @@ -27,6 +27,11 @@ on: required: true type: boolean default: true + disclamerAcknowledged: + description: 'I confirm that there is an emergency and I can not use regular release workflow' + required: true + type: boolean + default: false concurrency: group: deploy-prod @@ -36,7 +41,7 @@ jobs: deploy-prod-new: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployStorage + if: inputs.deployStorage && inputs.disclamerAcknowledged defaults: run: shell: bash @@ -67,7 +72,7 @@ jobs: deploy-proxy-prod-new: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployProxy + if: inputs.deployProxy && inputs.disclamerAcknowledged defaults: run: shell: bash @@ -125,7 +130,7 @@ jobs: deploy-storage-broker-prod-new: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployStorageBroker + if: inputs.deployStorageBroker && inputs.disclamerAcknowledged defaults: run: shell: bash @@ -164,7 +169,7 @@ jobs: deploy: runs-on: prod container: 093970136003.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - if: inputs.deployStorage + if: inputs.deployStorage && inputs.disclamerAcknowledged defaults: run: shell: bash @@ -202,7 +207,7 @@ jobs: deploy-proxy: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployProxy + if: inputs.deployProxy && inputs.disclamerAcknowledged defaults: run: shell: bash @@ -238,7 +243,7 @@ jobs: name: deploy storage broker on old staging and old prod runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployStorageBroker + if: inputs.deployStorageBroker && inputs.disclamerAcknowledged defaults: run: shell: bash From 567b71c1d2ad2e8e7a67eeb657abb37b9960b3c7 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Wed, 1 Feb 2023 18:11:00 +0000 Subject: [PATCH 081/102] Require poetry 1.3; regenerate poetry.lock (#3508) Ref https://python-poetry.org/blog/announcing-poetry-1.3.0/#new-lock-file-format --- .github/workflows/build_and_test.yml | 2 +- README.md | 2 +- poetry.lock | 2696 +++++++++++++------------- 3 files changed, 1350 insertions(+), 1350 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index de5bd7efd2..27b7f54856 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -54,7 +54,7 @@ jobs: check-codestyle-python: runs-on: [ self-hosted, gen3, small ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cloud:pinned + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init steps: diff --git a/README.md b/README.md index 7b629e71a5..29389e7a5d 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ make -j`sysctl -n hw.logicalcpu` To run the `psql` client, install the `postgresql-client` package or modify `PATH` and `LD_LIBRARY_PATH` to include `pg_install/bin` and `pg_install/lib`, respectively. To run the integration tests or Python scripts (not required to use the code), install -Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (requires [poetry](https://python-poetry.org/)) in the project directory. +Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (requires [poetry>=1.3](https://python-poetry.org/)) in the project directory. #### Running neon database diff --git a/poetry.lock b/poetry.lock index 6d7f39ecbd..f14c495556 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + [[package]] name = "aiohttp" version = "3.7.4" @@ -5,6 +7,45 @@ description = "Async http client/server framework (asyncio)" category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "aiohttp-3.7.4-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:6c8200abc9dc5f27203986100579fc19ccad7a832c07d2bc151ce4ff17190076"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:dd7936f2a6daa861143e376b3a1fb56e9b802f4980923594edd9ca5670974895"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:bc3d14bf71a3fb94e5acf5bbf67331ab335467129af6416a437bd6024e4f743d"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:8ec1a38074f68d66ccb467ed9a673a726bb397142c273f90d4ba954666e87d54"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:b84ad94868e1e6a5e30d30ec419956042815dfaea1b1df1cef623e4564c374d9"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:d5d102e945ecca93bcd9801a7bb2fa703e37ad188a2f81b1e65e4abe4b51b00c"}, + {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c2a80fd9a8d7e41b4e38ea9fe149deed0d6aaede255c497e66b8213274d6d61b"}, + {file = "aiohttp-3.7.4-cp36-cp36m-win32.whl", hash = "sha256:481d4b96969fbfdcc3ff35eea5305d8565a8300410d3d269ccac69e7256b1329"}, + {file = "aiohttp-3.7.4-cp36-cp36m-win_amd64.whl", hash = "sha256:16d0683ef8a6d803207f02b899c928223eb219111bd52420ef3d7a8aa76227b6"}, + {file = "aiohttp-3.7.4-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:eab51036cac2da8a50d7ff0ea30be47750547c9aa1aa2cf1a1b710a1827e7dbe"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:feb24ff1226beeb056e247cf2e24bba5232519efb5645121c4aea5b6ad74c1f2"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:119feb2bd551e58d83d1b38bfa4cb921af8ddedec9fad7183132db334c3133e0"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:6ca56bdfaf825f4439e9e3673775e1032d8b6ea63b8953d3812c71bd6a8b81de"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:5563ad7fde451b1986d42b9bb9140e2599ecf4f8e42241f6da0d3d624b776f40"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:62bc216eafac3204877241569209d9ba6226185aa6d561c19159f2e1cbb6abfb"}, + {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:f4496d8d04da2e98cc9133e238ccebf6a13ef39a93da2e87146c8c8ac9768242"}, + {file = "aiohttp-3.7.4-cp37-cp37m-win32.whl", hash = "sha256:2ffea7904e70350da429568113ae422c88d2234ae776519549513c8f217f58a9"}, + {file = "aiohttp-3.7.4-cp37-cp37m-win_amd64.whl", hash = "sha256:5e91e927003d1ed9283dee9abcb989334fc8e72cf89ebe94dc3e07e3ff0b11e9"}, + {file = "aiohttp-3.7.4-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:4c1bdbfdd231a20eee3e56bd0ac1cd88c4ff41b64ab679ed65b75c9c74b6c5c2"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:71680321a8a7176a58dfbc230789790639db78dad61a6e120b39f314f43f1907"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7dbd087ff2f4046b9b37ba28ed73f15fd0bc9f4fdc8ef6781913da7f808d9536"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:dee68ec462ff10c1d836c0ea2642116aba6151c6880b688e56b4c0246770f297"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:99c5a5bf7135607959441b7d720d96c8e5c46a1f96e9d6d4c9498be8d5f24212"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:5dde6d24bacac480be03f4f864e9a67faac5032e28841b00533cd168ab39cad9"}, + {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:418597633b5cd9639e514b1d748f358832c08cd5d9ef0870026535bd5eaefdd0"}, + {file = "aiohttp-3.7.4-cp38-cp38-win32.whl", hash = "sha256:e76e78863a4eaec3aee5722d85d04dcbd9844bc6cd3bfa6aa880ff46ad16bfcb"}, + {file = "aiohttp-3.7.4-cp38-cp38-win_amd64.whl", hash = "sha256:950b7ef08b2afdab2488ee2edaff92a03ca500a48f1e1aaa5900e73d6cf992bc"}, + {file = "aiohttp-3.7.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2eb3efe243e0f4ecbb654b08444ae6ffab37ac0ef8f69d3a2ffb958905379daf"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:822bd4fd21abaa7b28d65fc9871ecabaddc42767884a626317ef5b75c20e8a2d"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:58c62152c4c8731a3152e7e650b29ace18304d086cb5552d317a54ff2749d32a"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:7c7820099e8b3171e54e7eedc33e9450afe7cd08172632d32128bd527f8cb77d"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:5b50e0b9460100fe05d7472264d1975f21ac007b35dcd6fd50279b72925a27f4"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:c44d3c82a933c6cbc21039326767e778eface44fca55c65719921c4b9661a3f7"}, + {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:cc31e906be1cc121ee201adbdf844522ea3349600dd0a40366611ca18cd40e81"}, + {file = "aiohttp-3.7.4-cp39-cp39-win32.whl", hash = "sha256:fbd3b5e18d34683decc00d9a360179ac1e7a320a5fee10ab8053ffd6deab76e0"}, + {file = "aiohttp-3.7.4-cp39-cp39-win_amd64.whl", hash = "sha256:40bd1b101b71a18a528ffce812cc14ff77d4a2a1272dfb8b11b200967489ef3e"}, + {file = "aiohttp-3.7.4.tar.gz", hash = "sha256:5d84ecc73141d0a0d61ece0742bb7ff5751b0657dab8405f899d3ceb104cc7de"}, +] [package.dependencies] async-timeout = ">=3.0,<4.0" @@ -24,6 +65,10 @@ description = "Postgres integration with asyncio." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "aiopg-1.3.4-py3-none-any.whl", hash = "sha256:b5b74a124831aad71608c3c203479db90bac4a7eb3f8982bc48c3d3e6f1e57bf"}, + {file = "aiopg-1.3.4.tar.gz", hash = "sha256:23f9e4cd9f28e9d91a6de3b4fb517e8bed25511cd954acccba9fe3a702d9b7d0"}, +] [package.dependencies] async-timeout = ">=3.0,<5.0" @@ -39,6 +84,10 @@ description = "Allure pytest integration" category = "main" optional = false python-versions = "*" +files = [ + {file = "allure-pytest-2.10.0.tar.gz", hash = "sha256:3b2ab67629f4cbd8617abd817d2b22292c6eb7efd5584f992d1af8143aea6ee7"}, + {file = "allure_pytest-2.10.0-py3-none-any.whl", hash = "sha256:08274096594758447db54c3b2c382526ee04f1fe12119cdaee92d2d93c84b530"}, +] [package.dependencies] allure-python-commons = "2.10.0" @@ -52,6 +101,10 @@ description = "Common module for integrate allure with python-based frameworks" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "allure-python-commons-2.10.0.tar.gz", hash = "sha256:d4d31344b0f0037a4a11e16b91b28cf0eeb23ffa0e50c27fcfc6aabe72212d3c"}, + {file = "allure_python_commons-2.10.0-py3-none-any.whl", hash = "sha256:2a717e8ca8d296bf89cd57f38fc3c21893bd7ea8cd02a6ae5420e6d1a6eda5d0"}, +] [package.dependencies] attrs = ">=16.0.0" @@ -65,6 +118,10 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = false python-versions = ">=3.5.3" +files = [ + {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, + {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, +] [[package]] name = "asyncpg" @@ -73,6 +130,44 @@ description = "An asyncio PostgreSQL driver" category = "main" optional = false python-versions = ">=3.7.0" +files = [ + {file = "asyncpg-0.27.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fca608d199ffed4903dce1bcd97ad0fe8260f405c1c225bdf0002709132171c2"}, + {file = "asyncpg-0.27.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:20b596d8d074f6f695c13ffb8646d0b6bb1ab570ba7b0cfd349b921ff03cfc1e"}, + {file = "asyncpg-0.27.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a6206210c869ebd3f4eb9e89bea132aefb56ff3d1b7dd7e26b102b17e27bbb1"}, + {file = "asyncpg-0.27.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7a94c03386bb95456b12c66026b3a87d1b965f0f1e5733c36e7229f8f137747"}, + {file = "asyncpg-0.27.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bfc3980b4ba6f97138b04f0d32e8af21d6c9fa1f8e6e140c07d15690a0a99279"}, + {file = "asyncpg-0.27.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9654085f2b22f66952124de13a8071b54453ff972c25c59b5ce1173a4283ffd9"}, + {file = "asyncpg-0.27.0-cp310-cp310-win32.whl", hash = "sha256:879c29a75969eb2722f94443752f4720d560d1e748474de54ae8dd230bc4956b"}, + {file = "asyncpg-0.27.0-cp310-cp310-win_amd64.whl", hash = "sha256:ab0f21c4818d46a60ca789ebc92327d6d874d3b7ccff3963f7af0a21dc6cff52"}, + {file = "asyncpg-0.27.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:18f77e8e71e826ba2d0c3ba6764930776719ae2b225ca07e014590545928b576"}, + {file = "asyncpg-0.27.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c2232d4625c558f2aa001942cac1d7952aa9f0dbfc212f63bc754277769e1ef2"}, + {file = "asyncpg-0.27.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9a3a4ff43702d39e3c97a8786314123d314e0f0e4dabc8367db5b665c93914de"}, + {file = "asyncpg-0.27.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccddb9419ab4e1c48742457d0c0362dbdaeb9b28e6875115abfe319b29ee225d"}, + {file = "asyncpg-0.27.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:768e0e7c2898d40b16d4ef7a0b44e8150db3dd8995b4652aa1fe2902e92c7df8"}, + {file = "asyncpg-0.27.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609054a1f47292a905582a1cfcca51a6f3f30ab9d822448693e66fdddde27920"}, + {file = "asyncpg-0.27.0-cp311-cp311-win32.whl", hash = "sha256:8113e17cfe236dc2277ec844ba9b3d5312f61bd2fdae6d3ed1c1cdd75f6cf2d8"}, + {file = "asyncpg-0.27.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb71211414dd1eeb8d31ec529fe77cff04bf53efc783a5f6f0a32d84923f45cf"}, + {file = "asyncpg-0.27.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4750f5cf49ed48a6e49c6e5aed390eee367694636c2dcfaf4a273ca832c5c43c"}, + {file = "asyncpg-0.27.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:eca01eb112a39d31cc4abb93a5aef2a81514c23f70956729f42fb83b11b3483f"}, + {file = "asyncpg-0.27.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5710cb0937f696ce303f5eed6d272e3f057339bb4139378ccecafa9ee923a71c"}, + {file = "asyncpg-0.27.0-cp37-cp37m-win_amd64.whl", hash = "sha256:71cca80a056ebe19ec74b7117b09e650990c3ca535ac1c35234a96f65604192f"}, + {file = "asyncpg-0.27.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4bb366ae34af5b5cabc3ac6a5347dfb6013af38c68af8452f27968d49085ecc0"}, + {file = "asyncpg-0.27.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16ba8ec2e85d586b4a12bcd03e8d29e3d99e832764d6a1d0b8c27dbbe4a2569d"}, + {file = "asyncpg-0.27.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d20dea7b83651d93b1eb2f353511fe7fd554752844523f17ad30115d8b9c8cd6"}, + {file = "asyncpg-0.27.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e56ac8a8237ad4adec97c0cd4728596885f908053ab725e22900b5902e7f8e69"}, + {file = "asyncpg-0.27.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bf21ebf023ec67335258e0f3d3ad7b91bb9507985ba2b2206346de488267cad0"}, + {file = "asyncpg-0.27.0-cp38-cp38-win32.whl", hash = "sha256:69aa1b443a182b13a17ff926ed6627af2d98f62f2fe5890583270cc4073f63bf"}, + {file = "asyncpg-0.27.0-cp38-cp38-win_amd64.whl", hash = "sha256:62932f29cf2433988fcd799770ec64b374a3691e7902ecf85da14d5e0854d1ea"}, + {file = "asyncpg-0.27.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fddcacf695581a8d856654bc4c8cfb73d5c9df26d5f55201722d3e6a699e9629"}, + {file = "asyncpg-0.27.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7d8585707ecc6661d07367d444bbaa846b4e095d84451340da8df55a3757e152"}, + {file = "asyncpg-0.27.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:975a320baf7020339a67315284a4d3bf7460e664e484672bd3e71dbd881bc692"}, + {file = "asyncpg-0.27.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2232ebae9796d4600a7819fc383da78ab51b32a092795f4555575fc934c1c89d"}, + {file = "asyncpg-0.27.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:88b62164738239f62f4af92567b846a8ef7cf8abf53eddd83650603de4d52163"}, + {file = "asyncpg-0.27.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:eb4b2fdf88af4fb1cc569781a8f933d2a73ee82cd720e0cb4edabbaecf2a905b"}, + {file = "asyncpg-0.27.0-cp39-cp39-win32.whl", hash = "sha256:8934577e1ed13f7d2d9cea3cc016cc6f95c19faedea2c2b56a6f94f257cea672"}, + {file = "asyncpg-0.27.0-cp39-cp39-win_amd64.whl", hash = "sha256:1b6499de06fe035cf2fa932ec5617ed3f37d4ebbf663b655922e105a484a6af9"}, + {file = "asyncpg-0.27.0.tar.gz", hash = "sha256:720986d9a4705dd8a40fdf172036f5ae787225036a7eb46e704c45aa8f62c054"}, +] [package.extras] dev = ["Cython (>=0.29.24,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "flake8 (>=5.0.4,<5.1.0)", "pytest (>=6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)", "uvloop (>=0.15.3)"] @@ -86,6 +181,9 @@ description = "Atomic file writes." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] [[package]] name = "attrs" @@ -94,6 +192,10 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] [package.extras] dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] @@ -108,6 +210,11 @@ description = "AWS SAM Translator is a library that transform SAM templates into category = "main" optional = false python-versions = ">=3.7, <=4.0, !=4.0" +files = [ + {file = "aws-sam-translator-1.48.0.tar.gz", hash = "sha256:7171037323dfa30f8f73e9bccb9210e4c384a585e087219a9518a5204f0a2c44"}, + {file = "aws_sam_translator-1.48.0-py2-none-any.whl", hash = "sha256:be18dfa3dfe7ab291d281667c5f73ac62dbe6bfe86df7d122e4258b906b736f0"}, + {file = "aws_sam_translator-1.48.0-py3-none-any.whl", hash = "sha256:ca4f8f9910d7713aeaba59346775bfb3198f6acb47c6704572f9bd3fc0fb5bf0"}, +] [package.dependencies] boto3 = ">=1.19.5,<2.0.0" @@ -123,6 +230,10 @@ description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers category = "main" optional = false python-versions = "*" +files = [ + {file = "aws-xray-sdk-2.10.0.tar.gz", hash = "sha256:9b14924fd0628cf92936055864655354003f0b1acc3e1c3ffde6403d0799dd7a"}, + {file = "aws_xray_sdk-2.10.0-py2.py3-none-any.whl", hash = "sha256:7551e81a796e1a5471ebe84844c40e8edf7c218db33506d046fec61f7495eda4"}, +] [package.dependencies] botocore = ">=1.11.3" @@ -135,6 +246,10 @@ description = "Function decoration for backoff and retry" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "backoff-1.11.1-py2.py3-none-any.whl", hash = "sha256:61928f8fa48d52e4faa81875eecf308eccfb1016b018bb6bd21e05b5d90a96c5"}, + {file = "backoff-1.11.1.tar.gz", hash = "sha256:ccb962a2378418c667b3c979b504fdeb7d9e0d29c0579e3b13b86467177728cb"}, +] [[package]] name = "black" @@ -143,6 +258,31 @@ description = "The uncompromising code formatter." category = "dev" optional = false python-versions = ">=3.6.2" +files = [ + {file = "black-22.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f586c26118bc6e714ec58c09df0157fe2d9ee195c764f630eb0d8e7ccce72e69"}, + {file = "black-22.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b270a168d69edb8b7ed32c193ef10fd27844e5c60852039599f9184460ce0807"}, + {file = "black-22.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6797f58943fceb1c461fb572edbe828d811e719c24e03375fd25170ada53825e"}, + {file = "black-22.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c85928b9d5f83b23cee7d0efcb310172412fbf7cb9d9ce963bd67fd141781def"}, + {file = "black-22.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:f6fe02afde060bbeef044af7996f335fbe90b039ccf3f5eb8f16df8b20f77666"}, + {file = "black-22.6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cfaf3895a9634e882bf9d2363fed5af8888802d670f58b279b0bece00e9a872d"}, + {file = "black-22.6.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94783f636bca89f11eb5d50437e8e17fbc6a929a628d82304c80fa9cd945f256"}, + {file = "black-22.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2ea29072e954a4d55a2ff58971b83365eba5d3d357352a07a7a4df0d95f51c78"}, + {file = "black-22.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e439798f819d49ba1c0bd9664427a05aab79bfba777a6db94fd4e56fae0cb849"}, + {file = "black-22.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:187d96c5e713f441a5829e77120c269b6514418f4513a390b0499b0987f2ff1c"}, + {file = "black-22.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:074458dc2f6e0d3dab7928d4417bb6957bb834434516f21514138437accdbe90"}, + {file = "black-22.6.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a218d7e5856f91d20f04e931b6f16d15356db1c846ee55f01bac297a705ca24f"}, + {file = "black-22.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:568ac3c465b1c8b34b61cd7a4e349e93f91abf0f9371eda1cf87194663ab684e"}, + {file = "black-22.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6c1734ab264b8f7929cef8ae5f900b85d579e6cbfde09d7387da8f04771b51c6"}, + {file = "black-22.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9a3ac16efe9ec7d7381ddebcc022119794872abce99475345c5a61aa18c45ad"}, + {file = "black-22.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:b9fd45787ba8aa3f5e0a0a98920c1012c884622c6c920dbe98dbd05bc7c70fbf"}, + {file = "black-22.6.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7ba9be198ecca5031cd78745780d65a3f75a34b2ff9be5837045dce55db83d1c"}, + {file = "black-22.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3db5b6409b96d9bd543323b23ef32a1a2b06416d525d27e0f67e74f1446c8f2"}, + {file = "black-22.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:560558527e52ce8afba936fcce93a7411ab40c7d5fe8c2463e279e843c0328ee"}, + {file = "black-22.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b154e6bbde1e79ea3260c4b40c0b7b3109ffcdf7bc4ebf8859169a6af72cd70b"}, + {file = "black-22.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4af5bc0e1f96be5ae9bd7aaec219c901a94d6caa2484c21983d043371c733fc4"}, + {file = "black-22.6.0-py3-none-any.whl", hash = "sha256:ac609cf8ef5e7115ddd07d85d988d074ed00e10fbc3445aee393e70164a2219c"}, + {file = "black-22.6.0.tar.gz", hash = "sha256:6c6d39e28aed379aec40da1c65434c77d75e65bb59a1e1c283de545fb4e7c6c9"}, +] [package.dependencies] click = ">=8.0.0" @@ -165,6 +305,10 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "boto3-1.26.16-py3-none-any.whl", hash = "sha256:4f493a2aed71cee93e626de4f67ce58dd82c0473480a0fc45b131715cd8f4f30"}, + {file = "boto3-1.26.16.tar.gz", hash = "sha256:31c0adf71e4bd19a5428580bb229d7ea3b5795eecaa0847a85385df00c026116"}, +] [package.dependencies] botocore = ">=1.29.16,<1.30.0" @@ -181,6 +325,10 @@ description = "Type annotations for boto3 1.26.16 generated with mypy-boto3-buil category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "boto3-stubs-1.26.16.tar.gz", hash = "sha256:618253ae19f1480785759bcaee8c8b10ed3fc037027247c26a3461a50f58406d"}, + {file = "boto3_stubs-1.26.16-py3-none-any.whl", hash = "sha256:8cf2925bc3e1349c93eb0f49c1061affc5ca314d69eeb335349037969d0787ed"}, +] [package.dependencies] botocore-stubs = "*" @@ -522,6 +670,10 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "botocore-1.29.16-py3-none-any.whl", hash = "sha256:271b599e6cfe214405ed50d41cd967add1d5d469383dd81ff583bc818b47f59b"}, + {file = "botocore-1.29.16.tar.gz", hash = "sha256:8cfcc10f2f1751608c3cec694f2d6b5e16ebcd50d0a104f9914d5616227c62e9"}, +] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -538,6 +690,10 @@ description = "Type annotations for botocore 1.27.38 generated with mypy-boto3-b category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "botocore-stubs-1.27.38.tar.gz", hash = "sha256:408e8b86b5d171b58f81c74ca9d3b5317a5a8e2d3bc2073aa841ac13b8939e56"}, + {file = "botocore_stubs-1.27.38-py3-none-any.whl", hash = "sha256:7add7641e9a479a9c8366893bb522fd9ca3d58714201e43662a200a148a1bc38"}, +] [package.dependencies] typing-extensions = ">=4.1.0" @@ -549,6 +705,10 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] [[package]] name = "cffi" @@ -557,1253 +717,7 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" - -[package.dependencies] -pycparser = "*" - -[[package]] -name = "cfn-lint" -version = "0.61.3" -description = "Checks CloudFormation templates for practices and behaviour that could potentially be improved" -category = "main" -optional = false -python-versions = ">=3.6, <=4.0, !=4.0" - -[package.dependencies] -aws-sam-translator = ">=1.47.0" -jschema-to-python = ">=1.2.3,<1.3.0" -jsonpatch = "*" -jsonschema = ">=3.0,<4.0" -junit-xml = ">=1.9,<2.0" -networkx = ">=2.4,<3.0" -pyyaml = ">5.4" -sarif-om = ">=1.0.4,<1.1.0" - -[[package]] -name = "chardet" -version = "3.0.4" -description = "Universal encoding detector for Python 2 and 3" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "charset-normalizer" -version = "2.1.0" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" -optional = false -python-versions = ">=3.6.0" - -[package.extras] -unicode-backport = ["unicodedata2"] - -[[package]] -name = "click" -version = "8.1.3" -description = "Composable command line interface toolkit" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.5" -description = "Cross-platform colored terminal text." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "cryptography" -version = "38.0.3" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = ">=1.12" - -[package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] -sdist = ["setuptools-rust (>=0.11.4)"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] - -[[package]] -name = "docker" -version = "4.2.2" -description = "A Python library for the Docker Engine API." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.dependencies] -pypiwin32 = {version = "223", markers = "sys_platform == \"win32\" and python_version >= \"3.6\""} -requests = ">=2.14.2,<2.18.0 || >2.18.0" -six = ">=1.4.0" -websocket-client = ">=0.32.0" - -[package.extras] -ssh = ["paramiko (>=2.4.2)"] -tls = ["cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] - -[[package]] -name = "ecdsa" -version = "0.18.0" -description = "ECDSA cryptographic signature library (pure python)" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[package.dependencies] -six = ">=1.9.0" - -[package.extras] -gmpy = ["gmpy"] -gmpy2 = ["gmpy2"] - -[[package]] -name = "execnet" -version = "1.9.0" -description = "execnet: rapid multi-Python deployment" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -testing = ["pre-commit"] - -[[package]] -name = "flake8" -version = "5.0.4" -description = "the modular source code checker: pep8 pyflakes and co" -category = "dev" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -mccabe = ">=0.7.0,<0.8.0" -pycodestyle = ">=2.9.0,<2.10.0" -pyflakes = ">=2.5.0,<2.6.0" - -[[package]] -name = "flask" -version = "2.1.3" -description = "A simple framework for building complex web applications." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -click = ">=8.0" -importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} -itsdangerous = ">=2.0" -Jinja2 = ">=3.0" -Werkzeug = ">=2.0" - -[package.extras] -async = ["asgiref (>=3.2)"] -dotenv = ["python-dotenv"] - -[[package]] -name = "flask-cors" -version = "3.0.10" -description = "A Flask extension adding a decorator for CORS support" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -Flask = ">=0.9" -Six = "*" - -[[package]] -name = "graphql-core" -version = "3.2.1" -description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." -category = "main" -optional = false -python-versions = ">=3.6,<4" - -[[package]] -name = "idna" -version = "3.3" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "importlib-metadata" -version = "4.12.0" -description = "Read metadata from Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] -perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "isort" -version = "5.10.1" -description = "A Python utility / library to sort Python imports." -category = "dev" -optional = false -python-versions = ">=3.6.1,<4.0" - -[package.extras] -colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile-deprecated-finder = ["pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - -[[package]] -name = "itsdangerous" -version = "2.1.2" -description = "Safely pass data to untrusted environments and back." -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "jschema-to-python" -version = "1.2.3" -description = "Generate source code for Python classes from a JSON schema." -category = "main" -optional = false -python-versions = ">= 2.7" - -[package.dependencies] -attrs = "*" -jsonpickle = "*" -pbr = "*" - -[[package]] -name = "jsondiff" -version = "2.0.0" -description = "Diff JSON and JSON-like structures in Python" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "jsonpatch" -version = "1.32" -description = "Apply JSON-Patches (RFC 6902)" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.dependencies] -jsonpointer = ">=1.9" - -[[package]] -name = "jsonpickle" -version = "2.2.0" -description = "Python library for serializing any arbitrary object graph into JSON" -category = "main" -optional = false -python-versions = ">=2.7" - -[package.extras] -docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] -testing = ["ecdsa", "enum34", "feedparser", "jsonlib", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (<1.1.0)", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] -testing-libs = ["simplejson", "ujson", "yajl"] - -[[package]] -name = "jsonpointer" -version = "2.3" -description = "Identify specific nodes in a JSON document (RFC 6901)" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "jsonschema" -version = "3.2.0" -description = "An implementation of JSON Schema validation for Python" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -attrs = ">=17.4.0" -pyrsistent = ">=0.14.0" -setuptools = "*" -six = ">=1.11.0" - -[package.extras] -format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] -format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] - -[[package]] -name = "junit-xml" -version = "1.9" -description = "Creates JUnit XML test result documents that can be read by tools such as Jenkins" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -six = "*" - -[[package]] -name = "markupsafe" -version = "2.1.1" -description = "Safely add untrusted strings to HTML/XML markup." -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -category = "dev" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "moto" -version = "3.1.18" -description = "A library that allows your python tests to easily mock out the boto library" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -aws-xray-sdk = {version = ">=0.93,<0.96 || >0.96", optional = true, markers = "extra == \"server\""} -boto3 = ">=1.9.201" -botocore = ">=1.12.201" -cfn-lint = {version = ">=0.4.0", optional = true, markers = "extra == \"server\""} -cryptography = ">=3.3.1" -docker = {version = ">=2.5.1", optional = true, markers = "extra == \"server\""} -ecdsa = {version = "!=0.15", optional = true, markers = "extra == \"server\""} -flask = {version = "<2.2.0", optional = true, markers = "extra == \"server\""} -flask-cors = {version = "*", optional = true, markers = "extra == \"server\""} -graphql-core = {version = "*", optional = true, markers = "extra == \"server\""} -idna = {version = ">=2.5,<4", optional = true, markers = "extra == \"server\""} -Jinja2 = ">=2.10.1" -jsondiff = {version = ">=1.1.2", optional = true, markers = "extra == \"server\""} -MarkupSafe = "!=2.0.0a1" -openapi-spec-validator = {version = ">=0.2.8", optional = true, markers = "extra == \"server\""} -pyparsing = {version = ">=3.0.7", optional = true, markers = "extra == \"server\""} -python-dateutil = ">=2.1,<3.0.0" -python-jose = {version = ">=3.1.0,<4.0.0", extras = ["cryptography"], optional = true, markers = "extra == \"server\""} -pytz = "*" -PyYAML = {version = ">=5.1", optional = true, markers = "extra == \"server\""} -requests = ">=2.5" -responses = ">=0.9.0" -setuptools = {version = "*", optional = true, markers = "extra == \"server\""} -sshpubkeys = {version = ">=3.1.0", optional = true, markers = "extra == \"server\""} -werkzeug = ">=0.5,<2.2.0" -xmltodict = "*" - -[package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] -appsync = ["graphql-core"] -awslambda = ["docker (>=2.5.1)"] -batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -ds = ["sshpubkeys (>=3.1.0)"] -dynamodb = ["docker (>=2.5.1)"] -dynamodb2 = ["docker (>=2.5.1)"] -dynamodbstreams = ["docker (>=2.5.1)"] -ebs = ["sshpubkeys (>=3.1.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] -efs = ["sshpubkeys (>=3.1.0)"] -glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -route53resolver = ["sshpubkeys (>=3.1.0)"] -s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (<2.2.0)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -ssm = ["PyYAML (>=5.1)", "dataclasses"] -xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] - -[[package]] -name = "multidict" -version = "6.0.4" -description = "multidict implementation" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mypy" -version = "0.991" -description = "Optional static typing for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -mypy-extensions = ">=0.4.3" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=3.10" - -[package.extras] -dmypy = ["psutil (>=4.0)"] -install-types = ["pip"] -python2 = ["typed-ast (>=1.4.0,<2)"] -reports = ["lxml"] - -[[package]] -name = "mypy-boto3-s3" -version = "1.26.0.post1" -description = "Type annotations for boto3.S3 1.26.0 service generated with mypy-boto3-builder 7.11.10" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = ">=4.1.0" - -[[package]] -name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "networkx" -version = "2.8.5" -description = "Python package for creating and manipulating graphs and networks" -category = "main" -optional = false -python-versions = ">=3.8" - -[package.extras] -default = ["matplotlib (>=3.4)", "numpy (>=1.19)", "pandas (>=1.3)", "scipy (>=1.8)"] -developer = ["mypy (>=0.960)", "pre-commit (>=2.19)"] -doc = ["nb2plots (>=0.6)", "numpydoc (>=1.4)", "pillow (>=9.1)", "pydata-sphinx-theme (>=0.9)", "sphinx (>=5)", "sphinx-gallery (>=0.10)", "texext (>=0.6.6)"] -extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.9)", "sympy (>=1.10)"] -test = ["codecov (>=2.1)", "pytest (>=7.1)", "pytest-cov (>=3.0)"] - -[[package]] -name = "openapi-schema-validator" -version = "0.2.3" -description = "OpenAPI schema validation for Python" -category = "main" -optional = false -python-versions = ">=3.7.0,<4.0.0" - -[package.dependencies] -jsonschema = ">=3.0.0,<5.0.0" - -[package.extras] -isodate = ["isodate"] -rfc3339-validator = ["rfc3339-validator"] -strict-rfc3339 = ["strict-rfc3339"] - -[[package]] -name = "openapi-spec-validator" -version = "0.4.0" -description = "OpenAPI 2.0 (aka Swagger) and OpenAPI 3.0 spec validator" -category = "main" -optional = false -python-versions = ">=3.7.0,<4.0.0" - -[package.dependencies] -jsonschema = ">=3.2.0,<5.0.0" -openapi-schema-validator = ">=0.2.0,<0.3.0" -PyYAML = ">=5.1" -setuptools = "*" - -[package.extras] -requests = ["requests"] - -[[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - -[[package]] -name = "pathspec" -version = "0.9.0" -description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "pbr" -version = "5.9.0" -description = "Python Build Reasonableness" -category = "main" -optional = false -python-versions = ">=2.6" - -[[package]] -name = "platformdirs" -version = "2.5.2" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "prometheus-client" -version = "0.14.1" -description = "Python client for the Prometheus monitoring system." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -twisted = ["twisted"] - -[[package]] -name = "psutil" -version = "5.9.4" -description = "Cross-platform lib for process and system monitoring in Python." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] - -[[package]] -name = "psycopg2-binary" -version = "2.9.3" -description = "psycopg2 - Python-PostgreSQL Database Adapter" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "pyasn1" -version = "0.4.8" -description = "ASN.1 types and codecs" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "pycodestyle" -version = "2.9.1" -description = "Python style guide checker" -category = "dev" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pyflakes" -version = "2.5.0" -description = "passive checker of Python programs" -category = "dev" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "pyjwt" -version = "2.4.0" -description = "JSON Web Token implementation in Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cryptography = {version = ">=3.3.1", optional = true, markers = "extra == \"crypto\""} - -[package.extras] -crypto = ["cryptography (>=3.3.1)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.3.1)", "mypy", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pypiwin32" -version = "223" -description = "" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -pywin32 = ">=223" - -[[package]] -name = "pyrsistent" -version = "0.18.1" -description = "Persistent/Functional/Immutable data structures" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "pytest" -version = "6.2.5" -description = "pytest: simple powerful testing with Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -toml = "*" - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] - -[[package]] -name = "pytest-asyncio" -version = "0.19.0" -description = "Pytest support for asyncio" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -pytest = ">=6.1.0" - -[package.extras] -testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] - -[[package]] -name = "pytest-httpserver" -version = "1.0.6" -description = "pytest-httpserver is a httpserver for pytest" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" - -[package.dependencies] -Werkzeug = ">=2.0.0" - -[[package]] -name = "pytest-lazy-fixture" -version = "0.6.3" -description = "It helps to use fixtures in pytest.mark.parametrize" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -pytest = ">=3.2.5" - -[[package]] -name = "pytest-order" -version = "1.0.1" -description = "pytest plugin to run your tests in a specific order" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pytest = [ - {version = ">=5.0", markers = "python_version < \"3.10\""}, - {version = ">=6.2.4", markers = "python_version >= \"3.10\""}, -] - -[[package]] -name = "pytest-timeout" -version = "2.1.0" -description = "pytest plugin to abort hanging tests" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pytest = ">=5.0.0" - -[[package]] -name = "pytest-xdist" -version = "3.0.2" -description = "pytest xdist plugin for distributed testing and loop-on-failing modes" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -execnet = ">=1.1" -pytest = ">=6.2.0" - -[package.extras] -psutil = ["psutil (>=3.0)"] -setproctitle = ["setproctitle"] -testing = ["filelock"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-jose" -version = "3.3.0" -description = "JOSE implementation in Python" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"cryptography\""} -ecdsa = "!=0.15" -pyasn1 = "*" -rsa = "*" - -[package.extras] -cryptography = ["cryptography (>=3.4.0)"] -pycrypto = ["pyasn1", "pycrypto (>=2.6.0,<2.7.0)"] -pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] - -[[package]] -name = "pytz" -version = "2022.1" -description = "World timezone definitions, modern and historical" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "pywin32" -version = "301" -description = "Python for Window Extensions" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "requests" -version = "2.28.1" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7, <4" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "responses" -version = "0.21.0" -description = "A utility library for mocking out the `requests` Python library." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -requests = ">=2.0,<3.0" -urllib3 = ">=1.25.10" - -[package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-localserver", "types-mock", "types-requests"] - -[[package]] -name = "rsa" -version = "4.9" -description = "Pure-Python RSA implementation" -category = "main" -optional = false -python-versions = ">=3.6,<4" - -[package.dependencies] -pyasn1 = ">=0.1.3" - -[[package]] -name = "s3transfer" -version = "0.6.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.7" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "sarif-om" -version = "1.0.4" -description = "Classes implementing the SARIF 2.1.0 object model." -category = "main" -optional = false -python-versions = ">= 2.7" - -[package.dependencies] -attrs = "*" -pbr = "*" - -[[package]] -name = "setuptools" -version = "65.5.1" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "sshpubkeys" -version = "3.3.1" -description = "SSH public key parser" -category = "main" -optional = false -python-versions = ">=3" - -[package.dependencies] -cryptography = ">=2.1.4" -ecdsa = ">=0.13" - -[package.extras] -dev = ["twine", "wheel", "yapf"] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "types-psutil" -version = "5.9.5.4" -description = "Typing stubs for psutil" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "types-psycopg2" -version = "2.9.18" -description = "Typing stubs for psycopg2" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "types-requests" -version = "2.28.5" -description = "Typing stubs for requests" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -types-urllib3 = "<1.27" - -[[package]] -name = "types-s3transfer" -version = "0.6.0.post3" -description = "Type annotations and code completion for s3transfer" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" - -[[package]] -name = "types-toml" -version = "0.10.8" -description = "Typing stubs for toml" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "types-urllib3" -version = "1.26.17" -description = "Typing stubs for urllib3" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "typing-extensions" -version = "4.3.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "urllib3" -version = "1.26.11" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "websocket-client" -version = "1.3.3" -description = "WebSocket client for Python with low level API options" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - -[[package]] -name = "werkzeug" -version = "2.1.2" -description = "The comprehensive WSGI web application library." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "wrapt" -version = "1.14.1" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "xmltodict" -version = "0.13.0" -description = "Makes working with XML feel like you are working with JSON" -category = "main" -optional = false -python-versions = ">=3.4" - -[[package]] -name = "yarl" -version = "1.8.2" -description = "Yet another URL library" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.8.1" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.9" -content-hash = "7563a38912963d8cf20c99acb06fe55623e65b799c4b88d37dc672e5384c96a3" - -[metadata.files] -aiohttp = [ - {file = "aiohttp-3.7.4-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:6c8200abc9dc5f27203986100579fc19ccad7a832c07d2bc151ce4ff17190076"}, - {file = "aiohttp-3.7.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:dd7936f2a6daa861143e376b3a1fb56e9b802f4980923594edd9ca5670974895"}, - {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:bc3d14bf71a3fb94e5acf5bbf67331ab335467129af6416a437bd6024e4f743d"}, - {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:8ec1a38074f68d66ccb467ed9a673a726bb397142c273f90d4ba954666e87d54"}, - {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:b84ad94868e1e6a5e30d30ec419956042815dfaea1b1df1cef623e4564c374d9"}, - {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:d5d102e945ecca93bcd9801a7bb2fa703e37ad188a2f81b1e65e4abe4b51b00c"}, - {file = "aiohttp-3.7.4-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c2a80fd9a8d7e41b4e38ea9fe149deed0d6aaede255c497e66b8213274d6d61b"}, - {file = "aiohttp-3.7.4-cp36-cp36m-win32.whl", hash = "sha256:481d4b96969fbfdcc3ff35eea5305d8565a8300410d3d269ccac69e7256b1329"}, - {file = "aiohttp-3.7.4-cp36-cp36m-win_amd64.whl", hash = "sha256:16d0683ef8a6d803207f02b899c928223eb219111bd52420ef3d7a8aa76227b6"}, - {file = "aiohttp-3.7.4-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:eab51036cac2da8a50d7ff0ea30be47750547c9aa1aa2cf1a1b710a1827e7dbe"}, - {file = "aiohttp-3.7.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:feb24ff1226beeb056e247cf2e24bba5232519efb5645121c4aea5b6ad74c1f2"}, - {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:119feb2bd551e58d83d1b38bfa4cb921af8ddedec9fad7183132db334c3133e0"}, - {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:6ca56bdfaf825f4439e9e3673775e1032d8b6ea63b8953d3812c71bd6a8b81de"}, - {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:5563ad7fde451b1986d42b9bb9140e2599ecf4f8e42241f6da0d3d624b776f40"}, - {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:62bc216eafac3204877241569209d9ba6226185aa6d561c19159f2e1cbb6abfb"}, - {file = "aiohttp-3.7.4-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:f4496d8d04da2e98cc9133e238ccebf6a13ef39a93da2e87146c8c8ac9768242"}, - {file = "aiohttp-3.7.4-cp37-cp37m-win32.whl", hash = "sha256:2ffea7904e70350da429568113ae422c88d2234ae776519549513c8f217f58a9"}, - {file = "aiohttp-3.7.4-cp37-cp37m-win_amd64.whl", hash = "sha256:5e91e927003d1ed9283dee9abcb989334fc8e72cf89ebe94dc3e07e3ff0b11e9"}, - {file = "aiohttp-3.7.4-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:4c1bdbfdd231a20eee3e56bd0ac1cd88c4ff41b64ab679ed65b75c9c74b6c5c2"}, - {file = "aiohttp-3.7.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:71680321a8a7176a58dfbc230789790639db78dad61a6e120b39f314f43f1907"}, - {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7dbd087ff2f4046b9b37ba28ed73f15fd0bc9f4fdc8ef6781913da7f808d9536"}, - {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:dee68ec462ff10c1d836c0ea2642116aba6151c6880b688e56b4c0246770f297"}, - {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:99c5a5bf7135607959441b7d720d96c8e5c46a1f96e9d6d4c9498be8d5f24212"}, - {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:5dde6d24bacac480be03f4f864e9a67faac5032e28841b00533cd168ab39cad9"}, - {file = "aiohttp-3.7.4-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:418597633b5cd9639e514b1d748f358832c08cd5d9ef0870026535bd5eaefdd0"}, - {file = "aiohttp-3.7.4-cp38-cp38-win32.whl", hash = "sha256:e76e78863a4eaec3aee5722d85d04dcbd9844bc6cd3bfa6aa880ff46ad16bfcb"}, - {file = "aiohttp-3.7.4-cp38-cp38-win_amd64.whl", hash = "sha256:950b7ef08b2afdab2488ee2edaff92a03ca500a48f1e1aaa5900e73d6cf992bc"}, - {file = "aiohttp-3.7.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2eb3efe243e0f4ecbb654b08444ae6ffab37ac0ef8f69d3a2ffb958905379daf"}, - {file = "aiohttp-3.7.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:822bd4fd21abaa7b28d65fc9871ecabaddc42767884a626317ef5b75c20e8a2d"}, - {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:58c62152c4c8731a3152e7e650b29ace18304d086cb5552d317a54ff2749d32a"}, - {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:7c7820099e8b3171e54e7eedc33e9450afe7cd08172632d32128bd527f8cb77d"}, - {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:5b50e0b9460100fe05d7472264d1975f21ac007b35dcd6fd50279b72925a27f4"}, - {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:c44d3c82a933c6cbc21039326767e778eface44fca55c65719921c4b9661a3f7"}, - {file = "aiohttp-3.7.4-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:cc31e906be1cc121ee201adbdf844522ea3349600dd0a40366611ca18cd40e81"}, - {file = "aiohttp-3.7.4-cp39-cp39-win32.whl", hash = "sha256:fbd3b5e18d34683decc00d9a360179ac1e7a320a5fee10ab8053ffd6deab76e0"}, - {file = "aiohttp-3.7.4-cp39-cp39-win_amd64.whl", hash = "sha256:40bd1b101b71a18a528ffce812cc14ff77d4a2a1272dfb8b11b200967489ef3e"}, - {file = "aiohttp-3.7.4.tar.gz", hash = "sha256:5d84ecc73141d0a0d61ece0742bb7ff5751b0657dab8405f899d3ceb104cc7de"}, -] -aiopg = [ - {file = "aiopg-1.3.4-py3-none-any.whl", hash = "sha256:b5b74a124831aad71608c3c203479db90bac4a7eb3f8982bc48c3d3e6f1e57bf"}, - {file = "aiopg-1.3.4.tar.gz", hash = "sha256:23f9e4cd9f28e9d91a6de3b4fb517e8bed25511cd954acccba9fe3a702d9b7d0"}, -] -allure-pytest = [ - {file = "allure-pytest-2.10.0.tar.gz", hash = "sha256:3b2ab67629f4cbd8617abd817d2b22292c6eb7efd5584f992d1af8143aea6ee7"}, - {file = "allure_pytest-2.10.0-py3-none-any.whl", hash = "sha256:08274096594758447db54c3b2c382526ee04f1fe12119cdaee92d2d93c84b530"}, -] -allure-python-commons = [ - {file = "allure-python-commons-2.10.0.tar.gz", hash = "sha256:d4d31344b0f0037a4a11e16b91b28cf0eeb23ffa0e50c27fcfc6aabe72212d3c"}, - {file = "allure_python_commons-2.10.0-py3-none-any.whl", hash = "sha256:2a717e8ca8d296bf89cd57f38fc3c21893bd7ea8cd02a6ae5420e6d1a6eda5d0"}, -] -async-timeout = [ - {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, - {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, -] -asyncpg = [ - {file = "asyncpg-0.27.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fca608d199ffed4903dce1bcd97ad0fe8260f405c1c225bdf0002709132171c2"}, - {file = "asyncpg-0.27.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:20b596d8d074f6f695c13ffb8646d0b6bb1ab570ba7b0cfd349b921ff03cfc1e"}, - {file = "asyncpg-0.27.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a6206210c869ebd3f4eb9e89bea132aefb56ff3d1b7dd7e26b102b17e27bbb1"}, - {file = "asyncpg-0.27.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7a94c03386bb95456b12c66026b3a87d1b965f0f1e5733c36e7229f8f137747"}, - {file = "asyncpg-0.27.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bfc3980b4ba6f97138b04f0d32e8af21d6c9fa1f8e6e140c07d15690a0a99279"}, - {file = "asyncpg-0.27.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9654085f2b22f66952124de13a8071b54453ff972c25c59b5ce1173a4283ffd9"}, - {file = "asyncpg-0.27.0-cp310-cp310-win32.whl", hash = "sha256:879c29a75969eb2722f94443752f4720d560d1e748474de54ae8dd230bc4956b"}, - {file = "asyncpg-0.27.0-cp310-cp310-win_amd64.whl", hash = "sha256:ab0f21c4818d46a60ca789ebc92327d6d874d3b7ccff3963f7af0a21dc6cff52"}, - {file = "asyncpg-0.27.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:18f77e8e71e826ba2d0c3ba6764930776719ae2b225ca07e014590545928b576"}, - {file = "asyncpg-0.27.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c2232d4625c558f2aa001942cac1d7952aa9f0dbfc212f63bc754277769e1ef2"}, - {file = "asyncpg-0.27.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9a3a4ff43702d39e3c97a8786314123d314e0f0e4dabc8367db5b665c93914de"}, - {file = "asyncpg-0.27.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccddb9419ab4e1c48742457d0c0362dbdaeb9b28e6875115abfe319b29ee225d"}, - {file = "asyncpg-0.27.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:768e0e7c2898d40b16d4ef7a0b44e8150db3dd8995b4652aa1fe2902e92c7df8"}, - {file = "asyncpg-0.27.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609054a1f47292a905582a1cfcca51a6f3f30ab9d822448693e66fdddde27920"}, - {file = "asyncpg-0.27.0-cp311-cp311-win32.whl", hash = "sha256:8113e17cfe236dc2277ec844ba9b3d5312f61bd2fdae6d3ed1c1cdd75f6cf2d8"}, - {file = "asyncpg-0.27.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb71211414dd1eeb8d31ec529fe77cff04bf53efc783a5f6f0a32d84923f45cf"}, - {file = "asyncpg-0.27.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4750f5cf49ed48a6e49c6e5aed390eee367694636c2dcfaf4a273ca832c5c43c"}, - {file = "asyncpg-0.27.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:eca01eb112a39d31cc4abb93a5aef2a81514c23f70956729f42fb83b11b3483f"}, - {file = "asyncpg-0.27.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5710cb0937f696ce303f5eed6d272e3f057339bb4139378ccecafa9ee923a71c"}, - {file = "asyncpg-0.27.0-cp37-cp37m-win_amd64.whl", hash = "sha256:71cca80a056ebe19ec74b7117b09e650990c3ca535ac1c35234a96f65604192f"}, - {file = "asyncpg-0.27.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4bb366ae34af5b5cabc3ac6a5347dfb6013af38c68af8452f27968d49085ecc0"}, - {file = "asyncpg-0.27.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16ba8ec2e85d586b4a12bcd03e8d29e3d99e832764d6a1d0b8c27dbbe4a2569d"}, - {file = "asyncpg-0.27.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d20dea7b83651d93b1eb2f353511fe7fd554752844523f17ad30115d8b9c8cd6"}, - {file = "asyncpg-0.27.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e56ac8a8237ad4adec97c0cd4728596885f908053ab725e22900b5902e7f8e69"}, - {file = "asyncpg-0.27.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bf21ebf023ec67335258e0f3d3ad7b91bb9507985ba2b2206346de488267cad0"}, - {file = "asyncpg-0.27.0-cp38-cp38-win32.whl", hash = "sha256:69aa1b443a182b13a17ff926ed6627af2d98f62f2fe5890583270cc4073f63bf"}, - {file = "asyncpg-0.27.0-cp38-cp38-win_amd64.whl", hash = "sha256:62932f29cf2433988fcd799770ec64b374a3691e7902ecf85da14d5e0854d1ea"}, - {file = "asyncpg-0.27.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fddcacf695581a8d856654bc4c8cfb73d5c9df26d5f55201722d3e6a699e9629"}, - {file = "asyncpg-0.27.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7d8585707ecc6661d07367d444bbaa846b4e095d84451340da8df55a3757e152"}, - {file = "asyncpg-0.27.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:975a320baf7020339a67315284a4d3bf7460e664e484672bd3e71dbd881bc692"}, - {file = "asyncpg-0.27.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2232ebae9796d4600a7819fc383da78ab51b32a092795f4555575fc934c1c89d"}, - {file = "asyncpg-0.27.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:88b62164738239f62f4af92567b846a8ef7cf8abf53eddd83650603de4d52163"}, - {file = "asyncpg-0.27.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:eb4b2fdf88af4fb1cc569781a8f933d2a73ee82cd720e0cb4edabbaecf2a905b"}, - {file = "asyncpg-0.27.0-cp39-cp39-win32.whl", hash = "sha256:8934577e1ed13f7d2d9cea3cc016cc6f95c19faedea2c2b56a6f94f257cea672"}, - {file = "asyncpg-0.27.0-cp39-cp39-win_amd64.whl", hash = "sha256:1b6499de06fe035cf2fa932ec5617ed3f37d4ebbf663b655922e105a484a6af9"}, - {file = "asyncpg-0.27.0.tar.gz", hash = "sha256:720986d9a4705dd8a40fdf172036f5ae787225036a7eb46e704c45aa8f62c054"}, -] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] -attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] -aws-sam-translator = [ - {file = "aws-sam-translator-1.48.0.tar.gz", hash = "sha256:7171037323dfa30f8f73e9bccb9210e4c384a585e087219a9518a5204f0a2c44"}, - {file = "aws_sam_translator-1.48.0-py2-none-any.whl", hash = "sha256:be18dfa3dfe7ab291d281667c5f73ac62dbe6bfe86df7d122e4258b906b736f0"}, - {file = "aws_sam_translator-1.48.0-py3-none-any.whl", hash = "sha256:ca4f8f9910d7713aeaba59346775bfb3198f6acb47c6704572f9bd3fc0fb5bf0"}, -] -aws-xray-sdk = [ - {file = "aws-xray-sdk-2.10.0.tar.gz", hash = "sha256:9b14924fd0628cf92936055864655354003f0b1acc3e1c3ffde6403d0799dd7a"}, - {file = "aws_xray_sdk-2.10.0-py2.py3-none-any.whl", hash = "sha256:7551e81a796e1a5471ebe84844c40e8edf7c218db33506d046fec61f7495eda4"}, -] -backoff = [ - {file = "backoff-1.11.1-py2.py3-none-any.whl", hash = "sha256:61928f8fa48d52e4faa81875eecf308eccfb1016b018bb6bd21e05b5d90a96c5"}, - {file = "backoff-1.11.1.tar.gz", hash = "sha256:ccb962a2378418c667b3c979b504fdeb7d9e0d29c0579e3b13b86467177728cb"}, -] -black = [ - {file = "black-22.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f586c26118bc6e714ec58c09df0157fe2d9ee195c764f630eb0d8e7ccce72e69"}, - {file = "black-22.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b270a168d69edb8b7ed32c193ef10fd27844e5c60852039599f9184460ce0807"}, - {file = "black-22.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6797f58943fceb1c461fb572edbe828d811e719c24e03375fd25170ada53825e"}, - {file = "black-22.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c85928b9d5f83b23cee7d0efcb310172412fbf7cb9d9ce963bd67fd141781def"}, - {file = "black-22.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:f6fe02afde060bbeef044af7996f335fbe90b039ccf3f5eb8f16df8b20f77666"}, - {file = "black-22.6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cfaf3895a9634e882bf9d2363fed5af8888802d670f58b279b0bece00e9a872d"}, - {file = "black-22.6.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94783f636bca89f11eb5d50437e8e17fbc6a929a628d82304c80fa9cd945f256"}, - {file = "black-22.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2ea29072e954a4d55a2ff58971b83365eba5d3d357352a07a7a4df0d95f51c78"}, - {file = "black-22.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e439798f819d49ba1c0bd9664427a05aab79bfba777a6db94fd4e56fae0cb849"}, - {file = "black-22.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:187d96c5e713f441a5829e77120c269b6514418f4513a390b0499b0987f2ff1c"}, - {file = "black-22.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:074458dc2f6e0d3dab7928d4417bb6957bb834434516f21514138437accdbe90"}, - {file = "black-22.6.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a218d7e5856f91d20f04e931b6f16d15356db1c846ee55f01bac297a705ca24f"}, - {file = "black-22.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:568ac3c465b1c8b34b61cd7a4e349e93f91abf0f9371eda1cf87194663ab684e"}, - {file = "black-22.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6c1734ab264b8f7929cef8ae5f900b85d579e6cbfde09d7387da8f04771b51c6"}, - {file = "black-22.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9a3ac16efe9ec7d7381ddebcc022119794872abce99475345c5a61aa18c45ad"}, - {file = "black-22.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:b9fd45787ba8aa3f5e0a0a98920c1012c884622c6c920dbe98dbd05bc7c70fbf"}, - {file = "black-22.6.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7ba9be198ecca5031cd78745780d65a3f75a34b2ff9be5837045dce55db83d1c"}, - {file = "black-22.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3db5b6409b96d9bd543323b23ef32a1a2b06416d525d27e0f67e74f1446c8f2"}, - {file = "black-22.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:560558527e52ce8afba936fcce93a7411ab40c7d5fe8c2463e279e843c0328ee"}, - {file = "black-22.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b154e6bbde1e79ea3260c4b40c0b7b3109ffcdf7bc4ebf8859169a6af72cd70b"}, - {file = "black-22.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4af5bc0e1f96be5ae9bd7aaec219c901a94d6caa2484c21983d043371c733fc4"}, - {file = "black-22.6.0-py3-none-any.whl", hash = "sha256:ac609cf8ef5e7115ddd07d85d988d074ed00e10fbc3445aee393e70164a2219c"}, - {file = "black-22.6.0.tar.gz", hash = "sha256:6c6d39e28aed379aec40da1c65434c77d75e65bb59a1e1c283de545fb4e7c6c9"}, -] -boto3 = [ - {file = "boto3-1.26.16-py3-none-any.whl", hash = "sha256:4f493a2aed71cee93e626de4f67ce58dd82c0473480a0fc45b131715cd8f4f30"}, - {file = "boto3-1.26.16.tar.gz", hash = "sha256:31c0adf71e4bd19a5428580bb229d7ea3b5795eecaa0847a85385df00c026116"}, -] -boto3-stubs = [ - {file = "boto3-stubs-1.26.16.tar.gz", hash = "sha256:618253ae19f1480785759bcaee8c8b10ed3fc037027247c26a3461a50f58406d"}, - {file = "boto3_stubs-1.26.16-py3-none-any.whl", hash = "sha256:8cf2925bc3e1349c93eb0f49c1061affc5ca314d69eeb335349037969d0787ed"}, -] -botocore = [ - {file = "botocore-1.29.16-py3-none-any.whl", hash = "sha256:271b599e6cfe214405ed50d41cd967add1d5d469383dd81ff583bc818b47f59b"}, - {file = "botocore-1.29.16.tar.gz", hash = "sha256:8cfcc10f2f1751608c3cec694f2d6b5e16ebcd50d0a104f9914d5616227c62e9"}, -] -botocore-stubs = [ - {file = "botocore-stubs-1.27.38.tar.gz", hash = "sha256:408e8b86b5d171b58f81c74ca9d3b5317a5a8e2d3bc2073aa841ac13b8939e56"}, - {file = "botocore_stubs-1.27.38-py3-none-any.whl", hash = "sha256:7add7641e9a479a9c8366893bb522fd9ca3d58714201e43662a200a148a1bc38"}, -] -certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] -cffi = [ +files = [ {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, @@ -1869,27 +783,94 @@ cffi = [ {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, ] -cfn-lint = [ + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "cfn-lint" +version = "0.61.3" +description = "Checks CloudFormation templates for practices and behaviour that could potentially be improved" +category = "main" +optional = false +python-versions = ">=3.6, <=4.0, !=4.0" +files = [ {file = "cfn-lint-0.61.3.tar.gz", hash = "sha256:3806e010d77901f5e935496df690c10e39676434a738fce1a1161cf9c7bd36a2"}, {file = "cfn_lint-0.61.3-py3-none-any.whl", hash = "sha256:8e9522fad0c7c98b31ecbdd4724f8d8a5787457cc0f71e62ae0d11104d6e52ab"}, ] -chardet = [ + +[package.dependencies] +aws-sam-translator = ">=1.47.0" +jschema-to-python = ">=1.2.3,<1.3.0" +jsonpatch = "*" +jsonschema = ">=3.0,<4.0" +junit-xml = ">=1.9,<2.0" +networkx = ">=2.4,<3.0" +pyyaml = ">5.4" +sarif-om = ">=1.0.4,<1.1.0" + +[[package]] +name = "chardet" +version = "3.0.4" +description = "Universal encoding detector for Python 2 and 3" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] -charset-normalizer = [ + +[[package]] +name = "charset-normalizer" +version = "2.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.6.0" +files = [ {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, ] -click = [ + +[package.extras] +unicode-backport = ["unicodedata2"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] -colorama = [ + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.5" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -cryptography = [ + +[[package]] +name = "cryptography" +version = "38.0.3" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320"}, {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722"}, {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f"}, @@ -1917,91 +898,363 @@ cryptography = [ {file = "cryptography-38.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a"}, {file = "cryptography-38.0.3.tar.gz", hash = "sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd"}, ] -docker = [ + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "docker" +version = "4.2.2" +description = "A Python library for the Docker Engine API." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ {file = "docker-4.2.2-py2.py3-none-any.whl", hash = "sha256:03a46400c4080cb6f7aa997f881ddd84fef855499ece219d75fbdb53289c17ab"}, {file = "docker-4.2.2.tar.gz", hash = "sha256:26eebadce7e298f55b76a88c4f8802476c5eaddbdbe38dbc6cce8781c47c9b54"}, ] -ecdsa = [ + +[package.dependencies] +pypiwin32 = {version = "223", markers = "sys_platform == \"win32\" and python_version >= \"3.6\""} +requests = ">=2.14.2,<2.18.0 || >2.18.0" +six = ">=1.4.0" +websocket-client = ">=0.32.0" + +[package.extras] +ssh = ["paramiko (>=2.4.2)"] +tls = ["cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] + +[[package]] +name = "ecdsa" +version = "0.18.0" +description = "ECDSA cryptographic signature library (pure python)" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "ecdsa-0.18.0-py2.py3-none-any.whl", hash = "sha256:80600258e7ed2f16b9aa1d7c295bd70194109ad5a30fdee0eaeefef1d4c559dd"}, {file = "ecdsa-0.18.0.tar.gz", hash = "sha256:190348041559e21b22a1d65cee485282ca11a6f81d503fddb84d5017e9ed1e49"}, ] -execnet = [ + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +gmpy = ["gmpy"] +gmpy2 = ["gmpy2"] + +[[package]] +name = "execnet" +version = "1.9.0" +description = "execnet: rapid multi-Python deployment" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ {file = "execnet-1.9.0-py2.py3-none-any.whl", hash = "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"}, {file = "execnet-1.9.0.tar.gz", hash = "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5"}, ] -flake8 = [ + +[package.extras] +testing = ["pre-commit"] + +[[package]] +name = "flake8" +version = "5.0.4" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = ">=3.6.1" +files = [ {file = "flake8-5.0.4-py2.py3-none-any.whl", hash = "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248"}, {file = "flake8-5.0.4.tar.gz", hash = "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db"}, ] -flask = [ + +[package.dependencies] +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.9.0,<2.10.0" +pyflakes = ">=2.5.0,<2.6.0" + +[[package]] +name = "flask" +version = "2.1.3" +description = "A simple framework for building complex web applications." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "Flask-2.1.3-py3-none-any.whl", hash = "sha256:9013281a7402ad527f8fd56375164f3aa021ecfaff89bfe3825346c24f87e04c"}, {file = "Flask-2.1.3.tar.gz", hash = "sha256:15972e5017df0575c3d6c090ba168b6db90259e620ac8d7ea813a396bad5b6cb"}, ] -flask-cors = [ + +[package.dependencies] +click = ">=8.0" +importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} +itsdangerous = ">=2.0" +Jinja2 = ">=3.0" +Werkzeug = ">=2.0" + +[package.extras] +async = ["asgiref (>=3.2)"] +dotenv = ["python-dotenv"] + +[[package]] +name = "flask-cors" +version = "3.0.10" +description = "A Flask extension adding a decorator for CORS support" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "Flask-Cors-3.0.10.tar.gz", hash = "sha256:b60839393f3b84a0f3746f6cdca56c1ad7426aa738b70d6c61375857823181de"}, {file = "Flask_Cors-3.0.10-py2.py3-none-any.whl", hash = "sha256:74efc975af1194fc7891ff5cd85b0f7478be4f7f59fe158102e91abb72bb4438"}, ] -graphql-core = [ + +[package.dependencies] +Flask = ">=0.9" +Six = "*" + +[[package]] +name = "graphql-core" +version = "3.2.1" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +category = "main" +optional = false +python-versions = ">=3.6,<4" +files = [ {file = "graphql-core-3.2.1.tar.gz", hash = "sha256:9d1bf141427b7d54be944587c8349df791ce60ade2e3cccaf9c56368c133c201"}, {file = "graphql_core-3.2.1-py3-none-any.whl", hash = "sha256:f83c658e4968998eed1923a2e3e3eddd347e005ac0315fbb7ca4d70ea9156323"}, ] -idna = [ + +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -importlib-metadata = [ + +[[package]] +name = "importlib-metadata" +version = "4.12.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, ] -iniconfig = [ + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -isort = [ + +[[package]] +name = "isort" +version = "5.10.1" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.6.1,<4.0" +files = [ {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"}, {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"}, ] -itsdangerous = [ + +[package.extras] +colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] +plugins = ["setuptools"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] + +[[package]] +name = "itsdangerous" +version = "2.1.2" +description = "Safely pass data to untrusted environments and back." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, ] -jinja2 = [ + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [ + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] -jschema-to-python = [ + +[[package]] +name = "jschema-to-python" +version = "1.2.3" +description = "Generate source code for Python classes from a JSON schema." +category = "main" +optional = false +python-versions = ">= 2.7" +files = [ {file = "jschema_to_python-1.2.3-py3-none-any.whl", hash = "sha256:8a703ca7604d42d74b2815eecf99a33359a8dccbb80806cce386d5e2dd992b05"}, {file = "jschema_to_python-1.2.3.tar.gz", hash = "sha256:76ff14fe5d304708ccad1284e4b11f96a658949a31ee7faed9e0995279549b91"}, ] -jsondiff = [ + +[package.dependencies] +attrs = "*" +jsonpickle = "*" +pbr = "*" + +[[package]] +name = "jsondiff" +version = "2.0.0" +description = "Diff JSON and JSON-like structures in Python" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "jsondiff-2.0.0-py3-none-any.whl", hash = "sha256:689841d66273fc88fc79f7d33f4c074774f4f214b6466e3aff0e5adaf889d1e0"}, {file = "jsondiff-2.0.0.tar.gz", hash = "sha256:2795844ef075ec8a2b8d385c4d59f5ea48b08e7180fce3cb2787be0db00b1fb4"}, ] -jsonpatch = [ + +[[package]] +name = "jsonpatch" +version = "1.32" +description = "Apply JSON-Patches (RFC 6902)" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ {file = "jsonpatch-1.32-py2.py3-none-any.whl", hash = "sha256:26ac385719ac9f54df8a2f0827bb8253aa3ea8ab7b3368457bcdb8c14595a397"}, {file = "jsonpatch-1.32.tar.gz", hash = "sha256:b6ddfe6c3db30d81a96aaeceb6baf916094ffa23d7dd5fa2c13e13f8b6e600c2"}, ] -jsonpickle = [ + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpickle" +version = "2.2.0" +description = "Python library for serializing any arbitrary object graph into JSON" +category = "main" +optional = false +python-versions = ">=2.7" +files = [ {file = "jsonpickle-2.2.0-py2.py3-none-any.whl", hash = "sha256:de7f2613818aa4f234138ca11243d6359ff83ae528b2185efdd474f62bcf9ae1"}, {file = "jsonpickle-2.2.0.tar.gz", hash = "sha256:7b272918b0554182e53dc340ddd62d9b7f902fec7e7b05620c04f3ccef479a0e"}, ] -jsonpointer = [ + +[package.extras] +docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["ecdsa", "enum34", "feedparser", "jsonlib", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (<1.1.0)", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] +testing-libs = ["simplejson", "ujson", "yajl"] + +[[package]] +name = "jsonpointer" +version = "2.3" +description = "Identify specific nodes in a JSON document (RFC 6901)" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "jsonpointer-2.3-py2.py3-none-any.whl", hash = "sha256:51801e558539b4e9cd268638c078c6c5746c9ac96bc38152d443400e4f3793e9"}, {file = "jsonpointer-2.3.tar.gz", hash = "sha256:97cba51526c829282218feb99dab1b1e6bdf8efd1c43dc9d57be093c0d69c99a"}, ] -jsonschema = [ + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, ] -junit-xml = [ - {file = "junit-xml-1.9.tar.gz", hash = "sha256:de16a051990d4e25a3982b2dd9e89d671067548718866416faec14d9de56db9f"}, + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "junit-xml" +version = "1.9" +description = "Creates JUnit XML test result documents that can be read by tools such as Jenkins" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "junit_xml-1.9-py2.py3-none-any.whl", hash = "sha256:ec5ca1a55aefdd76d28fcc0b135251d156c7106fa979686a4b48d62b761b4732"}, ] -markupsafe = [ + +[package.dependencies] +six = "*" + +[[package]] +name = "markupsafe" +version = "2.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, @@ -2043,15 +1296,91 @@ markupsafe = [ {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] -mccabe = [ + +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] -moto = [ + +[[package]] +name = "moto" +version = "3.1.18" +description = "A library that allows your python tests to easily mock out the boto library" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "moto-3.1.18-py3-none-any.whl", hash = "sha256:b6eb096e7880c46ac44d6d90988c0043e31462115cfdc913a0ee8f470bd9555c"}, {file = "moto-3.1.18.tar.gz", hash = "sha256:1e05276a62aa5a4aa821b441647c2cbaa2ea175388980b10d5de88d41b327cf7"}, ] -multidict = [ + +[package.dependencies] +aws-xray-sdk = {version = ">=0.93,<0.96 || >0.96", optional = true, markers = "extra == \"server\""} +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cfn-lint = {version = ">=0.4.0", optional = true, markers = "extra == \"server\""} +cryptography = ">=3.3.1" +docker = {version = ">=2.5.1", optional = true, markers = "extra == \"server\""} +ecdsa = {version = "!=0.15", optional = true, markers = "extra == \"server\""} +flask = {version = "<2.2.0", optional = true, markers = "extra == \"server\""} +flask-cors = {version = "*", optional = true, markers = "extra == \"server\""} +graphql-core = {version = "*", optional = true, markers = "extra == \"server\""} +idna = {version = ">=2.5,<4", optional = true, markers = "extra == \"server\""} +Jinja2 = ">=2.10.1" +jsondiff = {version = ">=1.1.2", optional = true, markers = "extra == \"server\""} +MarkupSafe = "!=2.0.0a1" +openapi-spec-validator = {version = ">=0.2.8", optional = true, markers = "extra == \"server\""} +pyparsing = {version = ">=3.0.7", optional = true, markers = "extra == \"server\""} +python-dateutil = ">=2.1,<3.0.0" +python-jose = {version = ">=3.1.0,<4.0.0", extras = ["cryptography"], optional = true, markers = "extra == \"server\""} +pytz = "*" +PyYAML = {version = ">=5.1", optional = true, markers = "extra == \"server\""} +requests = ">=2.5" +responses = ">=0.9.0" +setuptools = {version = "*", optional = true, markers = "extra == \"server\""} +sshpubkeys = {version = ">=3.1.0", optional = true, markers = "extra == \"server\""} +werkzeug = ">=0.5,<2.2.0" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=2.5.1)"] +batch = ["docker (>=2.5.1)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=2.5.1)"] +dynamodb2 = ["docker (>=2.5.1)"] +dynamodbstreams = ["docker (>=2.5.1)"] +ebs = ["sshpubkeys (>=3.1.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +efs = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +route53resolver = ["sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (<2.2.0)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)", "dataclasses"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, @@ -2127,7 +1456,15 @@ multidict = [ {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] -mypy = [ + +[[package]] +name = "mypy" +version = "0.991" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, @@ -2159,51 +1496,199 @@ mypy = [ {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, ] -mypy-boto3-s3 = [ + +[package.dependencies] +mypy-extensions = ">=0.4.3" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + +[[package]] +name = "mypy-boto3-s3" +version = "1.26.0.post1" +description = "Type annotations for boto3.S3 1.26.0 service generated with mypy-boto3-builder 7.11.10" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "mypy-boto3-s3-1.26.0.post1.tar.gz", hash = "sha256:6d7079f8c739dc993cbedad0736299c413b297814b73795a3855a79169ecc938"}, {file = "mypy_boto3_s3-1.26.0.post1-py3-none-any.whl", hash = "sha256:7de2792ff0cc541b84cd46ff3a6aa2b6e5f267217f2203f27f6e4016bddc644d"}, ] -mypy-extensions = [ + +[package.dependencies] +typing-extensions = ">=4.1.0" + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] -networkx = [ + +[[package]] +name = "networkx" +version = "2.8.5" +description = "Python package for creating and manipulating graphs and networks" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ {file = "networkx-2.8.5-py3-none-any.whl", hash = "sha256:a762f4b385692d9c3a6f2912d058d76d29a827deaedf9e63ed14d397b8030687"}, {file = "networkx-2.8.5.tar.gz", hash = "sha256:15a7b81a360791c458c55a417418ea136c13378cfdc06a2dcdc12bd2f9cf09c1"}, ] -openapi-schema-validator = [ + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.19)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=0.960)", "pre-commit (>=2.19)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.4)", "pillow (>=9.1)", "pydata-sphinx-theme (>=0.9)", "sphinx (>=5)", "sphinx-gallery (>=0.10)", "texext (>=0.6.6)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.9)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.1)", "pytest-cov (>=3.0)"] + +[[package]] +name = "openapi-schema-validator" +version = "0.2.3" +description = "OpenAPI schema validation for Python" +category = "main" +optional = false +python-versions = ">=3.7.0,<4.0.0" +files = [ {file = "openapi-schema-validator-0.2.3.tar.gz", hash = "sha256:2c64907728c3ef78e23711c8840a423f0b241588c9ed929855e4b2d1bb0cf5f2"}, {file = "openapi_schema_validator-0.2.3-py3-none-any.whl", hash = "sha256:9bae709212a19222892cabcc60cafd903cbf4b220223f48583afa3c0e3cc6fc4"}, ] -openapi-spec-validator = [ + +[package.dependencies] +jsonschema = ">=3.0.0,<5.0.0" + +[package.extras] +isodate = ["isodate"] +rfc3339-validator = ["rfc3339-validator"] +strict-rfc3339 = ["strict-rfc3339"] + +[[package]] +name = "openapi-spec-validator" +version = "0.4.0" +description = "OpenAPI 2.0 (aka Swagger) and OpenAPI 3.0 spec validator" +category = "main" +optional = false +python-versions = ">=3.7.0,<4.0.0" +files = [ {file = "openapi-spec-validator-0.4.0.tar.gz", hash = "sha256:97f258850afc97b048f7c2653855e0f88fa66ac103c2be5077c7960aca2ad49a"}, {file = "openapi_spec_validator-0.4.0-py3-none-any.whl", hash = "sha256:06900ac4d546a1df3642a779da0055be58869c598e3042a2fef067cfd99d04d0"}, ] -packaging = [ + +[package.dependencies] +jsonschema = ">=3.2.0,<5.0.0" +openapi-schema-validator = ">=0.2.0,<0.3.0" +PyYAML = ">=5.1" +setuptools = "*" + +[package.extras] +requests = ["requests"] + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pathspec = [ + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "pathspec" +version = "0.9.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, ] -pbr = [ + +[[package]] +name = "pbr" +version = "5.9.0" +description = "Python Build Reasonableness" +category = "main" +optional = false +python-versions = ">=2.6" +files = [ {file = "pbr-5.9.0-py2.py3-none-any.whl", hash = "sha256:e547125940bcc052856ded43be8e101f63828c2d94239ffbe2b327ba3d5ccf0a"}, {file = "pbr-5.9.0.tar.gz", hash = "sha256:e8dca2f4b43560edef58813969f52a56cef023146cbb8931626db80e6c1c4308"}, ] -platformdirs = [ + +[[package]] +name = "platformdirs" +version = "2.5.2" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, ] -pluggy = [ + +[package.extras] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)"] +test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -prometheus-client = [ + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "prometheus-client" +version = "0.14.1" +description = "Python client for the Prometheus monitoring system." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "prometheus_client-0.14.1-py3-none-any.whl", hash = "sha256:522fded625282822a89e2773452f42df14b5a8e84a86433e3f8a189c1d54dc01"}, {file = "prometheus_client-0.14.1.tar.gz", hash = "sha256:5459c427624961076277fdc6dc50540e2bacb98eebde99886e59ec55ed92093a"}, ] -psutil = [ + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "psutil" +version = "5.9.4" +description = "Cross-platform lib for process and system monitoring in Python." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"}, {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe"}, {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549"}, @@ -2219,10 +1704,20 @@ psutil = [ {file = "psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e"}, {file = "psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"}, ] -psycopg2-binary = [ + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "psycopg2-binary" +version = "2.9.3" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "psycopg2-binary-2.9.3.tar.gz", hash = "sha256:761df5313dc15da1502b21453642d7599d26be88bff659382f8f9747c7ebea4e"}, {file = "psycopg2_binary-2.9.3-cp310-cp310-macosx_10_14_x86_64.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:539b28661b71da7c0e428692438efbcd048ca21ea81af618d845e06ebfd29478"}, - {file = "psycopg2_binary-2.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2f2534ab7dc7e776a263b463a16e189eb30e85ec9bbe1bff9e78dae802608932"}, {file = "psycopg2_binary-2.9.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e82d38390a03da28c7985b394ec3f56873174e2c88130e6966cb1c946508e65"}, {file = "psycopg2_binary-2.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57804fc02ca3ce0dbfbef35c4b3a4a774da66d66ea20f4bda601294ad2ea6092"}, {file = "psycopg2_binary-2.9.3-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:083a55275f09a62b8ca4902dd11f4b33075b743cf0d360419e2051a8a5d5ff76"}, @@ -2256,7 +1751,6 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.9.3-cp37-cp37m-win32.whl", hash = "sha256:adf20d9a67e0b6393eac162eb81fb10bc9130a80540f4df7e7355c2dd4af9fba"}, {file = "psycopg2_binary-2.9.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2f9ffd643bc7349eeb664eba8864d9e01f057880f510e4681ba40a6532f93c71"}, {file = "psycopg2_binary-2.9.3-cp38-cp38-macosx_10_14_x86_64.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:def68d7c21984b0f8218e8a15d514f714d96904265164f75f8d3a70f9c295667"}, - {file = "psycopg2_binary-2.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e6aa71ae45f952a2205377773e76f4e3f27951df38e69a4c95440c779e013560"}, {file = "psycopg2_binary-2.9.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dffc08ca91c9ac09008870c9eb77b00a46b3378719584059c034b8945e26b272"}, {file = "psycopg2_binary-2.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:280b0bb5cbfe8039205c7981cceb006156a675362a00fe29b16fbc264e242834"}, {file = "psycopg2_binary-2.9.3-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:af9813db73395fb1fc211bac696faea4ca9ef53f32dc0cfa27e4e7cf766dcf24"}, @@ -2268,7 +1762,6 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.9.3-cp38-cp38-win32.whl", hash = "sha256:6472a178e291b59e7f16ab49ec8b4f3bdada0a879c68d3817ff0963e722a82ce"}, {file = "psycopg2_binary-2.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:35168209c9d51b145e459e05c31a9eaeffa9a6b0fd61689b48e07464ffd1a83e"}, {file = "psycopg2_binary-2.9.3-cp39-cp39-macosx_10_14_x86_64.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:47133f3f872faf28c1e87d4357220e809dfd3fa7c64295a4a148bcd1e6e34ec9"}, - {file = "psycopg2_binary-2.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b3a24a1982ae56461cc24f6680604fffa2c1b818e9dc55680da038792e004d18"}, {file = "psycopg2_binary-2.9.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91920527dea30175cc02a1099f331aa8c1ba39bf8b7762b7b56cbf54bc5cce42"}, {file = "psycopg2_binary-2.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:887dd9aac71765ac0d0bac1d0d4b4f2c99d5f5c1382d8b770404f0f3d0ce8a39"}, {file = "psycopg2_binary-2.9.3-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:1f14c8b0942714eb3c74e1e71700cbbcb415acbc311c730370e70c578a44a25c"}, @@ -2280,39 +1773,137 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.9.3-cp39-cp39-win32.whl", hash = "sha256:46f0e0a6b5fa5851bbd9ab1bc805eef362d3a230fbdfbc209f4a236d0a7a990d"}, {file = "psycopg2_binary-2.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:accfe7e982411da3178ec690baaceaad3c278652998b2c45828aaac66cd8285f"}, ] -py = [ + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyasn1 = [ + +[[package]] +name = "pyasn1" +version = "0.4.8" +description = "ASN.1 types and codecs" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, + {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, + {file = "pyasn1-0.4.8-py2.6.egg", hash = "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00"}, + {file = "pyasn1-0.4.8-py2.7.egg", hash = "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8"}, {file = "pyasn1-0.4.8-py2.py3-none-any.whl", hash = "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d"}, + {file = "pyasn1-0.4.8-py3.1.egg", hash = "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86"}, + {file = "pyasn1-0.4.8-py3.2.egg", hash = "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7"}, + {file = "pyasn1-0.4.8-py3.3.egg", hash = "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576"}, + {file = "pyasn1-0.4.8-py3.4.egg", hash = "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12"}, + {file = "pyasn1-0.4.8-py3.5.egg", hash = "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2"}, + {file = "pyasn1-0.4.8-py3.6.egg", hash = "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359"}, + {file = "pyasn1-0.4.8-py3.7.egg", hash = "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776"}, {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, ] -pycodestyle = [ + +[[package]] +name = "pycodestyle" +version = "2.9.1" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "pycodestyle-2.9.1-py2.py3-none-any.whl", hash = "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b"}, {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, ] -pycparser = [ + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pyflakes = [ + +[[package]] +name = "pyflakes" +version = "2.5.0" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2"}, {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, ] -pyjwt = [ + +[[package]] +name = "pyjwt" +version = "2.4.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "PyJWT-2.4.0-py3-none-any.whl", hash = "sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf"}, {file = "PyJWT-2.4.0.tar.gz", hash = "sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba"}, ] -pyparsing = [ + +[package.dependencies] +cryptography = {version = ">=3.3.1", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.3.1)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.3.1)", "mypy", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" +files = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] -pypiwin32 = [ + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pypiwin32" +version = "223" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "pypiwin32-223-py3-none-any.whl", hash = "sha256:67adf399debc1d5d14dffc1ab5acacb800da569754fafdc576b2a039485aa775"}, {file = "pypiwin32-223.tar.gz", hash = "sha256:71be40c1fbd28594214ecaecb58e7aa8b708eabfa0125c8a109ebd51edbd776a"}, ] -pyrsistent = [ + +[package.dependencies] +pywin32 = ">=223" + +[[package]] +name = "pyrsistent" +version = "0.18.1" +description = "Persistent/Functional/Immutable data structures" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "pyrsistent-0.18.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1"}, {file = "pyrsistent-0.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26"}, {file = "pyrsistent-0.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e"}, @@ -2335,47 +1926,192 @@ pyrsistent = [ {file = "pyrsistent-0.18.1-cp39-cp39-win_amd64.whl", hash = "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07"}, {file = "pyrsistent-0.18.1.tar.gz", hash = "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96"}, ] -pytest = [ + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] -pytest-asyncio = [ + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "0.19.0" +description = "Pytest support for asyncio" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-asyncio-0.19.0.tar.gz", hash = "sha256:ac4ebf3b6207259750bc32f4c1d8fcd7e79739edbc67ad0c58dd150b1d072fed"}, {file = "pytest_asyncio-0.19.0-py3-none-any.whl", hash = "sha256:7a97e37cfe1ed296e2e84941384bdd37c376453912d397ed39293e0916f521fa"}, ] -pytest-httpserver = [ + +[package.dependencies] +pytest = ">=6.1.0" + +[package.extras] +testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] + +[[package]] +name = "pytest-httpserver" +version = "1.0.6" +description = "pytest-httpserver is a httpserver for pytest" +category = "main" +optional = false +python-versions = ">=3.7,<4.0" +files = [ {file = "pytest_httpserver-1.0.6-py3-none-any.whl", hash = "sha256:ac2379acc91fe8bdbe2911c93af8dd130e33b5899fb9934d15669480739c6d32"}, {file = "pytest_httpserver-1.0.6.tar.gz", hash = "sha256:9040d07bf59ac45d8de3db1d4468fd2d1d607975e4da4c872ecc0402cdbf7b3e"}, ] -pytest-lazy-fixture = [ + +[package.dependencies] +Werkzeug = ">=2.0.0" + +[[package]] +name = "pytest-lazy-fixture" +version = "0.6.3" +description = "It helps to use fixtures in pytest.mark.parametrize" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "pytest-lazy-fixture-0.6.3.tar.gz", hash = "sha256:0e7d0c7f74ba33e6e80905e9bfd81f9d15ef9a790de97993e34213deb5ad10ac"}, {file = "pytest_lazy_fixture-0.6.3-py3-none-any.whl", hash = "sha256:e0b379f38299ff27a653f03eaa69b08a6fd4484e46fd1c9907d984b9f9daeda6"}, ] -pytest-order = [ + +[package.dependencies] +pytest = ">=3.2.5" + +[[package]] +name = "pytest-order" +version = "1.0.1" +description = "pytest plugin to run your tests in a specific order" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "pytest-order-1.0.1.tar.gz", hash = "sha256:5dd6b929fbd7eaa6d0ee07586f65c623babb0afe72b4843c5f15055d6b3b1b1f"}, {file = "pytest_order-1.0.1-py3-none-any.whl", hash = "sha256:bbe6e63a8e23741ab3e810d458d1ea7317e797b70f9550512d77d6e9e8fd1bbb"}, ] -pytest-timeout = [ + +[package.dependencies] +pytest = [ + {version = ">=6.2.4", markers = "python_version >= \"3.10\""}, + {version = ">=5.0", markers = "python_version < \"3.10\""}, +] + +[[package]] +name = "pytest-timeout" +version = "2.1.0" +description = "pytest plugin to abort hanging tests" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "pytest-timeout-2.1.0.tar.gz", hash = "sha256:c07ca07404c612f8abbe22294b23c368e2e5104b521c1790195561f37e1ac3d9"}, {file = "pytest_timeout-2.1.0-py3-none-any.whl", hash = "sha256:f6f50101443ce70ad325ceb4473c4255e9d74e3c7cd0ef827309dfa4c0d975c6"}, ] -pytest-xdist = [ + +[package.dependencies] +pytest = ">=5.0.0" + +[[package]] +name = "pytest-xdist" +version = "3.0.2" +description = "pytest xdist plugin for distributed testing and loop-on-failing modes" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "pytest-xdist-3.0.2.tar.gz", hash = "sha256:688da9b814370e891ba5de650c9327d1a9d861721a524eb917e620eec3e90291"}, {file = "pytest_xdist-3.0.2-py3-none-any.whl", hash = "sha256:9feb9a18e1790696ea23e1434fa73b325ed4998b0e9fcb221f16fd1945e6df1b"}, ] -python-dateutil = [ + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.2.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] -python-jose = [ + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-jose" +version = "3.3.0" +description = "JOSE implementation in Python" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "python-jose-3.3.0.tar.gz", hash = "sha256:55779b5e6ad599c6336191246e95eb2293a9ddebd555f796a65f838f07e5d78a"}, {file = "python_jose-3.3.0-py2.py3-none-any.whl", hash = "sha256:9b1376b023f8b298536eedd47ae1089bcdb848f1535ab30555cd92002d78923a"}, ] -pytz = [ + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"cryptography\""} +ecdsa = "!=0.15" +pyasn1 = "*" +rsa = "*" + +[package.extras] +cryptography = ["cryptography (>=3.4.0)"] +pycrypto = ["pyasn1", "pycrypto (>=2.6.0,<2.7.0)"] +pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] + +[[package]] +name = "pytz" +version = "2022.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, ] -pywin32 = [ + +[[package]] +name = "pywin32" +version = "301" +description = "Python for Window Extensions" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "pywin32-301-cp35-cp35m-win32.whl", hash = "sha256:93367c96e3a76dfe5003d8291ae16454ca7d84bb24d721e0b74a07610b7be4a7"}, {file = "pywin32-301-cp35-cp35m-win_amd64.whl", hash = "sha256:9635df6998a70282bd36e7ac2a5cef9ead1627b0a63b17c731312c7a0daebb72"}, {file = "pywin32-301-cp36-cp36m-win32.whl", hash = "sha256:c866f04a182a8cb9b7855de065113bbd2e40524f570db73ef1ee99ff0a5cc2f0"}, @@ -2387,7 +2123,15 @@ pywin32 = [ {file = "pywin32-301-cp39-cp39-win32.whl", hash = "sha256:595d397df65f1b2e0beaca63a883ae6d8b6df1cdea85c16ae85f6d2e648133fe"}, {file = "pywin32-301-cp39-cp39-win_amd64.whl", hash = "sha256:87604a4087434cd814ad8973bd47d6524bd1fa9e971ce428e76b62a5e0860fdf"}, ] -pyyaml = [ + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -2395,13 +2139,6 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2429,87 +2166,313 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -requests = [ + +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" +files = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -responses = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "responses" +version = "0.21.0" +description = "A utility library for mocking out the `requests` Python library." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "responses-0.21.0-py3-none-any.whl", hash = "sha256:2dcc863ba63963c0c3d9ee3fa9507cbe36b7d7b0fccb4f0bdfd9e96c539b1487"}, {file = "responses-0.21.0.tar.gz", hash = "sha256:b82502eb5f09a0289d8e209e7bad71ef3978334f56d09b444253d5ad67bf5253"}, ] -rsa = [ + +[package.dependencies] +requests = ">=2.0,<3.0" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-localserver", "types-mock", "types-requests"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +category = "main" +optional = false +python-versions = ">=3.6,<4" +files = [ {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, ] -s3transfer = [ + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, ] -sarif-om = [ + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "sarif-om" +version = "1.0.4" +description = "Classes implementing the SARIF 2.1.0 object model." +category = "main" +optional = false +python-versions = ">= 2.7" +files = [ {file = "sarif_om-1.0.4-py3-none-any.whl", hash = "sha256:539ef47a662329b1c8502388ad92457425e95dc0aaaf995fe46f4984c4771911"}, {file = "sarif_om-1.0.4.tar.gz", hash = "sha256:cd5f416b3083e00d402a92e449a7ff67af46f11241073eea0461802a3b5aef98"}, ] -setuptools = [ + +[package.dependencies] +attrs = "*" +pbr = "*" + +[[package]] +name = "setuptools" +version = "65.5.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, ] -six = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -sshpubkeys = [ + +[[package]] +name = "sshpubkeys" +version = "3.3.1" +description = "SSH public key parser" +category = "main" +optional = false +python-versions = ">=3" +files = [ {file = "sshpubkeys-3.3.1-py2.py3-none-any.whl", hash = "sha256:946f76b8fe86704b0e7c56a00d80294e39bc2305999844f079a217885060b1ac"}, {file = "sshpubkeys-3.3.1.tar.gz", hash = "sha256:3020ed4f8c846849299370fbe98ff4157b0ccc1accec105e07cfa9ae4bb55064"}, ] -toml = [ + +[package.dependencies] +cryptography = ">=2.1.4" +ecdsa = ">=0.13" + +[package.extras] +dev = ["twine", "wheel", "yapf"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -types-psutil = [ + +[[package]] +name = "types-psutil" +version = "5.9.5.4" +description = "Typing stubs for psutil" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "types-psutil-5.9.5.4.tar.gz", hash = "sha256:aa09102b80c65a3b4573216614372398dab78972d650488eaff1ff05482cc18f"}, {file = "types_psutil-5.9.5.4-py3-none-any.whl", hash = "sha256:28e59764630187e462d43788efa16d59d5e77b510115f9e25901b2d4007fca62"}, ] -types-psycopg2 = [ + +[[package]] +name = "types-psycopg2" +version = "2.9.18" +description = "Typing stubs for psycopg2" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "types-psycopg2-2.9.18.tar.gz", hash = "sha256:9b0e9e1f097b15cd9fa8aad2596a9e3082fd72f8d9cfe52b190cfa709105b6c0"}, {file = "types_psycopg2-2.9.18-py3-none-any.whl", hash = "sha256:14c779dcab18c31453fa1cad3cf4b1601d33540a344adead3c47a6b8091cd2fa"}, ] -types-requests = [ + +[[package]] +name = "types-requests" +version = "2.28.5" +description = "Typing stubs for requests" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "types-requests-2.28.5.tar.gz", hash = "sha256:ac618bfefcb3742eaf97c961e13e9e5a226e545eda4a3dbe293b898d40933ad1"}, {file = "types_requests-2.28.5-py3-none-any.whl", hash = "sha256:98ab647ae88b5e2c41d6d20cfcb5117da1bea561110000b6fdeeea07b3e89877"}, ] -types-s3transfer = [ + +[package.dependencies] +types-urllib3 = "<1.27" + +[[package]] +name = "types-s3transfer" +version = "0.6.0.post3" +description = "Type annotations and code completion for s3transfer" +category = "main" +optional = false +python-versions = ">=3.7,<4.0" +files = [ {file = "types-s3transfer-0.6.0.post3.tar.gz", hash = "sha256:92c3704e5d041202bfb5ddb79d083fd1a02de2c5dfec6a91576823e6b5c93993"}, {file = "types_s3transfer-0.6.0.post3-py3-none-any.whl", hash = "sha256:eedc5117275565b3c83662c0ccc81662a34da5dda8bd502b89d296b6d5cb091d"}, ] -types-toml = [ + +[[package]] +name = "types-toml" +version = "0.10.8" +description = "Typing stubs for toml" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "types-toml-0.10.8.tar.gz", hash = "sha256:b7e7ea572308b1030dc86c3ba825c5210814c2825612ec679eb7814f8dd9295a"}, {file = "types_toml-0.10.8-py3-none-any.whl", hash = "sha256:8300fd093e5829eb9c1fba69cee38130347d4b74ddf32d0a7df650ae55c2b599"}, ] -types-urllib3 = [ + +[[package]] +name = "types-urllib3" +version = "1.26.17" +description = "Typing stubs for urllib3" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "types-urllib3-1.26.17.tar.gz", hash = "sha256:73fd274524c3fc7cd8cd9ceb0cb67ed99b45f9cb2831013e46d50c1451044800"}, {file = "types_urllib3-1.26.17-py3-none-any.whl", hash = "sha256:0d027fcd27dbb3cb532453b4d977e05bc1e13aefd70519866af211b3003d895d"}, ] -typing-extensions = [ + +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [ + +[[package]] +name = "urllib3" +version = "1.26.11" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +files = [ {file = "urllib3-1.26.11-py2.py3-none-any.whl", hash = "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc"}, {file = "urllib3-1.26.11.tar.gz", hash = "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"}, ] -websocket-client = [ + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "websocket-client" +version = "1.3.3" +description = "WebSocket client for Python with low level API options" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "websocket-client-1.3.3.tar.gz", hash = "sha256:d58c5f284d6a9bf8379dab423259fe8f85b70d5fa5d2916d5791a84594b122b1"}, {file = "websocket_client-1.3.3-py3-none-any.whl", hash = "sha256:5d55652dc1d0b3c734f044337d929aaf83f4f9138816ec680c1aefefb4dc4877"}, ] -werkzeug = [ + +[package.extras] +docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "werkzeug" +version = "2.1.2" +description = "The comprehensive WSGI web application library." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "Werkzeug-2.1.2-py3-none-any.whl", hash = "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255"}, {file = "Werkzeug-2.1.2.tar.gz", hash = "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6"}, ] -wrapt = [ + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, @@ -2575,11 +2538,27 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -xmltodict = [ + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "main" +optional = false +python-versions = ">=3.4" +files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -yarl = [ + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, @@ -2655,7 +2634,28 @@ yarl = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] -zipp = [ + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.8.1" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, ] + +[package.extras] +docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "7563a38912963d8cf20c99acb06fe55623e65b799c4b88d37dc672e5384c96a3" From bf1c36a30c5fc9f534f2b54eb412ee9414ab9df6 Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Thu, 2 Feb 2023 11:02:47 +0200 Subject: [PATCH 082/102] Moving the template file location (#3523) see https://github.com/appsmithorg/appsmith/issus/826#issuecomment-703093426 for details --- .github/{PULL_REQUEST_TEMPLATE => }/pull_request_template.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{PULL_REQUEST_TEMPLATE => }/pull_request_template.md (100%) diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/pull_request_template.md similarity index 100% rename from .github/PULL_REQUEST_TEMPLATE/pull_request_template.md rename to .github/pull_request_template.md From f474495ba058c3f46270b2c9d1d9f8444fa1c920 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 2 Feb 2023 11:18:42 +0200 Subject: [PATCH 083/102] Publish builds stats that are easy to browse (#3514) Adds two new tags, `run-extra-build-macos` and `run-extra-build-stats` to trigger corresponding build jobs on any PR. On every build for `main` or PR with `run-extra-build-stats` tag, publish a GitHub commit status with the link to the `cargo build --all --release --timings` report. --- .github/workflows/neon_extra_builds.yml | 46 +++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index b8600e0665..2ae517e5e7 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -4,6 +4,7 @@ on: push: branches: - main + pull_request: defaults: run: @@ -20,6 +21,7 @@ env: jobs: check-macos-build: + if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') timeout-minutes: 90 runs-on: macos-latest @@ -93,11 +95,16 @@ jobs: run: ./run_clippy.sh gather-rust-build-stats: - timeout-minutes: 90 - runs-on: ubuntu-latest + if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') + runs-on: [ self-hosted, gen3, large ] + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init env: BUILD_TYPE: release + # remove the cachepot wrapper and build without crate caches + RUSTC_WRAPPER: "" # build with incremental compilation produce partial results # so do not attempt to cache this build, also disable the incremental compilation CARGO_INCREMENTAL: 0 @@ -109,11 +116,6 @@ jobs: submodules: true fetch-depth: 1 - - name: Install Ubuntu postgres dependencies - run: | - sudo apt update - sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler - # Some of our rust modules use FFI and need those to be checked - name: Get postgres headers run: make postgres-headers -j$(nproc) @@ -122,7 +124,31 @@ jobs: run: cargo build --all --release --timings - name: Upload the build stats - uses: actions/upload-artifact@v3 + id: upload-stats + env: + BUCKET: neon-github-public-dev + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} + run: | + REPORT_URL=https://${BUCKET}.s3.amazonaws.com/build-stats/${SHA}/${GITHUB_RUN_ID}/cargo-timing.html + aws s3 cp --only-show-errors ./target/cargo-timings/cargo-timing.html "s3://${BUCKET}/build-stats/${SHA}/${GITHUB_RUN_ID}/" + echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT + + - name: Publish build stats report + uses: actions/github-script@v6 + env: + REPORT_URL: ${{ steps.upload-stats.outputs.report-url }} + SHA: ${{ github.event.pull_request.head.sha || github.sha }} with: - name: neon-${{ runner.os }}-release-build-stats - path: ./target/cargo-timings/ + script: | + const { REPORT_URL, SHA } = process.env + + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: `${SHA}`, + state: 'success', + target_url: `${REPORT_URL}`, + context: `Build stats (release)`, + }) From 2759f1a22ee82b08093e7e6a714d7ee099b2755a Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 2 Feb 2023 12:14:44 +0200 Subject: [PATCH 084/102] Evict layers on demand (#3486) Closes https://github.com/neondatabase/neon/issues/3439 Adds a set of commands to manipulate the layer map: * dump the layer map contents * evict the layer form the layer map (remove the local file, put the remote layer instead in the layer map) * download the layer (operation, reversing the eviction) The commands will change later, when the statistics is added on top, so the swagger schema is not adjusted. The commands might have issues with big amount of layers: no pagination is done for the dump command, eviction and download commands look for the layer to evict/download by iterating all layers sequentially and comparing the layer names. For now, that seems to be tolerable ("big" number of layers is ~2_000) and further experiments are needed. --------- Co-authored-by: Christian Schwarz --- libs/pageserver_api/src/models.rs | 46 +++++ pageserver/src/http/routes.rs | 110 +++++++++--- pageserver/src/tenant/storage_layer.rs | 19 +- .../src/tenant/storage_layer/delta_layer.rs | 14 ++ .../src/tenant/storage_layer/filename.rs | 15 -- .../src/tenant/storage_layer/image_layer.rs | 17 +- .../tenant/storage_layer/inmemory_layer.rs | 11 ++ .../src/tenant/storage_layer/remote_layer.rs | 23 +++ pageserver/src/tenant/timeline.rs | 166 +++++++++++++++--- test_runner/fixtures/neon_fixtures.py | 85 +++++++++ test_runner/regress/test_layer_eviction.py | 140 +++++++++++++++ 11 files changed, 576 insertions(+), 70 deletions(-) create mode 100644 test_runner/regress/test_layer_eviction.py diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 0d7aa2db55..7c0e3cc2b9 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -227,6 +227,52 @@ pub struct TimelineInfo { pub state: TimelineState, } +#[derive(Debug, Clone, Serialize)] +pub struct LayerMapInfo { + pub in_memory_layers: Vec, + pub historic_layers: Vec, +} + +#[serde_as] +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "kind")] +pub enum InMemoryLayerInfo { + Open { + #[serde_as(as = "DisplayFromStr")] + lsn_start: Lsn, + }, + Frozen { + #[serde_as(as = "DisplayFromStr")] + lsn_start: Lsn, + #[serde_as(as = "DisplayFromStr")] + lsn_end: Lsn, + }, +} + +#[serde_as] +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "kind")] +pub enum HistoricLayerInfo { + Delta { + layer_file_name: String, + layer_file_size: Option, + + #[serde_as(as = "DisplayFromStr")] + lsn_start: Lsn, + #[serde_as(as = "DisplayFromStr")] + lsn_end: Lsn, + remote: bool, + }, + Image { + layer_file_name: String, + layer_file_size: Option, + + #[serde_as(as = "DisplayFromStr")] + lsn_start: Lsn, + remote: bool, + }, +} + #[derive(Debug, Serialize, Deserialize)] pub struct DownloadRemoteLayersTaskSpawnRequest { pub max_concurrent_downloads: NonZeroUsize, diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index da31bd9bc0..77a8f63da9 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -7,7 +7,7 @@ use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest; use remote_storage::GenericRemoteStorage; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::http::request::{must_get_query_param, parse_query_param}; +use utils::http::request::{get_request_param, must_get_query_param, parse_query_param}; use super::models::{ StatusResponse, TenantConfigRequest, TenantCreateRequest, TenantCreateResponse, TenantInfo, @@ -317,10 +317,7 @@ async fn get_lsn_by_timestamp_handler(request: Request) -> Result) -> Result, A ) } +async fn layer_map_info_handler(request: Request) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + check_permission(&request, Some(tenant_id))?; + + let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; + let layer_map_info = timeline.layer_map_info(); + + json_response(StatusCode::OK, layer_map_info) +} + +async fn layer_download_handler(request: Request) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + check_permission(&request, Some(tenant_id))?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + let layer_file_name = get_request_param(&request, "layer_file_name")?; + check_permission(&request, Some(tenant_id))?; + + let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; + let downloaded = timeline + .download_layer(layer_file_name) + .await + .map_err(ApiError::InternalServerError)?; + + match downloaded { + Some(true) => json_response(StatusCode::OK, ()), + Some(false) => json_response(StatusCode::NOT_MODIFIED, ()), + None => json_response( + StatusCode::BAD_REQUEST, + format!("Layer {tenant_id}/{timeline_id}/{layer_file_name} not found"), + ), + } +} + +async fn evict_timeline_layer_handler(request: Request) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + check_permission(&request, Some(tenant_id))?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + let layer_file_name = get_request_param(&request, "layer_file_name")?; + + let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; + let evicted = timeline + .evict_layer(layer_file_name) + .await + .map_err(ApiError::InternalServerError)?; + + match evicted { + Some(true) => json_response(StatusCode::OK, ()), + Some(false) => json_response(StatusCode::NOT_MODIFIED, ()), + None => json_response( + StatusCode::BAD_REQUEST, + format!("Layer {tenant_id}/{timeline_id}/{layer_file_name} not found"), + ), + } +} + // Helper function to standardize the error messages we produce on bad durations // // Intended to be used with anyhow's `with_context`, e.g.: @@ -804,12 +857,7 @@ async fn timeline_checkpoint_handler(request: Request) -> Result json_response(StatusCode::ACCEPTED, st), Err(st) => json_response(StatusCode::CONFLICT, st), @@ -846,15 +889,10 @@ async fn timeline_download_remote_layers_handler_get( request: Request, ) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; - let tenant = mgr::get_tenant(tenant_id, true) - .await - .map_err(ApiError::NotFound)?; - let timeline = tenant - .get_timeline(timeline_id, true) - .map_err(ApiError::NotFound)?; + let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; let info = timeline .get_download_all_remote_layers_task_info() .context("task never started since last pageserver process start") @@ -862,6 +900,18 @@ async fn timeline_download_remote_layers_handler_get( json_response(StatusCode::OK, info) } +async fn active_timeline_of_active_tenant( + tenant_id: TenantId, + timeline_id: TimelineId, +) -> Result, ApiError> { + let tenant = mgr::get_tenant(tenant_id, true) + .await + .map_err(ApiError::NotFound)?; + tenant + .get_timeline(timeline_id, true) + .map_err(ApiError::NotFound) +} + async fn handler_404(_: Request) -> Result, ApiError> { json_response( StatusCode::NOT_FOUND, @@ -958,5 +1008,17 @@ pub fn make_router( "/v1/tenant/:tenant_id/timeline/:timeline_id", timeline_delete_handler, ) + .get( + "/v1/tenant/:tenant_id/timeline/:timeline_id/layer", + layer_map_info_handler, + ) + .get( + "/v1/tenant/:tenant_id/timeline/:timeline_id/layer/:layer_file_name", + layer_download_handler, + ) + .delete( + "/v1/tenant/:tenant_id/timeline/:timeline_id/layer/:layer_file_name", + evict_timeline_layer_handler, + ) .any(handler_404)) } diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 8d8aef4a68..48db364b8c 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -6,11 +6,13 @@ mod image_layer; mod inmemory_layer; mod remote_layer; +use crate::config::PageServerConf; use crate::context::RequestContext; use crate::repository::{Key, Value}; use crate::walrecord::NeonWalRecord; use anyhow::Result; use bytes::Bytes; +use pageserver_api::models::HistoricLayerInfo; use std::ops::Range; use std::path::PathBuf; use std::sync::Arc; @@ -21,7 +23,7 @@ use utils::{ }; pub use delta_layer::{DeltaLayer, DeltaLayerWriter}; -pub use filename::{DeltaFileName, ImageFileName, LayerFileName, PathOrConf}; +pub use filename::{DeltaFileName, ImageFileName, LayerFileName}; pub use image_layer::{ImageLayer, ImageLayerWriter}; pub use inmemory_layer::InMemoryLayer; pub use remote_layer::RemoteLayer; @@ -187,6 +189,8 @@ pub trait PersistentLayer: Layer { /// Should not change over the lifetime of the layer object because /// current_physical_size is computed as the som of this value. fn file_size(&self) -> Option; + + fn info(&self) -> HistoricLayerInfo; } pub fn downcast_remote_layer( @@ -246,3 +250,16 @@ impl Layer for LayerDescriptor { todo!() } } + +/// Helper enum to hold a PageServerConf, or a path +/// +/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the +/// global config, and paths to layer files are constructed using the tenant/timeline +/// path from the config. But in the 'pageserver_binutils' binary, we need to construct a Layer +/// struct for a file on disk, without having a page server running, so that we have no +/// config. In that case, we use the Path variant to hold the full path to the file on +/// disk. +enum PathOrConf { + Path(PathBuf), + Conf(&'static PageServerConf), +} diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 987b266c76..3a23e96325 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -37,6 +37,7 @@ use crate::virtual_file::VirtualFile; use crate::{walrecord, TEMP_FILE_SUFFIX}; use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION}; use anyhow::{bail, ensure, Context, Result}; +use pageserver_api::models::HistoricLayerInfo; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; use std::fs::{self, File}; @@ -417,6 +418,19 @@ impl PersistentLayer for DeltaLayer { fn file_size(&self) -> Option { Some(self.file_size) } + + fn info(&self) -> HistoricLayerInfo { + let layer_file_name = self.filename().file_name(); + let lsn_range = self.get_lsn_range(); + + HistoricLayerInfo::Delta { + layer_file_name, + layer_file_size: Some(self.file_size), + lsn_start: lsn_range.start, + lsn_end: lsn_range.end, + remote: false, + } + } } impl DeltaLayer { diff --git a/pageserver/src/tenant/storage_layer/filename.rs b/pageserver/src/tenant/storage_layer/filename.rs index 6ecf9227c7..7ebf346ec9 100644 --- a/pageserver/src/tenant/storage_layer/filename.rs +++ b/pageserver/src/tenant/storage_layer/filename.rs @@ -1,12 +1,10 @@ //! //! Helper functions for dealing with filenames of the image and delta layer files. //! -use crate::config::PageServerConf; use crate::repository::Key; use std::cmp::Ordering; use std::fmt; use std::ops::Range; -use std::path::PathBuf; use std::str::FromStr; use utils::lsn::Lsn; @@ -270,16 +268,3 @@ impl<'de> serde::de::Visitor<'de> for LayerFileNameVisitor { v.parse().map_err(|e| E::custom(e)) } } - -/// Helper enum to hold a PageServerConf, or a path -/// -/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the -/// global config, and paths to layer files are constructed using the tenant/timeline -/// path from the config. But in the 'pageserver_binutils' binary, we need to construct a Layer -/// struct for a file on disk, without having a page server running, so that we have no -/// config. In that case, we use the Path variant to hold the full path to the file on -/// disk. -pub enum PathOrConf { - Path(PathBuf), - Conf(&'static PageServerConf), -} diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 564f80d258..c11f671ffe 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -34,6 +34,7 @@ use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION, TEMP_FILE_SUFFIX}; use anyhow::{bail, ensure, Context, Result}; use bytes::Bytes; use hex; +use pageserver_api::models::HistoricLayerInfo; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; use std::fs::{self, File}; @@ -51,8 +52,8 @@ use utils::{ lsn::Lsn, }; -use super::filename::{ImageFileName, LayerFileName, PathOrConf}; -use super::{Layer, LayerIter}; +use super::filename::{ImageFileName, LayerFileName}; +use super::{Layer, LayerIter, PathOrConf}; /// /// Header stored in the beginning of the file @@ -235,6 +236,18 @@ impl PersistentLayer for ImageLayer { fn file_size(&self) -> Option { Some(self.file_size) } + + fn info(&self) -> HistoricLayerInfo { + let layer_file_name = self.filename().file_name(); + let lsn_range = self.get_lsn_range(); + + HistoricLayerInfo::Image { + layer_file_name, + layer_file_size: Some(self.file_size), + lsn_start: lsn_range.start, + remote: false, + } + } } impl ImageLayer { diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index e0b252189b..7249bfd115 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -13,6 +13,7 @@ use crate::tenant::ephemeral_file::EphemeralFile; use crate::tenant::storage_layer::{ValueReconstructResult, ValueReconstructState}; use crate::walrecord; use anyhow::{ensure, Result}; +use pageserver_api::models::InMemoryLayerInfo; use std::cell::RefCell; use std::collections::HashMap; use tracing::*; @@ -80,6 +81,16 @@ impl InMemoryLayer { pub fn get_timeline_id(&self) -> TimelineId { self.timeline_id } + + pub fn info(&self) -> InMemoryLayerInfo { + let lsn_start = self.start_lsn; + let lsn_end = self.inner.read().unwrap().end_lsn; + + match lsn_end { + Some(lsn_end) => InMemoryLayerInfo::Frozen { lsn_start, lsn_end }, + None => InMemoryLayerInfo::Open { lsn_start }, + } + } } impl Layer for InMemoryLayer { diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 0478cb82fb..29cb4af0fa 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -7,6 +7,7 @@ use crate::repository::Key; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState}; use anyhow::{bail, Result}; +use pageserver_api::models::HistoricLayerInfo; use std::ops::Range; use std::path::PathBuf; use std::sync::Arc; @@ -136,6 +137,28 @@ impl PersistentLayer for RemoteLayer { fn file_size(&self) -> Option { self.layer_metadata.file_size() } + + fn info(&self) -> HistoricLayerInfo { + let layer_file_name = self.filename().file_name(); + let lsn_range = self.get_lsn_range(); + + if self.is_delta { + HistoricLayerInfo::Delta { + layer_file_name, + layer_file_size: self.layer_metadata.file_size(), + lsn_start: lsn_range.start, + lsn_end: lsn_range.end, + remote: true, + } + } else { + HistoricLayerInfo::Image { + layer_file_name, + layer_file_size: self.layer_metadata.file_size(), + lsn_start: lsn_range.start, + remote: true, + } + } + } } impl RemoteLayer { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 8134d2a13a..22f1ce6e2d 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -10,7 +10,7 @@ use itertools::Itertools; use once_cell::sync::OnceCell; use pageserver_api::models::{ DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, - DownloadRemoteLayersTaskState, TimelineState, + DownloadRemoteLayersTaskState, LayerMapInfo, TimelineState, }; use tokio::sync::{oneshot, watch, Semaphore, TryAcquireError}; use tokio_util::sync::CancellationToken; @@ -69,6 +69,7 @@ use crate::ZERO_PAGE; use crate::{is_temporary, task_mgr}; use walreceiver::spawn_connection_manager_task; +use super::layer_map::BatchedUpdates; use super::remote_timeline_client::index::IndexPart; use super::remote_timeline_client::RemoteTimelineClient; use super::storage_layer::{DeltaLayer, ImageLayer, Layer}; @@ -91,7 +92,7 @@ pub struct Timeline { pub pg_version: u32, - pub layers: RwLock>, + pub(super) layers: RwLock>, last_freeze_at: AtomicLsn, // Atomic would be more appropriate here. @@ -663,7 +664,7 @@ impl Timeline { // Below are functions compact_level0() and create_image_layers() // but they are a bit ad hoc and don't quite work like it's explained // above. Rewrite it. - let _layer_removal_cs = self.layer_removal_cs.lock().await; + let layer_removal_cs = self.layer_removal_cs.lock().await; // Is the timeline being deleted? let state = *self.state.borrow(); if state == TimelineState::Stopping { @@ -696,7 +697,8 @@ impl Timeline { // 3. Compact let timer = self.metrics.compact_time_histo.start_timer(); - self.compact_level0(target_file_size, ctx).await?; + self.compact_level0(&layer_removal_cs, target_file_size, ctx) + .await?; timer.stop_and_record(); // If `create_image_layers' or `compact_level0` scheduled any @@ -832,6 +834,85 @@ impl Timeline { pub fn subscribe_for_state_updates(&self) -> watch::Receiver { self.state.subscribe() } + + pub fn layer_map_info(&self) -> LayerMapInfo { + let layer_map = self.layers.read().unwrap(); + let mut in_memory_layers = Vec::with_capacity(layer_map.frozen_layers.len() + 1); + if let Some(open_layer) = &layer_map.open_layer { + in_memory_layers.push(open_layer.info()); + } + for frozen_layer in &layer_map.frozen_layers { + in_memory_layers.push(frozen_layer.info()); + } + + let mut historic_layers = Vec::new(); + for historic_layer in layer_map.iter_historic_layers() { + historic_layers.push(historic_layer.info()); + } + + LayerMapInfo { + in_memory_layers, + historic_layers, + } + } + + pub async fn download_layer(&self, layer_file_name: &str) -> anyhow::Result> { + let Some(layer) = self.find_layer(layer_file_name) else { return Ok(None) }; + let Some(remote_layer) = layer.downcast_remote_layer() else { return Ok(Some(false)) }; + if self.remote_client.is_none() { + return Ok(Some(false)); + } + + self.download_remote_layer(remote_layer).await?; + Ok(Some(true)) + } + + pub async fn evict_layer(&self, layer_file_name: &str) -> anyhow::Result> { + let Some(local_layer) = self.find_layer(layer_file_name) else { return Ok(None) }; + if local_layer.is_remote_layer() { + return Ok(Some(false)); + } + let Some(remote_client) = &self.remote_client else { return Ok(Some(false)) }; + + // ensure the current layer is uploaded for sure + remote_client + .wait_completion() + .await + .context("wait for layer upload ops to complete")?; + + let layer_metadata = LayerFileMetadata::new( + local_layer + .file_size() + .expect("Local layer should have a file size"), + ); + let new_remote_layer = Arc::new(match local_layer.filename() { + LayerFileName::Image(image_name) => RemoteLayer::new_img( + self.tenant_id, + self.timeline_id, + &image_name, + &layer_metadata, + ), + LayerFileName::Delta(delta_name) => RemoteLayer::new_delta( + self.tenant_id, + self.timeline_id, + &delta_name, + &layer_metadata, + ), + #[cfg(test)] + LayerFileName::Test(_) => unreachable!(), + }); + + let gc_lock = self.layer_removal_cs.lock().await; + let mut layers = self.layers.write().unwrap(); + let mut updates = layers.batch_update(); + self.delete_historic_layer(&gc_lock, local_layer, &mut updates)?; + updates.insert_historic(new_remote_layer); + updates.flush(); + drop(layers); + drop(gc_lock); + + Ok(Some(true)) + } } // Private functions @@ -1621,6 +1702,43 @@ impl Timeline { Err(e) => error!("Failed to compute current logical size for metrics update: {e:?}"), } } + + fn find_layer(&self, layer_file_name: &str) -> Option> { + for historic_layer in self.layers.read().unwrap().iter_historic_layers() { + let historic_layer_name = historic_layer.filename().file_name(); + if layer_file_name == historic_layer_name { + return Some(historic_layer); + } + } + + None + } + + /// Removes the layer from local FS (if present) and from memory. + /// Remote storage is not affected by this operation. + fn delete_historic_layer( + &self, + // we cannot remove layers otherwise, since gc and compaction will race + _layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, + layer: Arc, + updates: &mut BatchedUpdates<'_, dyn PersistentLayer>, + ) -> anyhow::Result<()> { + let layer_size = layer.file_size(); + + layer.delete()?; + if let Some(layer_size) = layer_size { + self.metrics.resident_physical_size_gauge.sub(layer_size); + } + + // TODO Removing from the bottom of the layer map is expensive. + // Maybe instead discard all layer map historic versions that + // won't be needed for page reconstruction for this timeline, + // and mark what we can't delete yet as deleted from the layer + // map index without actually rebuilding the index. + updates.remove_historic(layer); + + Ok(()) + } } type TraversalId = String; @@ -2727,6 +2845,7 @@ impl Timeline { /// async fn compact_level0( &self, + layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, target_file_size: u64, ctx: &RequestContext, ) -> anyhow::Result<()> { @@ -2780,14 +2899,8 @@ impl Timeline { // delete the old ones let mut layer_names_to_delete = Vec::with_capacity(deltas_to_compact.len()); for l in deltas_to_compact { - if let Some(path) = l.local_path() { - self.metrics - .resident_physical_size_gauge - .sub(path.metadata()?.len()); - } layer_names_to_delete.push(l.filename()); - l.delete()?; - updates.remove_historic(l); + self.delete_historic_layer(layer_removal_cs, l, &mut updates)?; } updates.flush(); drop(layers); @@ -2907,7 +3020,7 @@ impl Timeline { fail_point!("before-timeline-gc"); - let _layer_removal_cs = self.layer_removal_cs.lock().await; + let layer_removal_cs = self.layer_removal_cs.lock().await; // Is the timeline being deleted? let state = *self.state.borrow(); if state == TimelineState::Stopping { @@ -2926,7 +3039,13 @@ impl Timeline { let new_gc_cutoff = Lsn::min(horizon_cutoff, pitr_cutoff); let res = self - .gc_timeline(horizon_cutoff, pitr_cutoff, retain_lsns, new_gc_cutoff) + .gc_timeline( + &layer_removal_cs, + horizon_cutoff, + pitr_cutoff, + retain_lsns, + new_gc_cutoff, + ) .instrument( info_span!("gc_timeline", timeline = %self.timeline_id, cutoff = %new_gc_cutoff), ) @@ -2940,6 +3059,7 @@ impl Timeline { async fn gc_timeline( &self, + layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, horizon_cutoff: Lsn, pitr_cutoff: Lsn, retain_lsns: Vec, @@ -3095,22 +3215,12 @@ impl Timeline { // (couldn't do this in the loop above, because you cannot modify a collection // while iterating it. BTreeMap::retain() would be another option) let mut layer_names_to_delete = Vec::with_capacity(layers_to_remove.len()); - for doomed_layer in layers_to_remove { - if let Some(path) = doomed_layer.local_path() { - self.metrics - .resident_physical_size_gauge - .sub(path.metadata()?.len()); + { + for doomed_layer in layers_to_remove { + layer_names_to_delete.push(doomed_layer.filename()); + self.delete_historic_layer(layer_removal_cs, doomed_layer, &mut updates)?; // FIXME: schedule succeeded deletions before returning? + result.layers_removed += 1; } - layer_names_to_delete.push(doomed_layer.filename()); - doomed_layer.delete()?; // FIXME: schedule succeeded deletions before returning? - - // TODO Removing from the bottom of the layer map is expensive. - // Maybe instead discard all layer map historic versions that - // won't be needed for page reconstruction for this timeline, - // and mark what we can't delete yet as deleted from the layer - // map index without actually rebuilding the index. - updates.remove_historic(doomed_layer); - result.layers_removed += 1; } if result.layers_removed != 0 { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index bd92130124..58e374eb3c 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1472,6 +1472,91 @@ class PageserverHttpClient(requests.Session): assert len(relevant) == 1 return relevant[0].lstrip(name).strip() + def layer_map_info( + self, + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> LayerMapInfo: + res = self.get( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/layer/", + ) + self.verbose_error(res) + return LayerMapInfo.from_json(res.json()) + + def download_layer(self, tenant_id: TenantId, timeline_id: TimelineId, layer_name: str): + res = self.get( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/layer/{layer_name}", + ) + self.verbose_error(res) + + assert res.status_code == 200 + + def evict_layer(self, tenant_id: TenantId, timeline_id: TimelineId, layer_name: str): + res = self.delete( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/layer/{layer_name}", + ) + self.verbose_error(res) + + assert res.status_code == 200 + + +@dataclass +class LayerMapInfo: + in_memory_layers: List[InMemoryLayerInfo] + historic_layers: List[HistoricLayerInfo] + + @classmethod + def from_json(cls, d: Dict[str, Any]) -> LayerMapInfo: + info = LayerMapInfo(in_memory_layers=[], historic_layers=[]) + + json_in_memory_layers = d["in_memory_layers"] + assert isinstance(json_in_memory_layers, List) + for json_in_memory_layer in json_in_memory_layers: + info.in_memory_layers.append(InMemoryLayerInfo.from_json(json_in_memory_layer)) + + json_historic_layers = d["historic_layers"] + assert isinstance(json_historic_layers, List) + for json_historic_layer in json_historic_layers: + info.historic_layers.append(HistoricLayerInfo.from_json(json_historic_layer)) + + return info + + +@dataclass +class InMemoryLayerInfo: + kind: str + lsn_start: str + lsn_end: Optional[str] + + @classmethod + def from_json(cls, d: Dict[str, Any]) -> InMemoryLayerInfo: + return InMemoryLayerInfo( + kind=d["kind"], + lsn_start=d["lsn_start"], + lsn_end=d.get("lsn_end"), + ) + + +@dataclass +class HistoricLayerInfo: + kind: str + layer_file_name: str + layer_file_size: Optional[int] + lsn_start: str + lsn_end: Optional[str] + remote: bool + + @classmethod + def from_json(cls, d: Dict[str, Any]) -> HistoricLayerInfo: + return HistoricLayerInfo( + kind=d["kind"], + layer_file_name=d["layer_file_name"], + layer_file_size=d.get("layer_file_size"), + lsn_start=d["lsn_start"], + lsn_end=d.get("lsn_end"), + remote=d["remote"], + ) + @dataclass class PageserverPort: diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py new file mode 100644 index 0000000000..a03dd88c41 --- /dev/null +++ b/test_runner/regress/test_layer_eviction.py @@ -0,0 +1,140 @@ +import pytest +from fixtures.neon_fixtures import ( + NeonEnvBuilder, + RemoteStorageKind, + wait_for_last_record_lsn, + wait_for_upload, +) +from fixtures.types import Lsn, TenantId, TimelineId +from fixtures.utils import query_scalar + + +# Crates a few layers, ensures that we can evict them (removing locally but keeping track of them anyway) +# and then download them back. +@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) +def test_basic_eviction( + neon_env_builder: NeonEnvBuilder, + remote_storage_kind: RemoteStorageKind, +): + neon_env_builder.enable_remote_storage( + remote_storage_kind=remote_storage_kind, + test_name="test_download_remote_layers_api", + ) + + env = neon_env_builder.init_start() + client = env.pageserver.http_client() + pg = env.postgres.create_start("main") + + tenant_id = TenantId(pg.safe_psql("show neon.tenant_id")[0][0]) + timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0]) + + # Create a number of layers in the tenant + with pg.cursor() as cur: + cur.execute("CREATE TABLE foo (t text)") + cur.execute( + """ + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 5000000) g + """ + ) + current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()")) + + wait_for_last_record_lsn(client, tenant_id, timeline_id, current_lsn) + client.timeline_checkpoint(tenant_id, timeline_id) + wait_for_upload(client, tenant_id, timeline_id, current_lsn) + + timeline_path = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + initial_local_layers = sorted( + list(filter(lambda path: path.name != "metadata", timeline_path.glob("*"))) + ) + assert ( + len(initial_local_layers) > 1 + ), f"Should create multiple layers for timeline, but got {initial_local_layers}" + + # Compare layer map dump with the local layers, ensure everything's present locally and matches + initial_layer_map_info = client.layer_map_info(tenant_id=tenant_id, timeline_id=timeline_id) + assert ( + not initial_layer_map_info.in_memory_layers + ), "Should have no in memory layers after flushing" + assert len(initial_local_layers) == len( + initial_layer_map_info.historic_layers + ), "Should have the same layers in memory and on disk" + for returned_layer in initial_layer_map_info.historic_layers: + assert ( + returned_layer.kind == "Delta" + ), f"Did not create and expect image layers, but got {returned_layer}" + assert ( + not returned_layer.remote + ), f"All created layers should be present locally, but got {returned_layer}" + + local_layers = list( + filter(lambda layer: layer.name == returned_layer.layer_file_name, initial_local_layers) + ) + assert ( + len(local_layers) == 1 + ), f"Did not find returned layer {returned_layer} in local layers {initial_local_layers}" + local_layer = local_layers[0] + assert ( + returned_layer.layer_file_size == local_layer.stat().st_size + ), f"Returned layer {returned_layer} has a different file size than local layer {local_layer}" + + # Detach all layers, ensre they are not in the local FS, but are still dumped as part of the layer map + for local_layer in initial_local_layers: + client.evict_layer( + tenant_id=tenant_id, timeline_id=timeline_id, layer_name=local_layer.name + ) + assert not any( + new_local_layer.name == local_layer.name for new_local_layer in timeline_path.glob("*") + ), f"Did not expect to find {local_layer} layer after evicting" + + empty_layers = list(filter(lambda path: path.name != "metadata", timeline_path.glob("*"))) + assert ( + not empty_layers + ), f"After evicting all layers, timeline {tenant_id}/{timeline_id} should have no layers locally, but got: {empty_layers}" + + evicted_layer_map_info = client.layer_map_info(tenant_id=tenant_id, timeline_id=timeline_id) + assert ( + not evicted_layer_map_info.in_memory_layers + ), "Should have no in memory layers after flushing and evicting" + assert len(initial_local_layers) == len( + evicted_layer_map_info.historic_layers + ), "Should have the same layers in memory and on disk initially" + for returned_layer in evicted_layer_map_info.historic_layers: + assert ( + returned_layer.kind == "Delta" + ), f"Did not create and expect image layers, but got {returned_layer}" + assert ( + returned_layer.remote + ), f"All layers should be evicted and not present locally, but got {returned_layer}" + assert any( + local_layer.name == returned_layer.layer_file_name + for local_layer in initial_local_layers + ), f"Did not find returned layer {returned_layer} in local layers {initial_local_layers}" + + # redownload all evicted layers and ensure the initial state is restored + for local_layer in initial_local_layers: + client.download_layer( + tenant_id=tenant_id, timeline_id=timeline_id, layer_name=local_layer.name + ) + client.timeline_download_remote_layers( + tenant_id, + timeline_id, + # allow some concurrency to unveil potential concurrency bugs + max_concurrent_downloads=10, + errors_ok=False, + at_least_one_download=False, + ) + + redownloaded_layers = sorted( + list(filter(lambda path: path.name != "metadata", timeline_path.glob("*"))) + ) + assert ( + redownloaded_layers == initial_local_layers + ), "Should have the same layers locally after redownloading the evicted layers" + redownloaded_layer_map_info = client.layer_map_info( + tenant_id=tenant_id, timeline_id=timeline_id + ) + assert ( + redownloaded_layer_map_info == initial_layer_map_info + ), "Should have the same layer map after redownloading the evicted layers" From d25307dced442fa69e2e5c9c72775ce023bb5ec2 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Wed, 18 Jan 2023 20:51:02 +0100 Subject: [PATCH 085/102] Compile postgres with ICU support --- Dockerfile.compute-node | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 7c40470ae2..5a3110141c 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -10,7 +10,8 @@ ARG TAG=pinned FROM debian:bullseye-slim AS build-deps RUN apt update && \ apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ - zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev + zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev \ + libicu-dev ######################################################################################### # @@ -22,7 +23,7 @@ FROM build-deps AS pg-build ARG PG_VERSION COPY vendor/postgres-${PG_VERSION} postgres RUN cd postgres && \ - ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \ + ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp --with-icu && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \ # Install headers @@ -234,10 +235,13 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb # Install: # libreadline8 for psql +# libicu67, locales for collations (including ICU) # libossp-uuid16 for extension ossp-uuid # libgeos, libgdal, libsfcgal1, libproj and libprotobuf-c1 for PostGIS RUN apt update && \ apt install --no-install-recommends -y \ + locales \ + libicu67 \ libreadline8 \ libossp-uuid16 \ libgeos-c1v5 \ @@ -246,7 +250,9 @@ RUN apt update && \ libprotobuf-c1 \ libsfcgal1 \ gdb && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 +ENV LANG en_US.utf8 USER postgres ENTRYPOINT ["/usr/local/bin/compute_ctl"] From f6a10f469330f3fd3672f22078362b73dbad14b1 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Thu, 2 Feb 2023 14:52:17 +0200 Subject: [PATCH 086/102] Use regular layer names instead of a special test ones (#3524) We do not need special enum variant for testing the file names, neither its special handling across the code. Current tests are able to create regular layers with normal layer names, as the PR shows. --- .../src/tenant/remote_timeline_client.rs | 57 ++++++++++++++----- .../tenant/remote_timeline_client/index.rs | 11 ++-- .../src/tenant/storage_layer/filename.rs | 37 +++--------- pageserver/src/tenant/timeline.rs | 4 -- 4 files changed, 58 insertions(+), 51 deletions(-) diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 3f69017160..985b480a76 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1135,18 +1135,29 @@ mod tests { client.init_upload_queue_for_empty_remote(&metadata)?; // Create a couple of dummy files, schedule upload for them - let content_foo = dummy_contents("foo"); - let content_bar = dummy_contents("bar"); - std::fs::write(timeline_path.join("foo"), &content_foo)?; - std::fs::write(timeline_path.join("bar"), &content_bar)?; + let layer_file_name_1: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(); + let layer_file_name_2: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D9-00000000016B5A52".parse().unwrap(); + let layer_file_name_3: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59DA-00000000016B5A53".parse().unwrap(); + let content_1 = dummy_contents("foo"); + let content_2 = dummy_contents("bar"); + let content_3 = dummy_contents("baz"); + std::fs::write( + timeline_path.join(layer_file_name_1.file_name()), + &content_1, + )?; + std::fs::write( + timeline_path.join(layer_file_name_2.file_name()), + &content_2, + )?; + std::fs::write(timeline_path.join(layer_file_name_3.file_name()), content_3)?; client.schedule_layer_file_upload( - &LayerFileName::Test("foo".to_owned()), - &LayerFileMetadata::new(content_foo.len() as u64), + &layer_file_name_1, + &LayerFileMetadata::new(content_1.len() as u64), )?; client.schedule_layer_file_upload( - &LayerFileName::Test("bar".to_owned()), - &LayerFileMetadata::new(content_bar.len() as u64), + &layer_file_name_2, + &LayerFileMetadata::new(content_2.len() as u64), )?; // Check that they are started immediately, not queued @@ -1183,7 +1194,13 @@ mod tests { // Download back the index.json, and check that the list of files is correct let index_part = runtime.block_on(client.download_index_file())?; - assert_file_list(&index_part.timeline_layers, &["foo", "bar"]); + assert_file_list( + &index_part.timeline_layers, + &[ + &layer_file_name_1.file_name(), + &layer_file_name_2.file_name(), + ], + ); let downloaded_metadata = index_part.parse_metadata()?; assert_eq!(downloaded_metadata, metadata); @@ -1191,10 +1208,10 @@ mod tests { let content_baz = dummy_contents("baz"); std::fs::write(timeline_path.join("baz"), &content_baz)?; client.schedule_layer_file_upload( - &LayerFileName::Test("baz".to_owned()), + &layer_file_name_3, &LayerFileMetadata::new(content_baz.len() as u64), )?; - client.schedule_layer_file_deletion(&[LayerFileName::Test("foo".to_owned())])?; + client.schedule_layer_file_deletion(&[layer_file_name_1.clone()])?; { let mut guard = client.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut().unwrap(); @@ -1206,12 +1223,26 @@ mod tests { assert!(upload_queue.num_inprogress_deletions == 0); assert!(upload_queue.latest_files_changes_since_metadata_upload_scheduled == 0); } - assert_remote_files(&["foo", "bar", "index_part.json"], &remote_timeline_dir); + assert_remote_files( + &[ + &layer_file_name_1.file_name(), + &layer_file_name_2.file_name(), + "index_part.json", + ], + &remote_timeline_dir, + ); // Finish them runtime.block_on(client.wait_completion())?; - assert_remote_files(&["bar", "baz", "index_part.json"], &remote_timeline_dir); + assert_remote_files( + &[ + &layer_file_name_2.file_name(), + &layer_file_name_3.file_name(), + "index_part.json", + ], + &remote_timeline_dir, + ); Ok(()) } diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index c199b7e10b..420edae6cd 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -8,7 +8,8 @@ use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use tracing::warn; -use crate::tenant::{metadata::TimelineMetadata, storage_layer::LayerFileName}; +use crate::tenant::metadata::TimelineMetadata; +use crate::tenant::storage_layer::LayerFileName; use utils::lsn::Lsn; @@ -274,7 +275,7 @@ mod tests { "timeline_layers":["000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9"], "layer_metadata":{ "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 }, - "LAYER_FILE_NAME::test/not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 } + "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 } }, "disk_consistent_lsn":"0/16960E8", "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] @@ -288,7 +289,7 @@ mod tests { ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: Some(25600000), }), - (LayerFileName::new_test("not_a_real_layer_but_adding_coverage"), IndexLayerMetadata { + ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: Some(9007199254741001), @@ -312,7 +313,7 @@ mod tests { "missing_layers":["This shouldn't fail deserialization"], "layer_metadata":{ "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 }, - "LAYER_FILE_NAME::test/not_a_real_layer_but_adding_coverage": { "file_size": 9007199254741001 } + "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 } }, "disk_consistent_lsn":"0/16960E8", "metadata_bytes":[112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] @@ -326,7 +327,7 @@ mod tests { ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: Some(25600000), }), - (LayerFileName::new_test("not_a_real_layer_but_adding_coverage"), IndexLayerMetadata { + ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: Some(9007199254741001), diff --git a/pageserver/src/tenant/storage_layer/filename.rs b/pageserver/src/tenant/storage_layer/filename.rs index 7ebf346ec9..e77f2b25c1 100644 --- a/pageserver/src/tenant/storage_layer/filename.rs +++ b/pageserver/src/tenant/storage_layer/filename.rs @@ -175,49 +175,32 @@ impl fmt::Display for ImageFileName { pub enum LayerFileName { Image(ImageFileName), Delta(DeltaFileName), - #[cfg(test)] - Test(String), } impl LayerFileName { pub fn file_name(&self) -> String { match self { - LayerFileName::Image(fname) => format!("{fname}"), - LayerFileName::Delta(fname) => format!("{fname}"), - #[cfg(test)] - LayerFileName::Test(fname) => fname.to_string(), + Self::Image(fname) => fname.to_string(), + Self::Delta(fname) => fname.to_string(), } } - #[cfg(test)] - pub(crate) fn new_test(name: &str) -> LayerFileName { - LayerFileName::Test(name.to_owned()) - } } impl From for LayerFileName { fn from(fname: ImageFileName) -> Self { - LayerFileName::Image(fname) + Self::Image(fname) } } impl From for LayerFileName { fn from(fname: DeltaFileName) -> Self { - LayerFileName::Delta(fname) + Self::Delta(fname) } } -// include a `/` in the name as an additional layer of robustness -// because `/` chars are not allowed in UNIX paths -#[cfg(test)] -const LAYER_FILE_NAME_TEST_PREFIX: &str = "LAYER_FILE_NAME::test/"; - impl FromStr for LayerFileName { type Err = String; fn from_str(value: &str) -> Result { - #[cfg(test)] - if let Some(value) = value.strip_prefix(LAYER_FILE_NAME_TEST_PREFIX) { - return Ok(LayerFileName::Test(value.to_owned())); - } let delta = DeltaFileName::parse_str(value); let image = ImageFileName::parse_str(value); let ok = match (delta, image) { @@ -226,8 +209,8 @@ impl FromStr for LayerFileName { "neither delta nor image layer file name: {value:?}" )) } - (Some(delta), None) => LayerFileName::Delta(delta), - (None, Some(image)) => LayerFileName::Image(image), + (Some(delta), None) => Self::Delta(delta), + (None, Some(image)) => Self::Image(image), (Some(_), Some(_)) => unreachable!(), }; Ok(ok) @@ -240,12 +223,8 @@ impl serde::Serialize for LayerFileName { S: serde::Serializer, { match self { - LayerFileName::Image(fname) => serializer.serialize_str(&format!("{}", fname)), - LayerFileName::Delta(fname) => serializer.serialize_str(&format!("{}", fname)), - #[cfg(test)] - LayerFileName::Test(t) => { - serializer.serialize_str(&format!("{LAYER_FILE_NAME_TEST_PREFIX}{t}")) - } + Self::Image(fname) => serializer.serialize_str(&fname.to_string()), + Self::Delta(fname) => serializer.serialize_str(&fname.to_string()), } } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 22f1ce6e2d..e80da302b7 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -898,8 +898,6 @@ impl Timeline { &delta_name, &layer_metadata, ), - #[cfg(test)] - LayerFileName::Test(_) => unreachable!(), }); let gc_lock = self.layer_removal_cs.lock().await; @@ -1370,8 +1368,6 @@ impl Timeline { let remote_layer = Arc::new(remote_layer); updates.insert_historic(remote_layer); } - #[cfg(test)] - LayerFileName::Test(_) => unreachable!(), } } From ada933eb426cc26fd95e9a03039531eea2522598 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 2 Feb 2023 15:33:40 -0500 Subject: [PATCH 087/102] Pageserver read trace utils (#2795) List, dump, and analyze read traces. --- .dockerignore | 1 + Cargo.lock | 294 +++++++++++++---------- Cargo.toml | 3 +- libs/pageserver_api/src/models.rs | 2 +- test_runner/fixtures/compare_fixtures.py | 27 ++- trace/Cargo.toml | 15 ++ trace/src/main.rs | 172 +++++++++++++ workspace_hack/Cargo.toml | 1 - 8 files changed, 373 insertions(+), 142 deletions(-) create mode 100644 trace/Cargo.toml create mode 100644 trace/src/main.rs diff --git a/.dockerignore b/.dockerignore index 2bbff86100..d256b21af1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,6 +15,7 @@ !proxy/ !safekeeper/ !storage_broker/ +!trace/ !vendor/postgres-v14/ !vendor/postgres-v15/ !workspace_hack/ diff --git a/Cargo.lock b/Cargo.lock index 9be9af35e2..1f10689902 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", "once_cell", @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", @@ -498,9 +498,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1304eab461cf02bd70b083ed8273388f9724c549b316ba3d1e213ce0e9e7fb7e" +checksum = "e5694b64066a2459918d8074c2ce0d5a88f409431994c2356617c8ae0c4721fc" dependencies = [ "async-trait", "axum-core", @@ -527,9 +527,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f487e40dc9daee24d8a1779df88522f159a54a980f99cfbe43db0be0bd3444a8" +checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" dependencies = [ "async-trait", "bytes", @@ -623,9 +623,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45ea9b00a7b3f2988e9a65ad3917e62123c38dba709b666506207be96d1790b" +checksum = "b7f0778972c64420fdedc63f09919c8a88bda7b25135357fd25a5d9f3257e832" dependencies = [ "memchr", "once_cell", @@ -647,9 +647,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" dependencies = [ "serde", ] @@ -672,9 +672,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cexpr" @@ -756,9 +756,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.1" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec7a4128863c188deefe750ac1d1dfe66c236909f845af04beed823638dc1b2" +checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" dependencies = [ "bitflags", "clap_derive", @@ -838,7 +838,7 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap 4.1.1", + "clap 4.1.4", "futures", "hyper", "notify", @@ -896,7 +896,7 @@ name = "control_plane" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.1.1", + "clap 4.1.4", "comfy-table", "git-version", "nix", @@ -1077,9 +1077,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +checksum = "bc831ee6a32dd495436e317595e639a587aa9907bef96fe6e6abc290ab6204e9" dependencies = [ "cc", "cxxbridge-flags", @@ -1089,9 +1089,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +checksum = "94331d54f1b1a8895cd81049f7eaaaef9d05a7dcb4d1fd08bf3ff0806246789d" dependencies = [ "cc", "codespan-reporting", @@ -1104,15 +1104,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" +checksum = "48dcd35ba14ca9b40d6e4b4b39961f23d835dbb8eed74565ded361d93e1feb8a" [[package]] name = "cxxbridge-macro" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "81bbeb29798b407ccd82a3324ade1a7286e0d29851475990b612670f6f5124d2" dependencies = [ "proc-macro2", "quote", @@ -1221,15 +1221,15 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -1303,7 +1303,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1348,9 +1348,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -1363,9 +1363,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -1373,15 +1373,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -1390,15 +1390,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -1407,15 +1407,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-timer" @@ -1425,9 +1425,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-channel", "futures-core", @@ -1464,9 +1464,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" +checksum = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec" [[package]] name = "git-version" @@ -1536,7 +1536,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.2", + "ahash 0.8.3", ] [[package]] @@ -1550,9 +1550,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -1814,7 +1814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1832,7 +1832,7 @@ dependencies = [ "hermit-abi 0.2.6", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1852,9 +1852,9 @@ checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -2039,9 +2039,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "f2e212582ede878b109755efd0773a4f0f4ec851584cf0aefbeb4d9ecc114822" dependencies = [ "adler", ] @@ -2055,7 +2055,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2099,9 +2099,9 @@ dependencies = [ [[package]] name = "notify" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2c66da08abae1c024c01d635253e402341b4060a12e99b31c7594063bf490a" +checksum = "58ea850aa68a06e48fdb069c0ec44d0d64c8dbffa49bf3b6f7f0a901fdea1ba9" dependencies = [ "bitflags", "crossbeam-channel", @@ -2112,7 +2112,7 @@ dependencies = [ "libc", "mio", "walkdir", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -2167,9 +2167,9 @@ dependencies = [ [[package]] name = "object" -version = "0.30.2" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b8c786513eb403643f2a88c244c2aaa270ef2153f55094587d0c48a3cf22a83" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" dependencies = [ "memchr", ] @@ -2305,9 +2305,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4750134fb6a5d49afc80777394ad5d95b04bc12068c6abb92fae8f43817270f" +checksum = "5c424bc68d15e0778838ac013b5b3449544d8133633d8016319e7e05a820b8c0" dependencies = [ "log", "serde", @@ -2336,7 +2336,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "clap 4.1.1", + "clap 4.1.4", "close_fds", "const_format", "consumption_metrics", @@ -2418,15 +2418,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -2782,7 +2782,7 @@ dependencies = [ "bstr", "bytes", "chrono", - "clap 4.1.1", + "clap 4.1.4", "consumption_metrics", "futures", "git-version", @@ -2882,9 +2882,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -2974,11 +2974,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ - "base64 0.13.1", + "base64 0.21.0", "bytes", "encoding_rs", "futures-core", @@ -3115,7 +3115,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3181,7 +3181,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "clap 4.1.1", + "clap 4.1.4", "const_format", "crc32c", "fs2", @@ -3242,7 +3242,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3269,9 +3269,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ "bitflags", "core-foundation", @@ -3282,9 +3282,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -3298,9 +3298,9 @@ checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" [[package]] name = "sentry" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ad137b9df78294b98cab1a650bef237cc6c950e82e5ce164655e674d07c5cc" +checksum = "a6097dc270a9c4555c5d6222ed243eaa97ff38e29299ed7c5cb36099033c604e" dependencies = [ "httpdate", "reqwest", @@ -3316,9 +3316,9 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afe4800806552aab314129761d5d3b3d422284eca3de2ab59e9fd133636cbd3d" +checksum = "9d92d1e4d591534ae4f872d6142f3b500f4ffc179a6aed8a3e86c7cc96d10a6a" dependencies = [ "backtrace", "once_cell", @@ -3328,9 +3328,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42938426670f6e7974989cd1417837a96dd8bbb01567094f567d6acb360bf88" +checksum = "3afa877b1898ff67dd9878cf4bec4e53cef7d3be9f14b1fc9e4fcdf36f8e4259" dependencies = [ "hostname", "libc", @@ -3342,9 +3342,9 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df9b9d8de2658a1ecd4e45f7b06c80c5dd97b891bfbc7c501186189b7e9bbdf" +checksum = "fc43eb7e4e3a444151a0fe8a0e9ce60eabd905dae33d66e257fa26f1b509c1bd" dependencies = [ "once_cell", "rand", @@ -3355,9 +3355,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0af37b8500f273e511ebd6eb0d342ff7937d64ce3f134764b2b4653112d48cb4" +checksum = "ccab4fab11e3e63c45f4524bee2e75cde39cdf164cb0b0cbe6ccd1948ceddf66" dependencies = [ "sentry-backtrace", "sentry-core", @@ -3365,9 +3365,9 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccc95faa4078768a6bf8df45e2b894bbf372b3dbbfb364e9429c1c58ab7545c6" +checksum = "f63708ec450b6bdcb657af760c447416d69c38ce421f34e5e2e9ce8118410bc7" dependencies = [ "debugid", "getrandom", @@ -3580,7 +3580,7 @@ dependencies = [ "anyhow", "async-stream", "bytes", - "clap 4.1.1", + "clap 4.1.4", "const_format", "futures", "futures-core", @@ -3661,9 +3661,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "synstructure" @@ -3822,9 +3822,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.24.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", "bytes", @@ -3836,7 +3836,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3961,18 +3961,18 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] [[package]] name = "toml_datetime" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808b51e57d0ef8f71115d8f3a01e7d3750d01c79cac4b3eda910f4389fdf92fd" +checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" dependencies = [ "serde", ] @@ -4089,6 +4089,17 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +[[package]] +name = "trace" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.1.4", + "pageserver_api", + "utils", + "workspace_hack", +] + [[package]] name = "tracing" version = "0.1.37" @@ -4247,9 +4258,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" @@ -4367,9 +4378,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" dependencies = [ "getrandom", "serde", @@ -4392,7 +4403,7 @@ name = "wal_craft" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.1.1", + "clap 4.1.4", "env_logger", "log", "once_cell", @@ -4431,9 +4442,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -4441,9 +4452,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -4456,9 +4467,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -4468,9 +4479,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4478,9 +4489,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -4491,15 +4502,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -4526,9 +4537,9 @@ dependencies = [ [[package]] name = "which" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", "libc", @@ -4581,6 +4592,30 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.1" @@ -4639,14 +4674,13 @@ dependencies = [ "anyhow", "bytes", "chrono", - "clap 4.1.1", + "clap 4.1.4", "crossbeam-utils", "either", "fail", "futures", "futures-channel", "futures-executor", - "futures-task", "futures-util", "hashbrown 0.12.3", "indexmap", diff --git a/Cargo.toml b/Cargo.toml index 9dcf1a265a..c335b64175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "safekeeper", "storage_broker", "workspace_hack", + "trace", "libs/*", ] @@ -31,7 +32,7 @@ bstr = "1.0" byteorder = "1.4" bytes = "1.0" chrono = { version = "0.4", default-features = false, features = ["clock"] } -clap = "4.0" +clap = { version = "4.0", features = ["derive"] } close_fds = "0.3.2" comfy-table = "6.1" const_format = "0.2" diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 7c0e3cc2b9..6f53f4a01d 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -313,7 +313,7 @@ pub struct TimelineGcRequest { } // Wrapped in libpq CopyData -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Debug)] pub enum PagestreamFeMessage { Exists(PagestreamExistsRequest), Nblocks(PagestreamNblocksRequest), diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index be1f146735..963d1bd19e 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -97,10 +97,19 @@ class NeonCompare(PgCompare): self._pg_bin = pg_bin self.pageserver_http_client = self.env.pageserver.http_client() - # We only use one branch and one timeline - self.env.neon_cli.create_branch(branch_name, "empty") - self._pg = self.env.postgres.create_start(branch_name) - self.timeline = self.pg.safe_psql("SHOW neon.timeline_id")[0][0] + # Create tenant + tenant_conf: Dict[str, str] = {} + if False: # TODO add pytest setting for this + tenant_conf["trace_read_requests"] = "true" + self.tenant, _ = self.env.neon_cli.create_tenant(conf=tenant_conf) + + # Create timeline + self.timeline = self.env.neon_cli.create_timeline(branch_name, tenant_id=self.tenant) + + # Start pg + self._pg = self.env.postgres.create_start( + branch_name, "main", self.tenant, config_lines=["shared_buffers=2GB"] + ) @property def pg(self) -> PgProtocol: @@ -115,11 +124,11 @@ class NeonCompare(PgCompare): return self._pg_bin def flush(self): - self.pageserver_http_client.timeline_checkpoint(self.env.initial_tenant, self.timeline) - self.pageserver_http_client.timeline_gc(self.env.initial_tenant, self.timeline, 0) + self.pageserver_http_client.timeline_checkpoint(self.tenant, self.timeline) + self.pageserver_http_client.timeline_gc(self.tenant, self.timeline, 0) def compact(self): - self.pageserver_http_client.timeline_compact(self.env.initial_tenant, self.timeline) + self.pageserver_http_client.timeline_compact(self.tenant, self.timeline) def report_peak_memory_use(self): self.zenbenchmark.record( @@ -131,13 +140,13 @@ class NeonCompare(PgCompare): def report_size(self): timeline_size = self.zenbenchmark.get_timeline_size( - self.env.repo_dir, self.env.initial_tenant, self.timeline + self.env.repo_dir, self.tenant, self.timeline ) self.zenbenchmark.record( "size", timeline_size / (1024 * 1024), "MB", report=MetricReport.LOWER_IS_BETTER ) - params = f'{{tenant_id="{self.env.initial_tenant}",timeline_id="{self.timeline}"}}' + params = f'{{tenant_id="{self.tenant}",timeline_id="{self.timeline}"}}' total_files = self.zenbenchmark.get_int_counter_value( self.env.pageserver, "pageserver_created_persistent_files_total" + params ) diff --git a/trace/Cargo.toml b/trace/Cargo.toml new file mode 100644 index 0000000000..6ced992d4c --- /dev/null +++ b/trace/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "trace" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap.workspace = true +anyhow.workspace = true + +pageserver_api.workspace = true +utils.workspace = true +workspace_hack.workspace = true diff --git a/trace/src/main.rs b/trace/src/main.rs new file mode 100644 index 0000000000..ddd970e95d --- /dev/null +++ b/trace/src/main.rs @@ -0,0 +1,172 @@ +//! A tool for working with read traces generated by the pageserver. +use std::collections::HashMap; +use std::path::PathBuf; +use std::str::FromStr; +use std::{ + fs::{read_dir, File}, + io::BufReader, +}; + +use pageserver_api::models::{PagestreamFeMessage, PagestreamGetPageRequest}; +use utils::id::{ConnectionId, TenantId, TimelineId}; + +use clap::{Parser, Subcommand}; + +/// Utils for working with pageserver read traces. For generating +/// traces, see the `trace_read_requests` tenant config option. +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Path of trace directory + #[arg(short, long)] + path: PathBuf, + + #[command(subcommand)] + command: Command, +} + +/// What to do with the read trace +#[derive(Subcommand, Debug)] +enum Command { + /// List traces in the directory + List, + + /// Print the traces in text format + Dump, + + /// Print stats and anomalies about the traces + Analyze, + + /// Draw the traces in svg format + Draw, + + /// Send the read requests to a pageserver + Replay, +} + +// HACK This function will change and improve as we see what kind of analysis is useful. +// Currently it collects the difference in blkno of consecutive GetPage requests, +// and counts the frequency of each value. This information is useful in order to: +// - see how sequential a workload is by seeing how often the delta is 1 +// - detect any prefetching anomalies by looking for negative deltas during seqscan +fn analyze_trace(mut reader: R) { + let mut total = 0; // Total requests traced + let mut cross_rel = 0; // Requests that ask for different rel than previous request + let mut deltas = HashMap::::new(); // Consecutive blkno differences + let mut prev: Option = None; + + // Compute stats + while let Ok(msg) = PagestreamFeMessage::parse(&mut reader) { + match msg { + PagestreamFeMessage::Exists(_) => {} + PagestreamFeMessage::Nblocks(_) => {} + PagestreamFeMessage::GetPage(req) => { + total += 1; + + if let Some(prev) = prev { + if prev.rel == req.rel { + let delta = (req.blkno as i32) - (prev.blkno as i32); + deltas.entry(delta).and_modify(|c| *c += 1).or_insert(1); + } else { + cross_rel += 1; + } + } + prev = Some(req); + } + PagestreamFeMessage::DbSize(_) => {} + }; + } + + // Print stats. + let mut other = deltas.len(); + deltas.retain(|_, count| *count > 300); + other -= deltas.len(); + dbg!(total); + dbg!(cross_rel); + dbg!(other); + dbg!(deltas); +} + +fn dump_trace(mut reader: R) { + while let Ok(msg) = PagestreamFeMessage::parse(&mut reader) { + println!("{msg:?}"); + } +} + +#[derive(Debug)] +struct TraceFile { + #[allow(dead_code)] + pub tenant_id: TenantId, + + #[allow(dead_code)] + pub timeline_id: TimelineId, + + #[allow(dead_code)] + pub connection_id: ConnectionId, + + pub path: PathBuf, +} + +fn get_trace_files(traces_dir: &PathBuf) -> anyhow::Result> { + let mut trace_files = Vec::::new(); + + // Trace files are organized as {tenant_id}/{timeline_id}/{connection_id} + for tenant_dir in read_dir(traces_dir)? { + let entry = tenant_dir?; + let path = entry.path(); + let tenant_id = TenantId::from_str(path.file_name().unwrap().to_str().unwrap())?; + + for timeline_dir in read_dir(path)? { + let entry = timeline_dir?; + let path = entry.path(); + let timeline_id = TimelineId::from_str(path.file_name().unwrap().to_str().unwrap())?; + + for trace_dir in read_dir(path)? { + let entry = trace_dir?; + let path = entry.path(); + let connection_id = + ConnectionId::from_str(path.file_name().unwrap().to_str().unwrap())?; + + trace_files.push(TraceFile { + tenant_id, + timeline_id, + connection_id, + path, + }); + } + } + } + + Ok(trace_files) +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse(); + + match args.command { + Command::List => { + for trace_file in get_trace_files(&args.path)? { + println!("{trace_file:?}"); + } + } + Command::Dump => { + for trace_file in get_trace_files(&args.path)? { + let file = File::open(trace_file.path.clone())?; + let reader = BufReader::new(file); + dump_trace(reader); + } + } + Command::Analyze => { + for trace_file in get_trace_files(&args.path)? { + println!("analyzing {trace_file:?}"); + let file = File::open(trace_file.path.clone())?; + let reader = BufReader::new(file); + analyze_trace(reader); + } + } + Command::Draw => todo!(), + Command::Replay => todo!(), + } + + Ok(()) +} diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 8bfc9e2770..30a6d3a92b 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -23,7 +23,6 @@ fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures = { version = "0.3" } futures-channel = { version = "0.3", features = ["sink"] } futures-executor = { version = "0.3" } -futures-task = { version = "0.3", default-features = false, features = ["std"] } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } hashbrown = { version = "0.12", features = ["raw"] } indexmap = { version = "1", default-features = false, features = ["std"] } From 96e78394f5b24bb2116f1e326e0d11c4138f1df0 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 2 Feb 2023 23:56:15 +0300 Subject: [PATCH 088/102] [proxy] Fix project (aka endpoint) init in the password hack handler (#3529) The project/endpoint should be set in the original (non-as_ref'd) creds, because we call `wake_compute` not only in `try_password_hack` but also later in the connection retry logic. This PR also removes the obsolete `as_ref` method and makes the code simpler because we no longer need this complication after a recent refactoring. Further action points: finally introduce typestate in creds (planned). --- proxy/src/auth/backend.rs | 22 +++++++++------------- proxy/src/auth/credentials.rs | 12 ------------ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index 60460e6722..50afbd2a27 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -149,36 +149,32 @@ impl<'l> BackendType<'l, ClientCredentials<'_>> { }; // TODO: find a proper way to merge those very similar blocks. - let (mut node, payload) = match self { + let (mut node, password) = match self { Console(api, creds) if creds.project.is_none() => { let payload = fetch_magic_payload(client).await?; + creds.project = Some(payload.project.into()); + let node = api.wake_compute(extra, creds).await?; - let mut creds = creds.as_ref(); - creds.project = Some(payload.project.as_str().into()); - let node = api.wake_compute(extra, &creds).await?; - - (node, payload) + (node, payload.password) } // This is a hack to allow cleartext password in secure connections (wss). Console(api, creds) if creds.use_cleartext_password_flow => { let payload = fetch_plaintext_password(client).await?; let node = api.wake_compute(extra, creds).await?; - (node, payload) + (node, payload.password) } Postgres(api, creds) if creds.project.is_none() => { let payload = fetch_magic_payload(client).await?; + creds.project = Some(payload.project.into()); + let node = api.wake_compute(extra, creds).await?; - let mut creds = creds.as_ref(); - creds.project = Some(payload.project.as_str().into()); - let node = api.wake_compute(extra, &creds).await?; - - (node, payload) + (node, payload.password) } _ => return Ok(None), }; - node.config.password(payload.password); + node.config.password(password); Ok(Some(AuthSuccess { reported_auth_ok: false, value: node, diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index e1943fe44c..66ca8be73e 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -47,18 +47,6 @@ impl ClientCredentials<'_> { } } -impl<'a> ClientCredentials<'a> { - #[inline] - pub fn as_ref(&'a self) -> ClientCredentials<'a> { - Self { - user: self.user, - dbname: self.dbname, - project: self.project().map(Cow::Borrowed), - use_cleartext_password_flow: self.use_cleartext_password_flow, - } - } -} - impl<'a> ClientCredentials<'a> { pub fn parse( params: &'a StartupMessageParams, From f71b1b174dd985cd7400f40c503308ffb425af42 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 3 Feb 2023 08:01:26 +0200 Subject: [PATCH 089/102] Check correctness of file_cache_size_limit (#3530) ## Describe your changes ## Issue ticket number and link ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. --- pgxn/neon/file_cache.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c index 96c2461e2d..3a2ac380f9 100644 --- a/pgxn/neon/file_cache.c +++ b/pgxn/neon/file_cache.c @@ -132,7 +132,7 @@ lfc_shmem_request(void) RequestNamedLWLockTranche("lfc_lock", 1); } -bool +static bool lfc_check_limit_hook(int *newval, void **extra, GucSource source) { if (*newval > lfc_max_size) @@ -143,7 +143,7 @@ lfc_check_limit_hook(int *newval, void **extra, GucSource source) return true; } -void +static void lfc_change_limit_hook(int newval, void *extra) { uint32 new_size = SIZE_MB_TO_CHUNKS(newval); @@ -213,7 +213,7 @@ lfc_init(void) INT_MAX, PGC_SIGHUP, GUC_UNIT_MB, - NULL, + lfc_check_limit_hook, lfc_change_limit_hook, NULL); @@ -472,7 +472,6 @@ local_cache_pages(PG_FUNCTION_ARGS) HASH_SEQ_STATUS status; FileCacheEntry* entry; uint32 n_pages = 0; - uint32 i; funcctx = SRF_FIRSTCALL_INIT(); From 83048a4adcd601960e96da84d5a064267a1df02d Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Fri, 3 Feb 2023 12:37:34 +0200 Subject: [PATCH 090/102] Handle errors during metric collection. (#3521) Don't exit the loop if one of the tenants failed to scrape its metrics. fixes #3490 --- pageserver/src/consumption_metrics.rs | 99 +++++++++++++++++---------- 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index d848ec5ee5..6204ab92ab 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -83,10 +83,7 @@ pub async fn collect_metrics( return Ok(()); }, _ = ticker.tick() => { - if let Err(err) = collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id, &ctx).await - { - error!("metrics collection failed: {err:?}"); - } + collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id, &ctx).await; } } } @@ -97,17 +94,18 @@ pub async fn collect_metrics( /// Gather per-tenant and per-timeline metrics and send them to the `metric_collection_endpoint`. /// Cache metrics to avoid sending the same metrics multiple times. /// +/// This function handles all errors internally +/// and doesn't break iteration if just one tenant fails. +/// /// TODO /// - refactor this function (chunking+sending part) to reuse it in proxy module; -/// - improve error handling. Now if one tenant fails to collect metrics, -/// the whole iteration fails and metrics for other tenants are not collected. pub async fn collect_metrics_iteration( client: &reqwest::Client, cached_metrics: &mut HashMap, metric_collection_endpoint: &reqwest::Url, node_id: NodeId, ctx: &RequestContext, -) -> anyhow::Result<()> { +) { let mut current_metrics: Vec<(PageserverConsumptionMetricsKey, u64)> = Vec::new(); trace!( "starting collect_metrics_iteration. metric_collection_endpoint: {}", @@ -115,7 +113,13 @@ pub async fn collect_metrics_iteration( ); // get list of tenants - let tenants = mgr::list_tenants().await?; + let tenants = match mgr::list_tenants().await { + Ok(tenants) => tenants, + Err(err) => { + error!("failed to list tenants: {:?}", err); + return; + } + }; // iterate through list of Active tenants and collect metrics for (tenant_id, tenant_state) in tenants { @@ -123,7 +127,15 @@ pub async fn collect_metrics_iteration( continue; } - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = match mgr::get_tenant(tenant_id, true).await { + Ok(tenant) => tenant, + Err(err) => { + // It is possible that tenant was deleted between + // `list_tenants` and `get_tenant`, so just warn about it. + warn!("failed to get tenant {tenant_id:?}: {err:?}"); + continue; + } + }; let mut tenant_resident_size = 0; @@ -142,29 +154,51 @@ pub async fn collect_metrics_iteration( timeline_written_size, )); - let (timeline_logical_size, is_exact) = timeline.get_current_logical_size(ctx)?; - // Only send timeline logical size when it is fully calculated. - if is_exact { - current_metrics.push(( - PageserverConsumptionMetricsKey { - tenant_id, - timeline_id: Some(timeline.timeline_id), - metric: TIMELINE_LOGICAL_SIZE, - }, - timeline_logical_size, - )); - } + match timeline.get_current_logical_size(ctx) { + // Only send timeline logical size when it is fully calculated. + Ok((size, is_exact)) if is_exact => { + current_metrics.push(( + PageserverConsumptionMetricsKey { + tenant_id, + timeline_id: Some(timeline.timeline_id), + metric: TIMELINE_LOGICAL_SIZE, + }, + size, + )); + } + Ok((_, _)) => {} + Err(err) => { + error!( + "failed to get current logical size for timeline {}: {err:?}", + timeline.timeline_id + ); + continue; + } + }; } let timeline_resident_size = timeline.get_resident_physical_size(); tenant_resident_size += timeline_resident_size; } - let tenant_remote_size = tenant.get_remote_size().await?; - debug!( - "collected current metrics for tenant: {}: state={:?} resident_size={} remote_size={}", - tenant_id, tenant_state, tenant_resident_size, tenant_remote_size - ); + match tenant.get_remote_size().await { + Ok(tenant_remote_size) => { + current_metrics.push(( + PageserverConsumptionMetricsKey { + tenant_id, + timeline_id: None, + metric: REMOTE_STORAGE_SIZE, + }, + tenant_remote_size, + )); + } + Err(err) => { + error!( + "failed to get remote size for tenant {}: {err:?}", + tenant_id + ); + } + } current_metrics.push(( PageserverConsumptionMetricsKey { @@ -175,15 +209,6 @@ pub async fn collect_metrics_iteration( tenant_resident_size, )); - current_metrics.push(( - PageserverConsumptionMetricsKey { - tenant_id, - timeline_id: None, - metric: REMOTE_STORAGE_SIZE, - }, - tenant_remote_size, - )); - // Note that this metric is calculated in a separate bgworker // Here we only use cached value, which may lag behind the real latest one let tenant_synthetic_size = tenant.get_cached_synthetic_size(); @@ -205,7 +230,7 @@ pub async fn collect_metrics_iteration( if current_metrics.is_empty() { trace!("no new metrics to send"); - return Ok(()); + return; } // Send metrics. @@ -256,8 +281,6 @@ pub async fn collect_metrics_iteration( } } } - - Ok(()) } /// Caclculate synthetic size for each active tenant From a0372158a048b4591867e59e6e969a06b7c58440 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Fri, 3 Feb 2023 12:23:27 +0100 Subject: [PATCH 091/102] Add build_info metric to storage broker (#3525) ## Describe your changes Add libmetrics_build_info metrics with commit sha to storage_broken /metrics, to match behaviour of proxy, pageserver and safekeeper. --- storage_broker/src/bin/storage_broker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/storage_broker/src/bin/storage_broker.rs b/storage_broker/src/bin/storage_broker.rs index e33369bbb1..c73206b7dc 100644 --- a/storage_broker/src/bin/storage_broker.rs +++ b/storage_broker/src/bin/storage_broker.rs @@ -431,6 +431,7 @@ async fn main() -> Result<(), Box> { logging::init(LogFormat::from_config(&args.log_format)?)?; info!("version: {GIT_VERSION}"); + ::metrics::set_build_info_metric(GIT_VERSION); let registry = Registry { shared_state: Arc::new(RwLock::new(SharedState::new(args.all_keys_chan_size))), From f2d89761c2da1bec3db32d894a68f381faa12152 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 3 Feb 2023 15:33:46 +0200 Subject: [PATCH 092/102] feat: LayerMap::replace (#3513) Cc: #3486 Adds a method to replace a particular layer from the LayerMap for the purposes of remote layer download and layer eviction. In those use cases read lock on layer map needs to be released after initial search, but other operations could modify layermap before replacing thread gets to run. Co-authored-by: bojanserafimov --- pageserver/benches/bench_layer_map.rs | 36 +-- pageserver/src/tenant/layer_map.rs | 200 +++++++++++-- .../layer_map/historic_layer_coverage.rs | 273 ++++++++++++++++-- pageserver/src/tenant/storage_layer.rs | 38 +++ .../src/tenant/storage_layer/filename.rs | 7 + .../src/tenant/storage_layer/remote_layer.rs | 2 +- pageserver/src/tenant/timeline.rs | 36 ++- 7 files changed, 517 insertions(+), 75 deletions(-) diff --git a/pageserver/benches/bench_layer_map.rs b/pageserver/benches/bench_layer_map.rs index e18c00da96..5edfa84d8a 100644 --- a/pageserver/benches/bench_layer_map.rs +++ b/pageserver/benches/bench_layer_map.rs @@ -1,8 +1,7 @@ use pageserver::keyspace::{KeyPartitioning, KeySpace}; use pageserver::repository::Key; use pageserver::tenant::layer_map::LayerMap; -use pageserver::tenant::storage_layer::Layer; -use pageserver::tenant::storage_layer::{DeltaFileName, ImageFileName, LayerDescriptor}; +use pageserver::tenant::storage_layer::{Layer, LayerDescriptor, LayerFileName}; use rand::prelude::{SeedableRng, SliceRandom, StdRng}; use std::cmp::{max, min}; use std::fs::File; @@ -26,30 +25,15 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap { let mut updates = layer_map.batch_update(); for fname in filenames { - let fname = &fname.unwrap(); - if let Some(imgfilename) = ImageFileName::parse_str(fname) { - let layer = LayerDescriptor { - key: imgfilename.key_range, - lsn: imgfilename.lsn..(imgfilename.lsn + 1), - is_incremental: false, - short_id: fname.to_string(), - }; - updates.insert_historic(Arc::new(layer)); - min_lsn = min(min_lsn, imgfilename.lsn); - max_lsn = max(max_lsn, imgfilename.lsn); - } else if let Some(deltafilename) = DeltaFileName::parse_str(fname) { - let layer = LayerDescriptor { - key: deltafilename.key_range.clone(), - lsn: deltafilename.lsn_range.clone(), - is_incremental: true, - short_id: fname.to_string(), - }; - updates.insert_historic(Arc::new(layer)); - min_lsn = min(min_lsn, deltafilename.lsn_range.start); - max_lsn = max(max_lsn, deltafilename.lsn_range.end); - } else { - panic!("unexpected filename {fname}"); - } + let fname = fname.unwrap(); + let fname = LayerFileName::from_str(&fname).unwrap(); + let layer = LayerDescriptor::from(fname); + + let lsn_range = layer.get_lsn_range(); + min_lsn = min(min_lsn, lsn_range.start); + max_lsn = max(max_lsn, Lsn(lsn_range.end.0 - 1)); + + updates.insert_historic(Arc::new(layer)); } println!("min: {min_lsn}, max: {max_lsn}"); diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index bb386b436b..59a358a355 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -59,6 +59,7 @@ use std::sync::Arc; use utils::lsn::Lsn; use historic_layer_coverage::BufferedHistoricLayerCoverage; +pub use historic_layer_coverage::Replacement; use super::storage_layer::range_eq; @@ -138,6 +139,24 @@ where self.layer_map.remove_historic_noflush(layer) } + /// Replaces existing layer iff it is the `expected`. + /// + /// If the expected layer has been removed it will not be inserted by this function. + /// + /// Returned `Replacement` describes succeeding in replacement or the reason why it could not + /// be done. + /// + /// TODO replacement can be done without buffering and rebuilding layer map updates. + /// One way to do that is to add a layer of indirection for returned values, so + /// that we can replace values only by updating a hashmap. + pub fn replace_historic( + &mut self, + expected: &Arc, + new: Arc, + ) -> anyhow::Result>> { + self.layer_map.replace_historic_noflush(expected, new) + } + // We will flush on drop anyway, but this method makes it // more explicit that there is some work being done. /// Apply all updates @@ -254,14 +273,8 @@ where /// Helper function for BatchedUpdates::insert_historic /// pub(self) fn insert_historic_noflush(&mut self, layer: Arc) { - let kr = layer.get_key_range(); - let lr = layer.get_lsn_range(); self.historic.insert( - historic_layer_coverage::LayerKey { - key: kr.start.to_i128()..kr.end.to_i128(), - lsn: lr.start.0..lr.end.0, - is_image: !layer.is_incremental(), - }, + historic_layer_coverage::LayerKey::from(&*layer), Arc::clone(&layer), ); @@ -278,30 +291,72 @@ where /// Helper function for BatchedUpdates::remove_historic /// pub fn remove_historic_noflush(&mut self, layer: Arc) { - let kr = layer.get_key_range(); - let lr = layer.get_lsn_range(); - self.historic.remove(historic_layer_coverage::LayerKey { - key: kr.start.to_i128()..kr.end.to_i128(), - lsn: lr.start.0..lr.end.0, - is_image: !layer.is_incremental(), - }); + self.historic + .remove(historic_layer_coverage::LayerKey::from(&*layer)); if Self::is_l0(&layer) { let len_before = self.l0_delta_layers.len(); - - // FIXME: ptr_eq might fail to return true for 'dyn' - // references. Clippy complains about this. In practice it - // seems to work, the assertion below would be triggered - // otherwise but this ought to be fixed. - #[allow(clippy::vtable_address_comparisons)] self.l0_delta_layers - .retain(|other| !Arc::ptr_eq(other, &layer)); - assert_eq!(self.l0_delta_layers.len(), len_before - 1); + .retain(|other| !Self::compare_arced_layers(other, &layer)); + // this assertion is related to use of Arc::ptr_eq in Self::compare_arced_layers, + // there's a chance that the comparison fails at runtime due to it comparing (pointer, + // vtable) pairs. + assert_eq!( + self.l0_delta_layers.len(), + len_before - 1, + "failed to locate removed historic layer from l0_delta_layers" + ); } NUM_ONDISK_LAYERS.dec(); } + pub(self) fn replace_historic_noflush( + &mut self, + expected: &Arc, + new: Arc, + ) -> anyhow::Result>> { + let key = historic_layer_coverage::LayerKey::from(&**expected); + let other = historic_layer_coverage::LayerKey::from(&*new); + + let expected_l0 = Self::is_l0(expected); + let new_l0 = Self::is_l0(&new); + + anyhow::ensure!( + key == other, + "expected and new must have equal LayerKeys: {key:?} != {other:?}" + ); + + anyhow::ensure!( + expected_l0 == new_l0, + "expected and new must both be l0 deltas or neither should be: {expected_l0} != {new_l0}" + ); + + let l0_index = if expected_l0 { + // find the index in case replace worked, we need to replace that as well + Some( + self.l0_delta_layers + .iter() + .position(|slot| Self::compare_arced_layers(slot, expected)) + .ok_or_else(|| anyhow::anyhow!("existing l0 delta layer was not found"))?, + ) + } else { + None + }; + + let replaced = self.historic.replace(&key, new.clone(), |existing| { + Self::compare_arced_layers(existing, expected) + }); + + if let Replacement::Replaced { .. } = &replaced { + if let Some(index) = l0_index { + self.l0_delta_layers[index] = new; + } + } + + Ok(replaced) + } + /// Helper function for BatchedUpdates::drop. pub(self) fn flush_updates(&mut self) { self.historic.rebuild(); @@ -675,4 +730,105 @@ where println!("End dump LayerMap"); Ok(()) } + + #[inline(always)] + fn compare_arced_layers(left: &Arc, right: &Arc) -> bool { + // FIXME: ptr_eq might fail to return true for 'dyn' references because of multiple vtables + // can be created in compilation. Clippy complains about this. In practice it seems to + // work. + // + // In future rust versions this might become Arc::as_ptr(left) as *const () == + // Arc::as_ptr(right) as *const (), we could change to that before. + #[allow(clippy::vtable_address_comparisons)] + Arc::ptr_eq(left, right) + } +} + +#[cfg(test)] +mod tests { + use super::{LayerMap, Replacement}; + use crate::tenant::storage_layer::{Layer, LayerDescriptor, LayerFileName}; + use std::str::FromStr; + use std::sync::Arc; + + mod l0_delta_layers_updated { + + use super::*; + + #[test] + fn for_full_range_delta() { + // l0_delta_layers are used by compaction, and should observe all buffered updates + l0_delta_layers_updated_scenario( + "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000053423C21-0000000053424D69", + true + ) + } + + #[test] + fn for_non_full_range_delta() { + // has minimal uncovered areas compared to l0_delta_layers_updated_on_insert_replace_remove_for_full_range_delta + l0_delta_layers_updated_scenario( + "000000000000000000000000000000000001-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE__0000000053423C21-0000000053424D69", + // because not full range + false + ) + } + + #[test] + fn for_image() { + l0_delta_layers_updated_scenario( + "000000000000000000000000000000000000-000000000000000000000000000000010000__0000000053424D69", + // code only checks if it is a full range layer, doesn't care about images, which must + // mean we should in practice never have full range images + false + ) + } + + fn l0_delta_layers_updated_scenario(layer_name: &str, expected_l0: bool) { + let name = LayerFileName::from_str(layer_name).unwrap(); + let skeleton = LayerDescriptor::from(name); + + let remote: Arc = Arc::new(skeleton.clone()); + let downloaded: Arc = Arc::new(skeleton); + + let mut map = LayerMap::default(); + + // two disjoint Arcs in different lifecycle phases. + assert!(!LayerMap::compare_arced_layers(&remote, &downloaded)); + + let expected_in_counts = (1, usize::from(expected_l0)); + + map.batch_update().insert_historic(remote.clone()); + assert_eq!(count_layer_in(&map, &remote), expected_in_counts); + + let replaced = map + .batch_update() + .replace_historic(&remote, downloaded.clone()) + .expect("name derived attributes are the same"); + assert!( + matches!(replaced, Replacement::Replaced { .. }), + "{replaced:?}" + ); + assert_eq!(count_layer_in(&map, &downloaded), expected_in_counts); + + map.batch_update().remove_historic(downloaded.clone()); + assert_eq!(count_layer_in(&map, &downloaded), (0, 0)); + } + + fn count_layer_in(map: &LayerMap, layer: &Arc) -> (usize, usize) { + let historic = map + .iter_historic_layers() + .filter(|x| LayerMap::compare_arced_layers(x, layer)) + .count(); + let l0s = map + .get_level0_deltas() + .expect("why does this return a result"); + let l0 = l0s + .iter() + .filter(|x| LayerMap::compare_arced_layers(x, layer)) + .count(); + + (historic, l0) + } + } } diff --git a/pageserver/src/tenant/layer_map/historic_layer_coverage.rs b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs index 46821aef15..b63c361314 100644 --- a/pageserver/src/tenant/layer_map/historic_layer_coverage.rs +++ b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs @@ -41,6 +41,18 @@ impl Ord for LayerKey { } } +impl<'a, L: crate::tenant::storage_layer::Layer + ?Sized> From<&'a L> for LayerKey { + fn from(layer: &'a L) -> Self { + let kr = layer.get_key_range(); + let lr = layer.get_lsn_range(); + LayerKey { + key: kr.start.to_i128()..kr.end.to_i128(), + lsn: lr.start.0..lr.end.0, + is_image: !layer.is_incremental(), + } + } +} + /// Efficiently queryable layer coverage for each LSN. /// /// Allows answering layer map queries very efficiently, @@ -82,15 +94,13 @@ impl HistoricLayerCoverage { } // Insert into data structure - if layer_key.is_image { - self.head - .image_coverage - .insert(layer_key.key, layer_key.lsn.clone(), value); + let target = if layer_key.is_image { + &mut self.head.image_coverage } else { - self.head - .delta_coverage - .insert(layer_key.key, layer_key.lsn.clone(), value); - } + &mut self.head.delta_coverage + }; + + target.insert(layer_key.key, layer_key.lsn.clone(), value); // Remember history. Clone is O(1) self.historic.insert(layer_key.lsn.start, self.head.clone()); @@ -415,6 +425,59 @@ impl BufferedHistoricLayerCoverage { self.buffer.insert(layer_key, None); } + /// Replaces a previous layer with a new layer value. + /// + /// The replacement is conditional on: + /// - there is an existing `LayerKey` record + /// - there is no buffered removal for the given `LayerKey` + /// - the given closure returns true for the current `Value` + /// + /// The closure is used to compare the latest value (buffered insert, or existing layer) + /// against some expectation. This allows to use `Arc::ptr_eq` or similar which would be + /// inaccessible via `PartialEq` trait. + /// + /// Returns a `Replacement` value describing the outcome; only the case of + /// `Replacement::Replaced` modifies the map and requires a rebuild. + pub fn replace( + &mut self, + layer_key: &LayerKey, + new: Value, + check_expected: F, + ) -> Replacement + where + F: FnOnce(&Value) -> bool, + { + let (slot, in_buffered) = match self.buffer.get(layer_key) { + Some(inner @ Some(_)) => { + // we compare against the buffered version, because there will be a later + // rebuild before querying + (inner.as_ref(), true) + } + Some(None) => { + // buffer has removal for this key; it will not be equivalent by any check_expected. + return Replacement::RemovalBuffered; + } + None => { + // no pending modification for the key, check layers + (self.layers.get(layer_key), false) + } + }; + + match slot { + Some(existing) if !check_expected(existing) => { + // unfortunate clone here, but otherwise the nll borrowck grows the region of + // 'a to cover the whole function, and we could not mutate in the other + // Some(existing) branch + Replacement::Unexpected(existing.clone()) + } + None => Replacement::NotFound, + Some(_existing) => { + self.insert(layer_key.to_owned(), new); + Replacement::Replaced { in_buffered } + } + } + } + pub fn rebuild(&mut self) { // Find the first LSN that needs to be rebuilt let rebuild_since: u64 = match self.buffer.iter().next() { @@ -483,6 +546,22 @@ impl BufferedHistoricLayerCoverage { } } +/// Outcome of the replace operation. +#[derive(Debug)] +pub enum Replacement { + /// Previous value was replaced with the new value. + Replaced { + /// Replacement happened for a scheduled insert. + in_buffered: bool, + }, + /// Key was not found buffered updates or existing layers. + NotFound, + /// Key has been scheduled for removal, it was not replaced. + RemovalBuffered, + /// Previous value was rejected by the closure. + Unexpected(Value), +} + #[test] fn test_retroactive_regression_1() { let mut map = BufferedHistoricLayerCoverage::new(); @@ -548,7 +627,7 @@ fn test_retroactive_simple() { LayerKey { key: 2..5, lsn: 105..106, - is_image: true, + is_image: false, }, "Delta 1".to_string(), ); @@ -556,17 +635,24 @@ fn test_retroactive_simple() { // Rebuild so we can start querying map.rebuild(); - // Query key 4 - let version = map.get().unwrap().get_version(90); - assert!(version.is_none()); - let version = map.get().unwrap().get_version(102).unwrap(); - assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string())); - let version = map.get().unwrap().get_version(107).unwrap(); - assert_eq!(version.image_coverage.query(4), Some("Delta 1".to_string())); - let version = map.get().unwrap().get_version(115).unwrap(); - assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string())); - let version = map.get().unwrap().get_version(125).unwrap(); - assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string())); + { + let map = map.get().expect("rebuilt"); + + let version = map.get_version(90); + assert!(version.is_none()); + let version = map.get_version(102).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string())); + + let version = map.get_version(107).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string())); + assert_eq!(version.delta_coverage.query(4), Some("Delta 1".to_string())); + + let version = map.get_version(115).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string())); + + let version = map.get_version(125).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string())); + } // Remove Image 3 map.remove(LayerKey { @@ -576,8 +662,147 @@ fn test_retroactive_simple() { }); map.rebuild(); - // Check deletion worked - let version = map.get().unwrap().get_version(125).unwrap(); - assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string())); - assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string())); + { + // Check deletion worked + let map = map.get().expect("rebuilt"); + let version = map.get_version(125).unwrap(); + assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string())); + assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string())); + } +} + +#[test] +fn test_retroactive_replacement() { + let mut map = BufferedHistoricLayerCoverage::new(); + + let keys = [ + LayerKey { + key: 0..5, + lsn: 100..101, + is_image: true, + }, + LayerKey { + key: 3..9, + lsn: 110..111, + is_image: true, + }, + LayerKey { + key: 4..6, + lsn: 120..121, + is_image: true, + }, + ]; + + let layers = [ + "Image 1".to_string(), + "Image 2".to_string(), + "Image 3".to_string(), + ]; + + for (key, layer) in keys.iter().zip(layers.iter()) { + map.insert(key.to_owned(), layer.to_owned()); + } + + // rebuild is not necessary here, because replace works for both buffered updates and existing + // layers. + + for (key, orig_layer) in keys.iter().zip(layers.iter()) { + let replacement = format!("Remote {orig_layer}"); + + // evict + let ret = map.replace(key, replacement.clone(), |l| l == orig_layer); + assert!( + matches!(ret, Replacement::Replaced { .. }), + "replace {orig_layer}: {ret:?}" + ); + map.rebuild(); + + let at = key.lsn.end + 1; + + let version = map.get().expect("rebuilt").get_version(at).unwrap(); + assert_eq!( + version.image_coverage.query(4).as_deref(), + Some(replacement.as_str()), + "query for 4 at version {at} after eviction", + ); + + // download + let ret = map.replace(key, orig_layer.clone(), |l| l == &replacement); + assert!( + matches!(ret, Replacement::Replaced { .. }), + "replace {orig_layer} back: {ret:?}" + ); + map.rebuild(); + let version = map.get().expect("rebuilt").get_version(at).unwrap(); + assert_eq!( + version.image_coverage.query(4).as_deref(), + Some(orig_layer.as_str()), + "query for 4 at version {at} after download", + ); + } +} + +#[test] +fn missing_key_is_not_inserted_with_replace() { + let mut map = BufferedHistoricLayerCoverage::new(); + let key = LayerKey { + key: 0..5, + lsn: 100..101, + is_image: true, + }; + + let ret = map.replace(&key, "should not replace", |_| true); + assert!(matches!(ret, Replacement::NotFound), "{ret:?}"); + map.rebuild(); + assert!(map + .get() + .expect("no changes to rebuild") + .get_version(102) + .is_none()); +} + +#[test] +fn replacing_buffered_insert_and_remove() { + let mut map = BufferedHistoricLayerCoverage::new(); + let key = LayerKey { + key: 0..5, + lsn: 100..101, + is_image: true, + }; + + map.insert(key.clone(), "Image 1"); + let ret = map.replace(&key, "Remote Image 1", |&l| l == "Image 1"); + assert!( + matches!(ret, Replacement::Replaced { in_buffered: true }), + "{ret:?}" + ); + map.rebuild(); + + assert_eq!( + map.get() + .expect("rebuilt") + .get_version(102) + .unwrap() + .image_coverage + .query(4), + Some("Remote Image 1") + ); + + map.remove(key.clone()); + let ret = map.replace(&key, "should not replace", |_| true); + assert!( + matches!(ret, Replacement::RemovalBuffered), + "cannot replace after scheduled remove: {ret:?}" + ); + + map.rebuild(); + + let ret = map.replace(&key, "should not replace", |_| true); + assert!( + matches!(ret, Replacement::NotFound), + "cannot replace after remove + rebuild: {ret:?}" + ); + + let at_version = map.get().expect("rebuilt").get_version(102); + assert!(at_version.is_none()); } diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 48db364b8c..d719e19bf9 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -212,6 +212,10 @@ impl std::fmt::Debug for dyn Layer { } /// Holds metadata about a layer without any content. Used mostly for testing. +/// +/// To use filenames as fixtures, parse them as [`LayerFileName`] then convert from that to a +/// LayerDescriptor. +#[derive(Clone)] pub struct LayerDescriptor { pub key: Range, pub lsn: Range, @@ -251,6 +255,40 @@ impl Layer for LayerDescriptor { } } +impl From for LayerDescriptor { + fn from(value: DeltaFileName) -> Self { + let short_id = value.to_string(); + LayerDescriptor { + key: value.key_range, + lsn: value.lsn_range, + is_incremental: true, + short_id, + } + } +} + +impl From for LayerDescriptor { + fn from(value: ImageFileName) -> Self { + let short_id = value.to_string(); + let lsn = value.lsn_as_range(); + LayerDescriptor { + key: value.key_range, + lsn, + is_incremental: false, + short_id, + } + } +} + +impl From for LayerDescriptor { + fn from(value: LayerFileName) -> Self { + match value { + LayerFileName::Delta(d) => Self::from(d), + LayerFileName::Image(i) => Self::from(i), + } + } +} + /// Helper enum to hold a PageServerConf, or a path /// /// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the diff --git a/pageserver/src/tenant/storage_layer/filename.rs b/pageserver/src/tenant/storage_layer/filename.rs index e77f2b25c1..bd3d2c42c1 100644 --- a/pageserver/src/tenant/storage_layer/filename.rs +++ b/pageserver/src/tenant/storage_layer/filename.rs @@ -128,6 +128,13 @@ impl Ord for ImageFileName { } } +impl ImageFileName { + pub fn lsn_as_range(&self) -> Range { + // Saves from having to copypaste this all over + self.lsn..(self.lsn + 1) + } +} + /// /// Represents the filename of an ImageLayer /// diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 29cb4af0fa..8656ad0289 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -172,7 +172,7 @@ impl RemoteLayer { tenantid, timelineid, key_range: fname.key_range.clone(), - lsn_range: fname.lsn..(fname.lsn + 1), + lsn_range: fname.lsn_as_range(), is_delta: false, is_incremental: false, file_name: fname.to_owned().into(), diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index e80da302b7..fc6f7ad96c 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3386,10 +3386,42 @@ impl Timeline { let mut layers = self_clone.layers.write().unwrap(); let mut updates = layers.batch_update(); { + use crate::tenant::layer_map::Replacement; let l: Arc = remote_layer.clone(); - updates.remove_historic(l); + match updates.replace_historic(&l, new_layer) { + Ok(Replacement::Replaced { .. }) => { /* expected */ } + Ok(Replacement::NotFound) => { + // TODO: the downloaded file should probably be removed, otherwise + // it will be added to the layermap on next load? we should + // probably restart any get_reconstruct_data search as well. + // + // See: https://github.com/neondatabase/neon/issues/3533 + error!("replacing downloaded layer into layermap failed because layer was not found"); + } + Ok(Replacement::RemovalBuffered) => { + unreachable!("current implementation does not remove anything") + } + Ok(Replacement::Unexpected(other)) => { + // if the other layer would have the same pointer value as + // expected, it means they differ only on vtables. + // + // otherwise there's no known reason for this to happen as + // compacted layers should have different covering rectangle + // leading to produce Replacement::NotFound. + + error!( + expected.ptr = ?Arc::as_ptr(&l), + other.ptr = ?Arc::as_ptr(&other), + "replacing downloaded layer into layermap failed because another layer was found instead of expected" + ); + } + Err(e) => { + // this is a precondition failure, the layer filename derived + // attributes didn't match up, which doesn't seem likely. + error!("replacing downloaded layer into layermap failed: {e:#?}") + } + } } - updates.insert_historic(new_layer); updates.flush(); drop(layers); From be81db21b91e9d82153dc6d770590810f36bbd19 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Fri, 3 Feb 2023 10:54:12 -0500 Subject: [PATCH 093/102] Revert accidental change (#3538) --- test_runner/fixtures/compare_fixtures.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 963d1bd19e..17c0b19447 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -107,9 +107,7 @@ class NeonCompare(PgCompare): self.timeline = self.env.neon_cli.create_timeline(branch_name, tenant_id=self.tenant) # Start pg - self._pg = self.env.postgres.create_start( - branch_name, "main", self.tenant, config_lines=["shared_buffers=2GB"] - ) + self._pg = self.env.postgres.create_start(branch_name, "main", self.tenant) @property def pg(self) -> PgProtocol: From 87cd2bae770fa85fd437f1a4fc0704be32830d80 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 2 Feb 2023 15:46:26 +0100 Subject: [PATCH 094/102] introduce LaunchTimestamp to identify process restarts This patch adds a LaunchTimestamp type to the `metrics` crate, along with a `libmetric_` Prometheus metric. The initial user is pageserver. In addition to exposing the Prometheus metric, it also reproduces the launch timestamp as a header in the API responses. The motivation for this is that we plan to scrape the pageserver's /v1/tenant/:tenant_id/timeline/:timeline_id/layer HTTP endpoint over time. It will soon expose access metrics (#3496) which reset upon process restart. We will use the pageserver's launch ID to identify a restart between two scrape points. However, there are other potential uses. For example, we could use the Prometheus metric to annotate Grafana plots whenever the launch timestamp changes. --- Cargo.lock | 1 + libs/metrics/Cargo.toml | 1 + libs/metrics/src/launch_timestamp.rs | 34 +++++++++++++++++++++++++ libs/metrics/src/lib.rs | 9 +++++++ libs/utils/src/http/endpoint.rs | 38 ++++++++++++++++++++++++++-- pageserver/src/bin/pageserver.rs | 24 ++++++++++++++---- pageserver/src/http/routes.rs | 10 ++++++++ test_runner/fixtures/metrics.py | 2 ++ 8 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 libs/metrics/src/launch_timestamp.rs diff --git a/Cargo.lock b/Cargo.lock index 1f10689902..612087c97a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2019,6 +2019,7 @@ dependencies = [ name = "metrics" version = "0.1.0" dependencies = [ + "chrono", "libc", "once_cell", "prometheus", diff --git a/libs/metrics/Cargo.toml b/libs/metrics/Cargo.toml index d7f7d84c27..d4323ae766 100644 --- a/libs/metrics/Cargo.toml +++ b/libs/metrics/Cargo.toml @@ -8,5 +8,6 @@ license.workspace = true prometheus.workspace = true libc.workspace = true once_cell.workspace = true +chrono.workspace = true workspace_hack.workspace = true diff --git a/libs/metrics/src/launch_timestamp.rs b/libs/metrics/src/launch_timestamp.rs new file mode 100644 index 0000000000..6b2307b3fb --- /dev/null +++ b/libs/metrics/src/launch_timestamp.rs @@ -0,0 +1,34 @@ +//! A timestamp captured at process startup to identify restarts of the process, e.g., in logs and metrics. + +use chrono::Utc; + +use super::register_uint_gauge; +use std::fmt::Display; + +pub struct LaunchTimestamp(chrono::DateTime); + +impl LaunchTimestamp { + pub fn generate() -> Self { + LaunchTimestamp(Utc::now()) + } +} + +impl Display for LaunchTimestamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +pub fn set_launch_timestamp_metric(launch_ts: &'static LaunchTimestamp) { + let millis_since_epoch: u64 = launch_ts + .0 + .timestamp_millis() + .try_into() + .expect("we're after the epoch, this should be positive"); + let metric = register_uint_gauge!( + "libmetrics_launch_timestamp", + "Timestamp (millis since epoch) at wich the process launched." + ) + .unwrap(); + metric.set(millis_since_epoch); +} diff --git a/libs/metrics/src/lib.rs b/libs/metrics/src/lib.rs index 07d220195b..6011713c8f 100644 --- a/libs/metrics/src/lib.rs +++ b/libs/metrics/src/lib.rs @@ -20,6 +20,7 @@ pub use prometheus::{register_int_gauge_vec, IntGaugeVec}; pub use prometheus::{Encoder, TextEncoder}; use prometheus::{Registry, Result}; +pub mod launch_timestamp; mod wrappers; pub use wrappers::{CountedReader, CountedWriter}; @@ -34,6 +35,14 @@ macro_rules! register_uint_gauge_vec { }}; } +#[macro_export] +macro_rules! register_uint_gauge { + ($NAME:expr, $HELP:expr $(,)?) => {{ + let gauge = $crate::UIntGauge::new($NAME, $HELP).unwrap(); + $crate::register(Box::new(gauge.clone())).map(|_| gauge) + }}; +} + /// Special internal registry, to collect metrics independently from the default registry. /// Was introduced to fix deadlock with lazy registration of metrics in the default registry. static INTERNAL_REGISTRY: Lazy = Lazy::new(Registry::new); diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index fecbbb945b..483ff15c55 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -1,7 +1,8 @@ use crate::auth::{Claims, JwtAuth}; use crate::http::error; -use anyhow::anyhow; -use hyper::header::AUTHORIZATION; +use anyhow::{anyhow, Context}; +use hyper::header::{HeaderName, AUTHORIZATION}; +use hyper::http::HeaderValue; use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server}; use metrics::{register_int_counter, Encoder, IntCounter, TextEncoder}; use once_cell::sync::Lazy; @@ -13,6 +14,7 @@ use tracing::info; use std::future::Future; use std::net::TcpListener; +use std::str::FromStr; use super::error::ApiError; @@ -143,6 +145,38 @@ pub fn auth_middleware( }) } +pub fn add_response_header_middleware( + header: &str, + value: &str, +) -> anyhow::Result> +where + B: hyper::body::HttpBody + Send + Sync + 'static, +{ + let name = + HeaderName::from_str(header).with_context(|| format!("invalid header name: {header}"))?; + let value = + HeaderValue::from_str(value).with_context(|| format!("invalid header value: {value}"))?; + Ok(Middleware::post_with_info( + move |mut response, request_info| { + let name = name.clone(); + let value = value.clone(); + async move { + let headers = response.headers_mut(); + if headers.contains_key(&name) { + tracing::warn!( + "{} response already contains header {:?}", + request_info.uri(), + &name, + ); + } else { + headers.insert(name, value); + } + Ok(response) + } + }, + )) +} + pub fn check_permission_with( req: &Request, check_permission: impl Fn(&Claims) -> Result<(), anyhow::Error>, diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index f2cd93bd3a..fede5c753b 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -7,6 +7,7 @@ use std::{env, ops::ControlFlow, path::Path, str::FromStr}; use anyhow::{anyhow, Context}; use clap::{Arg, ArgAction, Command}; use fail::FailScenario; +use metrics::launch_timestamp::{set_launch_timestamp_metric, LaunchTimestamp}; use remote_storage::GenericRemoteStorage; use tracing::*; @@ -52,6 +53,8 @@ fn version() -> String { } fn main() -> anyhow::Result<()> { + let launch_ts = Box::leak(Box::new(LaunchTimestamp::generate())); + let arg_matches = cli().get_matches(); if arg_matches.get_flag("enabled-features") { @@ -108,7 +111,7 @@ fn main() -> anyhow::Result<()> { virtual_file::init(conf.max_file_descriptors); page_cache::init(conf.page_cache_size); - start_pageserver(conf).context("Failed to start pageserver")?; + start_pageserver(launch_ts, conf).context("Failed to start pageserver")?; scenario.teardown(); Ok(()) @@ -203,13 +206,24 @@ fn initialize_config( }) } -fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { +fn start_pageserver( + launch_ts: &'static LaunchTimestamp, + conf: &'static PageServerConf, +) -> anyhow::Result<()> { // Initialize logging logging::init(conf.log_format)?; - // Print version to the log, and expose it as a prometheus metric too. - info!("version: {}", version()); + // Print version and launch timestamp to the log, + // and expose them as prometheus metrics. + // A changed version string indicates changed software. + // A changed launch timestamp indicates a pageserver restart. + info!( + "version: {} launch_timestamp: {}", + version(), + launch_ts.to_string() + ); set_build_info_metric(GIT_VERSION); + set_launch_timestamp_metric(launch_ts); // If any failpoints were set from FAILPOINTS environment variable, // print them to the log for debugging purposes @@ -307,7 +321,7 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> { { let _rt_guard = MGMT_REQUEST_RUNTIME.enter(); - let router = http::make_router(conf, auth.clone(), remote_storage)? + let router = http::make_router(conf, launch_ts, auth.clone(), remote_storage)? .build() .map_err(|err| anyhow!(err))?; let service = utils::http::RouterService::new(router).unwrap(); diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 77a8f63da9..f81f5da84c 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; +use metrics::launch_timestamp::LaunchTimestamp; use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest; use remote_storage::GenericRemoteStorage; use tokio_util::sync::CancellationToken; @@ -921,6 +922,7 @@ async fn handler_404(_: Request) -> Result, ApiError> { pub fn make_router( conf: &'static PageServerConf, + launch_ts: &'static LaunchTimestamp, auth: Option>, remote_storage: Option, ) -> anyhow::Result> { @@ -937,6 +939,14 @@ pub fn make_router( })) } + router = router.middleware( + endpoint::add_response_header_middleware( + "PAGESERVER_LAUNCH_TIMESTAMP", + &launch_ts.to_string(), + ) + .expect("construct launch timestamp header middleware"), + ); + macro_rules! testing_api { ($handler_desc:literal, $handler:path $(,)?) => {{ #[cfg(not(feature = "testing"))] diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index bdaaa95216..ba0d325c39 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -50,6 +50,8 @@ PAGESERVER_GLOBAL_METRICS: Tuple[str, ...] = ( "pageserver_storage_operations_seconds_global_count", "pageserver_storage_operations_seconds_global_sum", "pageserver_storage_operations_seconds_global_bucket", + "libmetrics_launch_timestamp", + "libmetrics_build_info", ) PAGESERVER_PER_TENANT_METRICS: Tuple[str, ...] = ( From ec3a3aed3764730c4b3331a09e39820a38a0e4ae Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Sat, 4 Feb 2023 01:32:29 +0200 Subject: [PATCH 095/102] Dump current tenant config (#3534) The PR adds an endpoint to show tenant's current config: `GET /v1/tenant/:tenant_id/config` Tenant's config consists of two parts: tenant overrides (could be changed via other management API requests) and the default part, substituting all missing overrides (constant, hardcoded in pageserver). The API returns the custom overrides and the final tenant config, after applying all the defaults. Along the way, it had to fix two things in the config: * allow to shorten the json version and omit all `null`'s (same as toml serializer behaves by default), and to understand such shortened format when deserialized. A unit test is added * fix a bug, when `PUT /v1/tenant/config` endpoint rewritten the local file with what had came in the request, but updating (not rewriting the old values) the in-memory state instead. That got uncovered during adjusting the e2e test and fixed to do the replacement everywhere, otherwise there's no way to revert existing overrides. Fixes #3471 (commit https://github.com/neondatabase/neon/commit/dc688affe8f9daa601fb4af700998bcd6f21ea57) * fixes https://github.com/neondatabase/neon/issues/3472 by reordering the config saving operations --- libs/pageserver_api/src/models.rs | 1 - pageserver/src/http/openapi_spec.yml | 72 ++++++++++++++ pageserver/src/http/routes.rs | 46 +++++++-- pageserver/src/tenant.rs | 15 ++- pageserver/src/tenant/config.rs | 60 ++++++++++++ pageserver/src/tenant/mgr.rs | 13 ++- test_runner/fixtures/neon_fixtures.py | 18 ++++ test_runner/regress/test_tenant_conf.py | 120 ++++++++++++++++++++---- 8 files changed, 307 insertions(+), 38 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 6f53f4a01d..8827235d90 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -137,7 +137,6 @@ pub struct TenantConfigRequest { #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, #[serde(default)] - #[serde_as(as = "Option")] pub checkpoint_distance: Option, pub checkpoint_timeout: Option, pub compaction_target_size: Option, diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 23faff7ace..fc271fe83b 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -664,6 +664,55 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + /v1/tenant/{tenant_id}/config/: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex + get: + description: | + Returns tenant's config description: specific config overrides a tenant has + and the effective config. + responses: + "200": + description: Tenant config, specific and effective + content: + application/json: + schema: + $ref: "#/components/schemas/TenantConfig" + "400": + description: Malformed get tenanant config request + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "401": + description: Unauthorized Error + content: + application/json: + schema: + $ref: "#/components/schemas/UnauthorizedError" + "403": + description: Forbidden Error + content: + application/json: + schema: + $ref: "#/components/schemas/ForbiddenError" + "404": + description: Tenand or timeline were not found + content: + application/json: + schema: + $ref: "#/components/schemas/NotFoundError" + "500": + description: Generic operation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" components: securitySchemes: JWT: @@ -724,10 +773,33 @@ components: type: integer checkpoint_timeout: type: string + compaction_target_size: + type: integer compaction_period: type: string compaction_threshold: type: string + image_creation_threshold: + type: integer + walreceiver_connect_timeout: + type: string + lagging_wal_timeout: + type: string + max_lsn_wal_lag: + type: integer + trace_read_requests: + type: boolean + TenantConfig: + type: object + properties: + tenant_specific_overrides: + type: object + schema: + $ref: "#/components/schemas/TenantConfigInfo" + effective_config: + type: object + schema: + $ref: "#/components/schemas/TenantConfigInfo" TimelineInfo: type: object required: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index f81f5da84c..b0b07428f9 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::Arc; use anyhow::{anyhow, Context, Result}; @@ -698,12 +699,40 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result, ApiError> { +async fn get_tenant_config_handler(request: Request) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + check_permission(&request, Some(tenant_id))?; + + let tenant = mgr::get_tenant(tenant_id, false) + .await + .map_err(ApiError::NotFound)?; + + let response = HashMap::from([ + ( + "tenant_specific_overrides", + serde_json::to_value(tenant.tenant_specific_overrides()) + .context("serializing tenant specific overrides") + .map_err(ApiError::InternalServerError)?, + ), + ( + "effective_config", + serde_json::to_value(tenant.effective_config()) + .context("serializing effective config") + .map_err(ApiError::InternalServerError)?, + ), + ]); + + json_response(StatusCode::OK, response) +} + +async fn update_tenant_config_handler( + mut request: Request, +) -> Result, ApiError> { let request_data: TenantConfigRequest = json_request(&mut request).await?; let tenant_id = request_data.tenant_id; check_permission(&request, Some(tenant_id))?; - let mut tenant_conf: TenantConfOpt = Default::default(); + let mut tenant_conf = TenantConfOpt::default(); if let Some(gc_period) = request_data.gc_period { tenant_conf.gc_period = Some( humantime::parse_duration(&gc_period) @@ -738,12 +767,8 @@ async fn tenant_config_handler(mut request: Request) -> Result) -> Result TenantConfOpt { + *self.tenant_conf.read().unwrap() + } + + pub fn effective_config(&self) -> TenantConf { + self.tenant_specific_overrides() + .merge(self.conf.default_tenant_conf) + } + pub fn get_checkpoint_distance(&self) -> u64 { let tenant_conf = self.tenant_conf.read().unwrap(); tenant_conf @@ -1690,8 +1699,8 @@ impl Tenant { .unwrap_or(self.conf.default_tenant_conf.trace_read_requests) } - pub fn update_tenant_config(&self, new_tenant_conf: TenantConfOpt) { - self.tenant_conf.write().unwrap().update(&new_tenant_conf); + pub fn set_new_tenant_config(&self, new_tenant_conf: TenantConfOpt) { + *self.tenant_conf.write().unwrap() = new_tenant_conf; } fn create_timeline_data( diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index e66ee0ae36..087cff2537 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -51,6 +51,7 @@ pub struct TenantConf { pub checkpoint_distance: u64, // Inmemory layer is also flushed at least once in checkpoint_timeout to // eventually upload WAL after activity is stopped. + #[serde(with = "humantime_serde")] pub checkpoint_timeout: Duration, // Target file size, when creating image and delta layers. // This parameter determines L1 layer file size. @@ -96,23 +97,61 @@ pub struct TenantConf { /// which parameters are set and which are not. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct TenantConfOpt { + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub checkpoint_distance: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub checkpoint_timeout: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub compaction_target_size: Option, + + #[serde(skip_serializing_if = "Option::is_none")] #[serde(with = "humantime_serde")] + #[serde(default)] pub compaction_period: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub compaction_threshold: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub gc_horizon: Option, + + #[serde(skip_serializing_if = "Option::is_none")] #[serde(with = "humantime_serde")] + #[serde(default)] pub gc_period: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub image_creation_threshold: Option, + + #[serde(skip_serializing_if = "Option::is_none")] #[serde(with = "humantime_serde")] + #[serde(default)] pub pitr_interval: Option, + + #[serde(skip_serializing_if = "Option::is_none")] #[serde(with = "humantime_serde")] + #[serde(default)] pub walreceiver_connect_timeout: Option, + + #[serde(skip_serializing_if = "Option::is_none")] #[serde(with = "humantime_serde")] + #[serde(default)] pub lagging_wal_timeout: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub max_lsn_wal_lag: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] pub trace_read_requests: Option, } @@ -225,3 +264,24 @@ impl Default for TenantConf { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn de_serializing_pageserver_config_omits_empty_values() { + let small_conf = TenantConfOpt { + gc_horizon: Some(42), + ..TenantConfOpt::default() + }; + + let toml_form = toml_edit::easy::to_string(&small_conf).unwrap(); + assert_eq!(toml_form, "gc_horizon = 42\n"); + assert_eq!(small_conf, toml_edit::easy::from_str(&toml_form).unwrap()); + + let json_form = serde_json::to_string(&small_conf).unwrap(); + assert_eq!(json_form, "{\"gc_horizon\":42}"); + assert_eq!(small_conf, serde_json::from_str(&json_form).unwrap()); + } +} diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index a9edee3794..a74dfdea04 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -285,17 +285,22 @@ pub async fn create_tenant( }).await } -pub async fn update_tenant_config( +pub async fn set_new_tenant_config( conf: &'static PageServerConf, - tenant_conf: TenantConfOpt, + new_tenant_conf: TenantConfOpt, tenant_id: TenantId, ) -> anyhow::Result<()> { info!("configuring tenant {tenant_id}"); let tenant = get_tenant(tenant_id, true).await?; - tenant.update_tenant_config(tenant_conf); let tenant_config_path = conf.tenant_config_path(tenant_id); - Tenant::persist_tenant_config(&tenant.tenant_id(), &tenant_config_path, tenant_conf, false)?; + Tenant::persist_tenant_config( + &tenant.tenant_id(), + &tenant_config_path, + new_tenant_conf, + false, + )?; + tenant.set_new_tenant_config(new_tenant_conf); Ok(()) } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 58e374eb3c..b35252243e 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1205,6 +1205,11 @@ class PageserverHttpClient(requests.Session): assert isinstance(res_json, dict) return res_json + def tenant_config(self, tenant_id: TenantId) -> TenantConfig: + res = self.get(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/config") + self.verbose_error(res) + return TenantConfig.from_json(res.json()) + def tenant_size(self, tenant_id: TenantId) -> int: return self.tenant_size_and_modelinputs(tenant_id)[0] @@ -1500,6 +1505,19 @@ class PageserverHttpClient(requests.Session): assert res.status_code == 200 +@dataclass +class TenantConfig: + tenant_specific_overrides: Dict[str, Any] + effective_config: Dict[str, Any] + + @classmethod + def from_json(cls, d: Dict[str, Any]) -> TenantConfig: + return TenantConfig( + tenant_specific_overrides=d["tenant_specific_overrides"], + effective_config=d["effective_config"], + ) + + @dataclass class LayerMapInfo: in_memory_layers: List[InMemoryLayerInfo] diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index cbbf01a285..e087891bba 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -22,6 +22,7 @@ wait_lsn_timeout='111 s'; tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" env = neon_env_builder.init_start() + http_client = env.pageserver.http_client() # Check that we raise on misspelled configs invalid_conf_key = "some_invalid_setting_name_blah_blah_123" @@ -36,12 +37,11 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" else: raise AssertionError("Expected validation error") - tenant, _ = env.neon_cli.create_tenant( - conf={ - "checkpoint_distance": "20000", - "gc_period": "30sec", - } - ) + new_conf = { + "checkpoint_distance": "20000", + "gc_period": "30sec", + } + tenant, _ = env.neon_cli.create_tenant(conf=new_conf) env.neon_cli.create_timeline("test_tenant_conf", tenant_id=tenant) env.postgres.create_start( @@ -69,7 +69,20 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" "image_creation_threshold": 3, "pitr_interval": 604800, # 7 days }.items() - ) + ), f"Unexpected res: {res}" + default_tenant_config = http_client.tenant_config(tenant_id=env.initial_tenant) + assert ( + not default_tenant_config.tenant_specific_overrides + ), "Should have no specific settings yet" + effective_config = default_tenant_config.effective_config + assert effective_config["checkpoint_distance"] == 10000 + assert effective_config["compaction_target_size"] == 1048576 + assert effective_config["compaction_period"] == "20s" + assert effective_config["compaction_threshold"] == 10 + assert effective_config["gc_horizon"] == 67108864 + assert effective_config["gc_period"] == "1h" + assert effective_config["image_creation_threshold"] == 3 + assert effective_config["pitr_interval"] == "7days" # check the configuration of the new tenant with closing(env.pageserver.connect()) as psconn: @@ -89,15 +102,37 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" "image_creation_threshold": 3, "pitr_interval": 604800, }.items() - ) + ), f"Unexpected res: {res}" + new_tenant_config = http_client.tenant_config(tenant_id=tenant) + new_specific_config = new_tenant_config.tenant_specific_overrides + assert new_specific_config["checkpoint_distance"] == 20000 + assert new_specific_config["gc_period"] == "30s" + assert len(new_specific_config) == len( + new_conf + ), f"No more specific properties were expected, but got: {new_specific_config}" + new_effective_config = new_tenant_config.effective_config + assert ( + new_effective_config["checkpoint_distance"] == 20000 + ), "Specific 'checkpoint_distance' config should override the default value" + assert ( + new_effective_config["gc_period"] == "30s" + ), "Specific 'gc_period' config should override the default value" + assert new_effective_config["compaction_target_size"] == 1048576 + assert new_effective_config["compaction_period"] == "20s" + assert new_effective_config["compaction_threshold"] == 10 + assert new_effective_config["gc_horizon"] == 67108864 + assert new_effective_config["image_creation_threshold"] == 3 + assert new_effective_config["pitr_interval"] == "7days" # update the config and ensure that it has changed + conf_update = { + "checkpoint_distance": "15000", + "gc_period": "80sec", + "compaction_period": "80sec", + } env.neon_cli.config_tenant( tenant_id=tenant, - conf={ - "checkpoint_distance": "15000", - "gc_period": "80sec", - }, + conf=conf_update, ) with closing(env.pageserver.connect()) as psconn: @@ -110,14 +145,37 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" for i in { "checkpoint_distance": 15000, "compaction_target_size": 1048576, - "compaction_period": 20, + "compaction_period": 80, "compaction_threshold": 10, "gc_horizon": 67108864, "gc_period": 80, "image_creation_threshold": 3, "pitr_interval": 604800, }.items() - ) + ), f"Unexpected res: {res}" + updated_tenant_config = http_client.tenant_config(tenant_id=tenant) + updated_specific_config = updated_tenant_config.tenant_specific_overrides + assert updated_specific_config["checkpoint_distance"] == 15000 + assert updated_specific_config["gc_period"] == "1m 20s" + assert updated_specific_config["compaction_period"] == "1m 20s" + assert len(updated_specific_config) == len( + conf_update + ), f"No more specific properties were expected, but got: {updated_specific_config}" + updated_effective_config = updated_tenant_config.effective_config + assert ( + updated_effective_config["checkpoint_distance"] == 15000 + ), "Specific 'checkpoint_distance' config should override the default value" + assert ( + updated_effective_config["gc_period"] == "1m 20s" + ), "Specific 'gc_period' config should override the default value" + assert ( + updated_effective_config["compaction_period"] == "1m 20s" + ), "Specific 'compaction_period' config should override the default value" + assert updated_effective_config["compaction_target_size"] == 1048576 + assert updated_effective_config["compaction_threshold"] == 10 + assert updated_effective_config["gc_horizon"] == 67108864 + assert updated_effective_config["image_creation_threshold"] == 3 + assert updated_effective_config["pitr_interval"] == "7days" # restart the pageserver and ensure that the config is still correct env.pageserver.stop() @@ -133,22 +191,44 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" for i in { "checkpoint_distance": 15000, "compaction_target_size": 1048576, - "compaction_period": 20, + "compaction_period": 80, "compaction_threshold": 10, "gc_horizon": 67108864, "gc_period": 80, "image_creation_threshold": 3, "pitr_interval": 604800, }.items() - ) + ), f"Unexpected res: {res}" + restarted_tenant_config = http_client.tenant_config(tenant_id=tenant) + assert ( + restarted_tenant_config == updated_tenant_config + ), "Updated config should not change after the restart" # update the config with very short config and make sure no trailing chars are left from previous config + final_conf = { + "pitr_interval": "1 min", + } env.neon_cli.config_tenant( tenant_id=tenant, - conf={ - "pitr_interval": "1 min", - }, + conf=final_conf, ) + final_tenant_config = http_client.tenant_config(tenant_id=tenant) + final_specific_config = final_tenant_config.tenant_specific_overrides + assert final_specific_config["pitr_interval"] == "1m" + assert len(final_specific_config) == len( + final_conf + ), f"No more specific properties were expected, but got: {final_specific_config}" + final_effective_config = final_tenant_config.effective_config + assert ( + final_effective_config["pitr_interval"] == "1m" + ), "Specific 'pitr_interval' config should override the default value" + assert final_effective_config["checkpoint_distance"] == 10000 + assert final_effective_config["compaction_target_size"] == 1048576 + assert final_effective_config["compaction_period"] == "20s" + assert final_effective_config["compaction_threshold"] == 10 + assert final_effective_config["gc_horizon"] == 67108864 + assert final_effective_config["gc_period"] == "1h" + assert final_effective_config["image_creation_threshold"] == 3 # restart the pageserver and ensure that the config is still correct env.pageserver.stop() @@ -165,7 +245,7 @@ tenant_config={checkpoint_distance = 10000, compaction_target_size = 1048576}""" "compaction_period": 20, "pitr_interval": 60, }.items() - ) + ), f"Unexpected res: {res}" def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder): From 121d535068bc9cbe4d6de1b0ac7fe9c7270ac9d7 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 3 Feb 2023 00:41:32 +0100 Subject: [PATCH 096/102] Add a test for the PostgresNIO Swift client library. It supports SNI. See discussion at https://community.neon.tech/t/postgresnio-swift-sni-support/419/4 --- .../swift/PostgresNIOExample/.dockerignore | 1 + .../swift/PostgresNIOExample/.gitignore | 1 + .../swift/PostgresNIOExample/Dockerfile | 10 +++ .../swift/PostgresNIOExample/Package.resolved | 86 +++++++++++++++++++ .../swift/PostgresNIOExample/Package.swift | 17 ++++ .../Sources/PostgresNIOExample/main.swift | 49 +++++++++++ test_runner/pg_clients/test_pg_clients.py | 1 + 7 files changed, 165 insertions(+) create mode 100644 test_runner/pg_clients/swift/PostgresNIOExample/.dockerignore create mode 100644 test_runner/pg_clients/swift/PostgresNIOExample/.gitignore create mode 100644 test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile create mode 100644 test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved create mode 100644 test_runner/pg_clients/swift/PostgresNIOExample/Package.swift create mode 100644 test_runner/pg_clients/swift/PostgresNIOExample/Sources/PostgresNIOExample/main.swift diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/.dockerignore b/test_runner/pg_clients/swift/PostgresNIOExample/.dockerignore new file mode 100644 index 0000000000..30bcfa4ed5 --- /dev/null +++ b/test_runner/pg_clients/swift/PostgresNIOExample/.dockerignore @@ -0,0 +1 @@ +.build/ diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/.gitignore b/test_runner/pg_clients/swift/PostgresNIOExample/.gitignore new file mode 100644 index 0000000000..30bcfa4ed5 --- /dev/null +++ b/test_runner/pg_clients/swift/PostgresNIOExample/.gitignore @@ -0,0 +1 @@ +.build/ diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile b/test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile new file mode 100644 index 0000000000..629422d220 --- /dev/null +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile @@ -0,0 +1,10 @@ +FROM swift:5.7 AS build +WORKDIR /source + +COPY . . +RUN swift build --configuration release + +FROM swift:5.7 +WORKDIR /app +COPY --from=build /source/.build/release . +CMD ["/app/PostgresNIOExample"] diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved b/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved new file mode 100644 index 0000000000..8246567b24 --- /dev/null +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved @@ -0,0 +1,86 @@ +{ + "pins" : [ + { + "identity" : "postgres-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vapor/postgres-nio.git", + "state" : { + "revision" : "7daf026e145de2c07d6e37f4171b1acb4b5f22b1", + "version" : "1.12.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "ff3d2212b6b093db7f177d0855adbc4ef9c5f036", + "version" : "1.0.3" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "937e904258d22af6e447a0b72c0bc67583ef64a2", + "version" : "1.0.4" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "75ec60b8b4cc0f085c3ac414f3dca5625fa3588e", + "version" : "2.2.4" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "32e8d724467f8fe623624570367e3d50c5638e46", + "version" : "1.5.2" + } + }, + { + "identity" : "swift-metrics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-metrics.git", + "state" : { + "revision" : "9b39d811a83cf18b79d7d5513b06f8b290198b10", + "version" : "2.3.3" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "45167b8006448c79dda4b7bd604e07a034c15c49", + "version" : "2.48.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "4fb7ead803e38949eb1d6fabb849206a72c580f3", + "version" : "2.23.0" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "c0d9a144cfaec8d3d596aadde3039286a266c15c", + "version" : "1.15.0" + } + } + ], + "version" : 2 +} diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift b/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift new file mode 100644 index 0000000000..c64013b9ee --- /dev/null +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift @@ -0,0 +1,17 @@ +// swift-tools-version:5.7 +import PackageDescription + +let package = Package( + name: "PostgresNIOExample", + dependencies: [ + .package(url: "https://github.com/vapor/postgres-nio.git", from: "1.8.0") + ], + targets: [ + .executableTarget( + name: "PostgresNIOExample", + dependencies: [ + .product(name: "PostgresNIO", package: "postgres-nio"), + ] + ) + ] +) diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Sources/PostgresNIOExample/main.swift b/test_runner/pg_clients/swift/PostgresNIOExample/Sources/PostgresNIOExample/main.swift new file mode 100644 index 0000000000..092a0b31f3 --- /dev/null +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Sources/PostgresNIOExample/main.swift @@ -0,0 +1,49 @@ +import Foundation + +import PostgresNIO +import NIOPosix +import Logging + +await Task { + do { + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) + let logger = Logger(label: "postgres-logger") + + let env = ProcessInfo.processInfo.environment + + let sslContext = try! NIOSSLContext(configuration: .makeClientConfiguration()) + + let config = PostgresConnection.Configuration( + connection: .init( + host: env["NEON_HOST"] ?? "", + port: 5432 + ), + authentication: .init( + username: env["NEON_USER"] ?? "", + database: env["NEON_DATABASE"] ?? "", + password: env["NEON_PASSWORD"] ?? "" + ), + tls: .require(sslContext) + ) + + let connection = try await PostgresConnection.connect( + on: eventLoopGroup.next(), + configuration: config, + id: 1, + logger: logger + ) + + let rows = try await connection.query("SELECT 1 as col", logger: logger) + for try await (n) in rows.decode((Int).self, context: .default) { + print(n) + } + + // Close your connection once done + try await connection.close() + + // Shutdown the EventLoopGroup, once all connections are closed. + try eventLoopGroup.syncShutdownGracefully() + } catch { + print(error) + } +}.value diff --git a/test_runner/pg_clients/test_pg_clients.py b/test_runner/pg_clients/test_pg_clients.py index 8018f1d004..5a8da56680 100644 --- a/test_runner/pg_clients/test_pg_clients.py +++ b/test_runner/pg_clients/test_pg_clients.py @@ -19,6 +19,7 @@ from fixtures.utils import subprocess_capture "swift/PostgresClientKitExample", # See https://github.com/neondatabase/neon/pull/2008#discussion_r911896592 marks=pytest.mark.xfail(reason="Neither SNI nor parameters is supported"), ), + "swift/PostgresNIOExample", "typescript/postgresql-client", ], ) From c9821f13e0c8a601d1b2ffa50ad1b8d6c0beb1de Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Mon, 6 Feb 2023 09:25:15 +0200 Subject: [PATCH 097/102] Expose the tenant calculated synthetic size as a Prometheus metric (#3541) ## Describe your changes Expose the currently calculated synthetic size as a Prometheus metric ## Issue ticket number and link #3509 ## Checklist before requesting a review - [X] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. --- pageserver/src/http/routes.rs | 6 +++++- pageserver/src/metrics.rs | 10 ++++++++++ pageserver/src/tenant.rs | 16 +++++++++++++++- test_runner/regress/test_tenant_size.py | 12 ++++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index b0b07428f9..7279ce0f63 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -496,7 +496,11 @@ async fn tenant_size_handler(request: Request) -> Result, A .map_err(ApiError::InternalServerError)?; let size = if !inputs_only.unwrap_or(false) { - Some(inputs.calculate().map_err(ApiError::InternalServerError)?) + Some( + tenant + .calc_and_update_cached_synthetic_size(&inputs) + .map_err(ApiError::InternalServerError)?, + ) } else { None }; diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 6bd0eddbb5..9d3d11eba8 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -150,6 +150,15 @@ pub static TENANT_STATE_METRIC: Lazy = Lazy::new(|| { .expect("Failed to register pageserver_tenant_states_count metric") }); +pub static TENANT_SYNTHETIC_SIZE_METRIC: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_tenant_synthetic_size", + "Synthetic size of each tenant", + &["tenant_id"] + ) + .expect("Failed to register pageserver_tenant_synthetic_size metric") +}); + // Metrics for cloud upload. These metrics reflect data uploaded to cloud storage, // or in testing they estimate how much we would upload if we did. static NUM_PERSISTENT_FILES_CREATED: Lazy = Lazy::new(|| { @@ -593,6 +602,7 @@ impl Drop for TimelineMetrics { pub fn remove_tenant_metrics(tenant_id: &TenantId) { let tid = tenant_id.to_string(); + let _ = TENANT_SYNTHETIC_SIZE_METRIC.remove_label_values(&[&tid]); for state in TENANT_STATE_OPTIONS { let _ = TENANT_STATE_METRIC.remove_label_values(&[&tid, state]); } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 4bcf9bb06a..bc943372f8 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -52,7 +52,7 @@ use crate::config::PageServerConf; use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir; use crate::is_uninit_mark; -use crate::metrics::{remove_tenant_metrics, TENANT_STATE_METRIC}; +use crate::metrics::{remove_tenant_metrics, TENANT_STATE_METRIC, TENANT_SYNTHETIC_SIZE_METRIC}; use crate::repository::GcResult; use crate::task_mgr; use crate::task_mgr::TaskKind; @@ -2441,13 +2441,27 @@ impl Tenant { pub async fn calculate_synthetic_size(&self, ctx: &RequestContext) -> anyhow::Result { let inputs = self.gather_size_inputs(ctx).await?; + self.calc_and_update_cached_synthetic_size(&inputs) + } + + /// Calculate synthetic size , cache it and set metric value + pub fn calc_and_update_cached_synthetic_size( + &self, + inputs: &size::ModelInputs, + ) -> anyhow::Result { let size = inputs.calculate()?; self.cached_synthetic_tenant_size .store(size, Ordering::Relaxed); + TENANT_SYNTHETIC_SIZE_METRIC + .get_metric_with_label_values(&[&self.tenant_id.to_string()]) + .unwrap() + .set(size); + Ok(size) } + pub fn get_cached_synthetic_size(&self) -> u64 { self.cached_synthetic_tenant_size.load(Ordering::Relaxed) } diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index 72cfbc9dda..bb3bca8782 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -2,6 +2,7 @@ from typing import Any, List, Tuple import pytest from fixtures.log_helper import log +from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, wait_for_last_flush_lsn from fixtures.types import Lsn @@ -368,6 +369,17 @@ def test_single_branch_get_tenant_size_grows(neon_env_builder: NeonEnvBuilder): assert size_after == prev, "size after restarting pageserver should not have changed" + ps_metrics = parse_metrics(http_client.get_metrics(), "pageserver") + tenant_metric_filter = { + "tenant_id": str(tenant_id), + } + + tenant_size_metric = int( + ps_metrics.query_one("pageserver_tenant_synthetic_size", filter=tenant_metric_filter).value + ) + + assert tenant_size_metric == size_after, "API size value should be equal to metric size value" + def test_get_tenant_size_with_multiple_branches(neon_env_builder: NeonEnvBuilder): """ From 678fe0684fa28dd168f56cf7b0ea71ab71efc580 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 6 Feb 2023 14:21:51 +0200 Subject: [PATCH 098/102] std::fmt::Debug for Layer implementations (#3536) Follow-up to #3513. This removes the old blanket `std::fmt::Debug` impl on `dyn Layer` which did not seem to be used from anywhere (no compilation errors after removing). Adds `std::fmt::Debug` requirement and implementations for `trait Layer` implementors: - LayerDescriptor (derived) - RemoteLayer (manual) - DeltaLayer (manual) - ImageLayer (manual) Manual implementations are used to skip PageserverConf, tenant and timeline ids, large collections. Adds and adjusts some doc comments to be more rustdoc alike. --- pageserver/src/tenant/storage_layer.rs | 19 ++++------- .../src/tenant/storage_layer/delta_layer.rs | 34 +++++++++++++++---- .../src/tenant/storage_layer/image_layer.rs | 29 +++++++++++++--- .../tenant/storage_layer/inmemory_layer.rs | 17 ++++++++++ .../src/tenant/storage_layer/remote_layer.rs | 19 ++++++++++- 5 files changed, 94 insertions(+), 24 deletions(-) diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index d719e19bf9..1867d489c8 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -85,7 +85,11 @@ pub enum ValueReconstructResult { /// Supertrait of the [`Layer`] trait that captures the bare minimum interface /// required by [`LayerMap`]. -pub trait Layer: Send + Sync { +/// +/// All layers should implement a minimal `std::fmt::Debug` without tenant or +/// timeline names, because those are known in the context of which the layers +/// are used in (timeline). +pub trait Layer: std::fmt::Debug + Send + Sync { /// Range of keys that this layer covers fn get_key_range(&self) -> Range; @@ -148,8 +152,7 @@ pub type LayerKeyIter<'i> = Box + 'i>; /// Furthermore, there are two kinds of on-disk layers: delta and image layers. /// A delta layer contains all modifications within a range of LSNs and keys. /// An image layer is a snapshot of all the data in a key-range, at a single -/// LSN -/// +/// LSN. pub trait PersistentLayer: Layer { fn get_tenant_id(&self) -> TenantId; @@ -203,19 +206,11 @@ pub fn downcast_remote_layer( } } -impl std::fmt::Debug for dyn Layer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Layer") - .field("short_id", &self.short_id()) - .finish() - } -} - /// Holds metadata about a layer without any content. Used mostly for testing. /// /// To use filenames as fixtures, parse them as [`LayerFileName`] then convert from that to a /// LayerDescriptor. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct LayerDescriptor { pub key: Range, pub lsn: Range, diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 3a23e96325..142a8ce913 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -167,14 +167,13 @@ impl DeltaKey { } } +/// DeltaLayer is the in-memory data structure associated with an on-disk delta +/// file. /// -/// DeltaLayer is the in-memory data structure associated with an -/// on-disk delta file. We keep a DeltaLayer in memory for each -/// file, in the LayerMap. If a layer is in "loaded" state, we have a -/// copy of the index in memory, in 'inner'. Otherwise the struct is -/// just a placeholder for a file that exists on disk, and it needs to -/// be loaded before using it in queries. -/// +/// We keep a DeltaLayer in memory for each file, in the LayerMap. If a layer +/// is in "loaded" state, we have a copy of the index in memory, in 'inner'. +/// Otherwise the struct is just a placeholder for a file that exists on disk, +/// and it needs to be loaded before using it in queries. pub struct DeltaLayer { path_or_conf: PathOrConf, @@ -188,6 +187,17 @@ pub struct DeltaLayer { inner: RwLock, } +impl std::fmt::Debug for DeltaLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeltaLayer") + .field("key_range", &self.key_range) + .field("lsn_range", &self.lsn_range) + .field("file_size", &self.file_size) + .field("inner", &self.inner) + .finish() + } +} + pub struct DeltaLayerInner { /// If false, the fields below have not been loaded into memory yet. loaded: bool, @@ -200,6 +210,16 @@ pub struct DeltaLayerInner { file: Option>, } +impl std::fmt::Debug for DeltaLayerInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeltaLayerInner") + .field("loaded", &self.loaded) + .field("index_start_blk", &self.index_start_blk) + .field("index_root_blk", &self.index_root_blk) + .finish() + } +} + impl Layer for DeltaLayer { fn get_key_range(&self) -> Range { self.key_range.clone() diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index c11f671ffe..8eacaeb936 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -95,13 +95,13 @@ impl From<&ImageLayer> for Summary { } } -/// /// ImageLayer is the in-memory data structure associated with an on-disk image -/// file. We keep an ImageLayer in memory for each file, in the LayerMap. If a -/// layer is in "loaded" state, we have a copy of the index in memory, in 'inner'. +/// file. +/// +/// We keep an ImageLayer in memory for each file, in the LayerMap. If a layer +/// is in "loaded" state, we have a copy of the index in memory, in 'inner'. /// Otherwise the struct is just a placeholder for a file that exists on disk, /// and it needs to be loaded before using it in queries. -/// pub struct ImageLayer { path_or_conf: PathOrConf, pub tenant_id: TenantId, @@ -115,6 +115,17 @@ pub struct ImageLayer { inner: RwLock, } +impl std::fmt::Debug for ImageLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ImageLayer") + .field("key_range", &self.key_range) + .field("file_size", &self.file_size) + .field("lsn", &self.lsn) + .field("inner", &self.inner) + .finish() + } +} + pub struct ImageLayerInner { /// If false, the 'index' has not been loaded into memory yet. loaded: bool, @@ -127,6 +138,16 @@ pub struct ImageLayerInner { file: Option>, } +impl std::fmt::Debug for ImageLayerInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ImageLayerInner") + .field("loaded", &self.loaded) + .field("index_start_blk", &self.index_start_blk) + .field("index_root_blk", &self.index_root_blk) + .finish() + } +} + impl Layer for ImageLayer { fn get_key_range(&self) -> Range { self.key_range.clone() diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 7249bfd115..c453683fea 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -53,6 +53,15 @@ pub struct InMemoryLayer { inner: RwLock, } +impl std::fmt::Debug for InMemoryLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InMemoryLayer") + .field("start_lsn", &self.start_lsn) + .field("inner", &self.inner) + .finish() + } +} + pub struct InMemoryLayerInner { /// Frozen layers have an exclusive end LSN. /// Writes are only allowed when this is None @@ -71,6 +80,14 @@ pub struct InMemoryLayerInner { file: EphemeralFile, } +impl std::fmt::Debug for InMemoryLayerInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InMemoryLayerInner") + .field("end_lsn", &self.end_lsn) + .finish() + } +} + impl InMemoryLayerInner { fn assert_writeable(&self) { assert!(self.end_lsn.is_none()); diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 8656ad0289..8a12d67c49 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -21,7 +21,14 @@ use super::filename::{DeltaFileName, ImageFileName, LayerFileName}; use super::image_layer::ImageLayer; use super::{DeltaLayer, LayerIter, LayerKeyIter, PersistentLayer}; -#[derive(Debug)] +/// RemoteLayer is a not yet downloaded [`ImageLayer`] or +/// [`crate::storage_layer::DeltaLayer`]. +/// +/// RemoteLayer might be downloaded on-demand during operations which are +/// allowed download remote layers and during which, it gets replaced with a +/// concrete `DeltaLayer` or `ImageLayer`. +/// +/// See: [`crate::context::RequestContext`] for authorization to download pub struct RemoteLayer { tenantid: TenantId, timelineid: TimelineId, @@ -39,6 +46,16 @@ pub struct RemoteLayer { pub(crate) ongoing_download: Arc, } +impl std::fmt::Debug for RemoteLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RemoteLayer") + .field("file_name", &self.file_name) + .field("layer_metadata", &self.layer_metadata) + .field("is_incremental", &self.is_incremental) + .finish() + } +} + impl Layer for RemoteLayer { fn get_key_range(&self) -> Range { self.key_range.clone() From 959f5c6f40479cdd347188195aaf22f33dc9b351 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Mon, 6 Feb 2023 15:51:20 +0100 Subject: [PATCH 099/102] Do not deploy legacy scram proxy (*.cloud.neon.tech) to the old account (#3546) We have migrated to the new proxy, which was setup in https://github.com/neondatabase/neon/pull/3461 --- .../helm-values/production.proxy-scram.yaml | 54 ------------------- .github/workflows/deploy-prod.yml | 35 ------------ 2 files changed, 89 deletions(-) delete mode 100644 .github/helm-values/production.proxy-scram.yaml diff --git a/.github/helm-values/production.proxy-scram.yaml b/.github/helm-values/production.proxy-scram.yaml deleted file mode 100644 index 8cc9d16e2a..0000000000 --- a/.github/helm-values/production.proxy-scram.yaml +++ /dev/null @@ -1,54 +0,0 @@ -settings: - authBackend: "console" - authEndpoint: "http://console-release.local/management/api/v2" - domain: "*.cloud.neon.tech" - sentryEnvironment: "production" - wssPort: 8443 - metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events" - metricCollectionInterval: "10min" - -podLabels: - zenith_service: proxy-scram - zenith_env: production - zenith_region: us-west-2 - zenith_region_slug: oregon - -exposedService: - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: external - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip - service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing - external-dns.alpha.kubernetes.io/hostname: '*.cloud.neon.tech' - httpsPort: 443 - -metrics: - enabled: true - serviceMonitor: - enabled: true - selector: - release: kube-prometheus-stack - -extraManifests: - - apiVersion: operator.victoriametrics.com/v1beta1 - kind: VMServiceScrape - metadata: - name: "{{ include \"neon-proxy.fullname\" . }}" - labels: - helm.sh/chart: neon-proxy-{{ .Chart.Version }} - app.kubernetes.io/name: neon-proxy - app.kubernetes.io/instance: "{{ include \"neon-proxy.fullname\" . }}" - app.kubernetes.io/version: "{{ .Chart.AppVersion }}" - app.kubernetes.io/managed-by: Helm - namespace: "{{ .Release.Namespace }}" - spec: - selector: - matchLabels: - app.kubernetes.io/name: "neon-proxy" - endpoints: - - port: http - path: /metrics - interval: 10s - scrapeTimeout: 10s - namespaceSelector: - matchNames: - - "{{ .Release.Namespace }}" diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 16041d0fe8..b6800a8f7a 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -204,41 +204,6 @@ jobs: - name: Cleanup ansible folder run: rm -rf ~/.ansible - deploy-proxy: - runs-on: [ self-hosted, gen3, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:pinned - if: inputs.deployProxy && inputs.disclamerAcknowledged - defaults: - run: - shell: bash - environment: - name: prod-old - env: - KUBECONFIG: .kubeconfig - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - ref: ${{ inputs.branch }} - - - name: Store kubeconfig file - run: | - echo "${{ secrets.PRODUCTION_KUBECONFIG_DATA }}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - - name: Add neon helm chart - run: helm repo add neondatabase https://neondatabase.github.io/helm-charts - - - name: Re-deploy proxy - run: | - DOCKER_TAG=${{ inputs.dockerTag }} - helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/production.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s - - - name: Cleanup helm folder - run: rm -rf ~/.cache - deploy-storage-broker: name: deploy storage broker on old staging and old prod runs-on: [ self-hosted, gen3, small ] From 877a2d70e33c683333d9d2ab34c34c45382da502 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Mon, 6 Feb 2023 17:53:10 +0200 Subject: [PATCH 100/102] Periodically send cached consumption metrics (#3520) Add new pageserver config setting `cached_metric_collection_interval` with default `1 hour`. This setting controls how often unchanged cached consumption metrics are sent to the HTTP endpoint. This is a workaround for billing service limitations. fixes #3485 --- pageserver/src/bin/pageserver.rs | 1 + pageserver/src/config.rs | 27 +++++++++++++++++++++++++++ pageserver/src/consumption_metrics.rs | 27 ++++++++++++++++++++------- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index fede5c753b..50eefa8c77 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -361,6 +361,7 @@ fn start_pageserver( pageserver::consumption_metrics::collect_metrics( metric_collection_endpoint, conf.metric_collection_interval, + conf.cached_metric_collection_interval, conf.synthetic_size_calculation_interval, conf.id, metrics_ctx, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index a3b051279d..f88895a970 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -58,6 +58,7 @@ pub mod defaults { super::ConfigurableSemaphore::DEFAULT_INITIAL.get(); pub const DEFAULT_METRIC_COLLECTION_INTERVAL: &str = "10 min"; + pub const DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL: &str = "1 hour"; pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option = None; pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min"; @@ -85,6 +86,7 @@ pub mod defaults { #concurrent_tenant_size_logical_size_queries = '{DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES}' #metric_collection_interval = '{DEFAULT_METRIC_COLLECTION_INTERVAL}' +#cached_metric_collection_interval = '{DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL}' #synthetic_size_calculation_interval = '{DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL}' # [tenant_config] @@ -154,6 +156,8 @@ pub struct PageServerConf { // How often to collect metrics and send them to the metrics endpoint. pub metric_collection_interval: Duration, + // How often to send unchanged cached metrics to the metrics endpoint. + pub cached_metric_collection_interval: Duration, pub metric_collection_endpoint: Option, pub synthetic_size_calculation_interval: Duration, @@ -220,6 +224,7 @@ struct PageServerConfigBuilder { concurrent_tenant_size_logical_size_queries: BuilderValue, metric_collection_interval: BuilderValue, + cached_metric_collection_interval: BuilderValue, metric_collection_endpoint: BuilderValue>, synthetic_size_calculation_interval: BuilderValue, @@ -264,6 +269,10 @@ impl Default for PageServerConfigBuilder { DEFAULT_METRIC_COLLECTION_INTERVAL, ) .expect("cannot parse default metric collection interval")), + cached_metric_collection_interval: Set(humantime::parse_duration( + DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL, + ) + .expect("cannot parse default cached_metric_collection_interval")), synthetic_size_calculation_interval: Set(humantime::parse_duration( DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL, ) @@ -353,6 +362,14 @@ impl PageServerConfigBuilder { self.metric_collection_interval = BuilderValue::Set(metric_collection_interval) } + pub fn cached_metric_collection_interval( + &mut self, + cached_metric_collection_interval: Duration, + ) { + self.cached_metric_collection_interval = + BuilderValue::Set(cached_metric_collection_interval) + } + pub fn metric_collection_endpoint(&mut self, metric_collection_endpoint: Option) { self.metric_collection_endpoint = BuilderValue::Set(metric_collection_endpoint) } @@ -427,6 +444,9 @@ impl PageServerConfigBuilder { metric_collection_interval: self .metric_collection_interval .ok_or(anyhow!("missing metric_collection_interval"))?, + cached_metric_collection_interval: self + .cached_metric_collection_interval + .ok_or(anyhow!("missing cached_metric_collection_interval"))?, metric_collection_endpoint: self .metric_collection_endpoint .ok_or(anyhow!("missing metric_collection_endpoint"))?, @@ -612,6 +632,7 @@ impl PageServerConf { ConfigurableSemaphore::new(permits) }), "metric_collection_interval" => builder.metric_collection_interval(parse_toml_duration(key, item)?), + "cached_metric_collection_interval" => builder.cached_metric_collection_interval(parse_toml_duration(key, item)?), "metric_collection_endpoint" => { let endpoint = parse_toml_string(key, item)?.parse().context("failed to parse metric_collection_endpoint")?; builder.metric_collection_endpoint(Some(endpoint)); @@ -741,6 +762,7 @@ impl PageServerConf { log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(), concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(), metric_collection_interval: Duration::from_secs(60), + cached_metric_collection_interval: Duration::from_secs(60 * 60), metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, synthetic_size_calculation_interval: Duration::from_secs(60), test_remote_failures: 0, @@ -881,6 +903,7 @@ initial_superuser_name = 'zzzz' id = 10 metric_collection_interval = '222 s' +cached_metric_collection_interval = '22200 s' metric_collection_endpoint = 'http://localhost:80/metrics' synthetic_size_calculation_interval = '333 s' log_format = 'json' @@ -928,6 +951,9 @@ log_format = 'json' metric_collection_interval: humantime::parse_duration( defaults::DEFAULT_METRIC_COLLECTION_INTERVAL )?, + cached_metric_collection_interval: humantime::parse_duration( + defaults::DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL + )?, metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, synthetic_size_calculation_interval: humantime::parse_duration( defaults::DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL @@ -978,6 +1004,7 @@ log_format = 'json' log_format: LogFormat::Json, concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(), metric_collection_interval: Duration::from_secs(222), + cached_metric_collection_interval: Duration::from_secs(22200), metric_collection_endpoint: Some(Url::parse("http://localhost:80/metrics")?), synthetic_size_calculation_interval: Duration::from_secs(333), test_remote_failures: 0, diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 6204ab92ab..a730d39339 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -46,12 +46,12 @@ pub struct PageserverConsumptionMetricsKey { pub async fn collect_metrics( metric_collection_endpoint: &Url, metric_collection_interval: Duration, + cached_metric_collection_interval: Duration, synthetic_size_calculation_interval: Duration, node_id: NodeId, ctx: RequestContext, ) -> anyhow::Result<()> { let mut ticker = tokio::time::interval(metric_collection_interval); - info!("starting collect_metrics"); // spin up background worker that caclulates tenant sizes @@ -75,6 +75,7 @@ pub async fn collect_metrics( // define client here to reuse it for all requests let client = reqwest::Client::new(); let mut cached_metrics: HashMap = HashMap::new(); + let mut prev_iteration_time: Option = None; loop { tokio::select! { @@ -83,7 +84,15 @@ pub async fn collect_metrics( return Ok(()); }, _ = ticker.tick() => { - collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id, &ctx).await; + + // send cached metrics every cached_metric_collection_interval + let send_cached = prev_iteration_time + .map(|x| x.elapsed() >= cached_metric_collection_interval) + .unwrap_or(false); + + prev_iteration_time = Some(std::time::Instant::now()); + + collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id, &ctx, send_cached).await; } } } @@ -105,6 +114,7 @@ pub async fn collect_metrics_iteration( metric_collection_endpoint: &reqwest::Url, node_id: NodeId, ctx: &RequestContext, + send_cached: bool, ) { let mut current_metrics: Vec<(PageserverConsumptionMetricsKey, u64)> = Vec::new(); trace!( @@ -222,11 +232,14 @@ pub async fn collect_metrics_iteration( )); } - // Filter metrics - current_metrics.retain(|(curr_key, curr_val)| match cached_metrics.get(curr_key) { - Some(val) => val != curr_val, - None => true, - }); + // Filter metrics, unless we want to send all metrics, including cached ones. + // See: https://github.com/neondatabase/neon/issues/3485 + if !send_cached { + current_metrics.retain(|(curr_key, curr_val)| match cached_metrics.get(curr_key) { + Some(val) => val != curr_val, + None => true, + }); + } if current_metrics.is_empty() { trace!("no new metrics to send"); From 58fa4f0eb7f237e5d327566819eb4317b951554f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 31 Jan 2023 11:49:15 +0100 Subject: [PATCH 101/102] maintain access stats for historic layers This patch adds basic access statistics for historic layers and exposes them in the management API's `LayerMapInfo`. We record the accesses in the `{Delta,Image}Layer::load()` function because it's the common path of * page_service (`Timline::get_reconstruct_data()`) * Compaction (`PersistentLayer::iter()` and `PersistentLayer::key_iter()`) The stats survive residence status changes, and record these as well. When scraping the layer map endpoint to record its evolution over time, one must account for stat resets because they are in-memory only and will reset on pageserver restart. Use the launch timestamp header added by (#3527) to identify pageserver restarts. This is PR https://github.com/neondatabase/neon/pull/3496 --- Cargo.lock | 100 ++++++++++- Cargo.toml | 5 + libs/pageserver_api/Cargo.toml | 1 + libs/pageserver_api/src/models.rs | 85 ++++++++- libs/utils/Cargo.toml | 1 + libs/utils/src/history_buffer.rs | 161 ++++++++++++++++++ libs/utils/src/lib.rs | 2 + pageserver/Cargo.toml | 4 + pageserver/src/http/routes.rs | 6 +- pageserver/src/task_mgr.rs | 9 +- pageserver/src/tenant/storage_layer.rs | 159 ++++++++++++++++- .../src/tenant/storage_layer/delta_layer.rs | 40 ++++- .../src/tenant/storage_layer/image_layer.rs | 31 +++- .../src/tenant/storage_layer/remote_layer.rs | 23 ++- pageserver/src/tenant/timeline.rs | 22 ++- 15 files changed, 619 insertions(+), 30 deletions(-) create mode 100644 libs/utils/src/history_buffer.rs diff --git a/Cargo.lock b/Cargo.lock index 612087c97a..6be08d16b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,6 +152,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-polyfill" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d299f547288d6db8d5c3a2916f7b2f66134b15b8c1ac1c4357dd3b8752af7bb2" +dependencies = [ + "critical-section", +] + [[package]] name = "atty" version = "0.2.14" @@ -997,6 +1006,12 @@ dependencies = [ "itertools", ] +[[package]] +name = "critical-section" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -1234,6 +1249,47 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-map" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c25992259941eb7e57b936157961b217a4fc8597829ddef0596d6c3cd86e1a" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4da76b3b6116d758c7ba93f7ec6a35d2e2cf24feda76c6e38a375f4d5c59f2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "enumset" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19be8061a06ab6f3a6cf21106c873578bf01bd42ad15e0311a9c76161cb1c753" +dependencies = [ + "enumset_derive", +] + +[[package]] +name = "enumset_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e7b551eba279bf0fa88b83a46330168c1560a52a94f5126f892f0b364ab3e0" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -1521,6 +1577,15 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1548,6 +1613,18 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "git+https://github.com/japaric/heapless.git?rev=644653bf3b831c6bb4963be2de24804acf5e5001#644653bf3b831c6bb4963be2de24804acf5e5001" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "spin 0.9.4", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -2344,6 +2421,8 @@ dependencies = [ "crc32c", "criterion", "crossbeam-utils", + "enum-map", + "enumset", "fail", "futures", "git-version", @@ -2376,6 +2455,8 @@ dependencies = [ "serde_with", "signal-hook", "storage_broker", + "strum", + "strum_macros", "svg_fmt", "tempfile", "tenant_size_model", @@ -2400,6 +2481,7 @@ dependencies = [ "byteorder", "bytes", "const_format", + "enum-map", "postgres_ffi", "serde", "serde_with", @@ -3021,7 +3103,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -3568,6 +3650,21 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -4347,6 +4444,7 @@ dependencies = [ "bytes", "criterion", "git-version", + "heapless", "hex", "hex-literal", "hyper", diff --git a/Cargo.toml b/Cargo.toml index c335b64175..9033671f55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,8 @@ comfy-table = "6.1" const_format = "0.2" crc32c = "0.6" crossbeam-utils = "0.8.5" +enum-map = "2.4.2" +enumset = "1.0.12" fail = "0.5.0" fs2 = "0.4.3" futures = "0.3" @@ -120,6 +122,9 @@ postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", re tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" } tokio-tar = { git = "https://github.com/neondatabase/tokio-tar.git", rev="404df61437de0feef49ba2ccdbdd94eb8ad6e142" } +## Other git libraries +heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending + ## Local libraries consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" } metrics = { version = "0.1", path = "./libs/metrics/" } diff --git a/libs/pageserver_api/Cargo.toml b/libs/pageserver_api/Cargo.toml index a3883f07a7..dafb246632 100644 --- a/libs/pageserver_api/Cargo.toml +++ b/libs/pageserver_api/Cargo.toml @@ -13,5 +13,6 @@ bytes.workspace = true byteorder.workspace = true utils.workspace = true postgres_ffi.workspace = true +enum-map.workspace = true workspace_hack.workspace = true diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 8827235d90..9cdcf3a173 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -1,9 +1,14 @@ -use std::num::{NonZeroU64, NonZeroUsize}; +use std::{ + collections::HashMap, + num::{NonZeroU64, NonZeroUsize}, + time::SystemTime, +}; use byteorder::{BigEndian, ReadBytesExt}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use utils::{ + history_buffer::HistoryBufferWithDropCounter, id::{NodeId, TenantId, TimelineId}, lsn::Lsn, }; @@ -232,6 +237,82 @@ pub struct LayerMapInfo { pub historic_layers: Vec, } +#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, enum_map::Enum)] +#[repr(usize)] +pub enum LayerAccessKind { + GetValueReconstructData, + Iter, + KeyIter, + Dump, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LayerAccessStatFullDetails { + pub when_millis_since_epoch: u64, + pub task_kind: &'static str, + pub access_kind: LayerAccessKind, +} + +/// An event that impacts the layer's residence status. +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LayerResidenceEvent { + /// The time when the event occurred. + /// NB: this timestamp is captured while the residence status changes. + /// So, it might be behind/ahead of the actual residence change by a short amount of time. + /// + #[serde(rename = "timestamp_millis_since_epoch")] + #[serde_as(as = "serde_with::TimestampMilliSeconds")] + timestamp: SystemTime, + /// The new residence status of the layer. + status: LayerResidenceStatus, + /// The reason why we had to record this event. + reason: LayerResidenceEventReason, +} + +/// The reason for recording a given [`ResidenceEvent`]. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum LayerResidenceEventReason { + /// The layer map is being populated, e.g. during timeline load or attach. + /// This includes [`RemoteLayer`] objects created in [`reconcile_with_remote`]. + /// We need to record such events because there is no persistent storage for the events. + LayerLoad, + /// We just created the layer (e.g., freeze_and_flush or compaction). + /// Such layers are always [`LayerResidenceStatus::Resident`]. + LayerCreate, + /// We on-demand downloaded or evicted the given layer. + ResidenceChange, +} + +/// The residence status of the layer, after the given [`LayerResidenceEvent`]. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum LayerResidenceStatus { + /// Residence status for a layer file that exists locally. + /// It may also exist on the remote, we don't care here. + Resident, + /// Residence status for a layer file that only exists on the remote. + Evicted, +} + +impl LayerResidenceEvent { + pub fn new(status: LayerResidenceStatus, reason: LayerResidenceEventReason) -> Self { + Self { + status, + reason, + timestamp: SystemTime::now(), + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct LayerAccessStats { + pub access_count_by_access_kind: HashMap, + pub task_kind_access_flag: Vec<&'static str>, + pub first: Option, + pub accesses_history: HistoryBufferWithDropCounter, + pub residence_events_history: HistoryBufferWithDropCounter, +} + #[serde_as] #[derive(Debug, Clone, Serialize)] #[serde(tag = "kind")] @@ -261,6 +342,7 @@ pub enum HistoricLayerInfo { #[serde_as(as = "DisplayFromStr")] lsn_end: Lsn, remote: bool, + access_stats: LayerAccessStats, }, Image { layer_file_name: String, @@ -269,6 +351,7 @@ pub enum HistoricLayerInfo { #[serde_as(as = "DisplayFromStr")] lsn_start: Lsn, remote: bool, + access_stats: LayerAccessStats, }, } diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 1ce3155083..92e805ac58 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -11,6 +11,7 @@ async-trait.workspace = true anyhow.workspace = true bincode.workspace = true bytes.workspace = true +heapless.workspace = true hyper = { workspace = true, features = ["full"] } routerify.workspace = true serde.workspace = true diff --git a/libs/utils/src/history_buffer.rs b/libs/utils/src/history_buffer.rs new file mode 100644 index 0000000000..1f07f5560f --- /dev/null +++ b/libs/utils/src/history_buffer.rs @@ -0,0 +1,161 @@ +//! A heapless buffer for events of sorts. + +use std::ops; + +use heapless::HistoryBuffer; + +#[derive(Debug, Clone)] +pub struct HistoryBufferWithDropCounter { + buffer: HistoryBuffer, + drop_count: u64, +} + +impl HistoryBufferWithDropCounter { + pub fn write(&mut self, data: T) { + let len_before = self.buffer.len(); + self.buffer.write(data); + let len_after = self.buffer.len(); + self.drop_count += u64::from(len_before == len_after); + } + pub fn drop_count(&self) -> u64 { + self.drop_count + } + pub fn map U>(&self, f: F) -> HistoryBufferWithDropCounter { + let mut buffer = HistoryBuffer::new(); + buffer.extend(self.buffer.oldest_ordered().map(f)); + HistoryBufferWithDropCounter:: { + buffer, + drop_count: self.drop_count, + } + } +} + +impl Default for HistoryBufferWithDropCounter { + fn default() -> Self { + Self { + buffer: HistoryBuffer::default(), + drop_count: 0, + } + } +} + +impl ops::Deref for HistoryBufferWithDropCounter { + type Target = HistoryBuffer; + + fn deref(&self) -> &Self::Target { + &self.buffer + } +} + +#[derive(serde::Serialize)] +struct SerdeRepr { + buffer: Vec, + drop_count: u64, +} + +impl<'a, T, const L: usize> From<&'a HistoryBufferWithDropCounter> for SerdeRepr +where + T: Clone + serde::Serialize, +{ + fn from(value: &'a HistoryBufferWithDropCounter) -> Self { + let HistoryBufferWithDropCounter { buffer, drop_count } = value; + SerdeRepr { + buffer: buffer.iter().cloned().collect(), + drop_count: *drop_count, + } + } +} + +impl serde::Serialize for HistoryBufferWithDropCounter +where + T: Clone + serde::Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + SerdeRepr::from(self).serialize(serializer) + } +} + +#[cfg(test)] +mod test { + use super::HistoryBufferWithDropCounter; + + #[test] + fn test_basics() { + let mut b = HistoryBufferWithDropCounter::<_, 2>::default(); + b.write(1); + b.write(2); + b.write(3); + assert!(b.iter().any(|e| *e == 2)); + assert!(b.iter().any(|e| *e == 3)); + assert!(!b.iter().any(|e| *e == 1)); + } + + #[test] + fn test_drop_count_works() { + let mut b = HistoryBufferWithDropCounter::<_, 2>::default(); + b.write(1); + assert_eq!(b.drop_count(), 0); + b.write(2); + assert_eq!(b.drop_count(), 0); + b.write(3); + assert_eq!(b.drop_count(), 1); + b.write(4); + assert_eq!(b.drop_count(), 2); + } + + #[test] + fn test_clone_works() { + let mut b = HistoryBufferWithDropCounter::<_, 2>::default(); + b.write(1); + b.write(2); + b.write(3); + assert_eq!(b.drop_count(), 1); + let mut c = b.clone(); + assert_eq!(c.drop_count(), 1); + assert!(c.iter().any(|e| *e == 2)); + assert!(c.iter().any(|e| *e == 3)); + assert!(!c.iter().any(|e| *e == 1)); + + c.write(4); + assert!(c.iter().any(|e| *e == 4)); + assert!(!b.iter().any(|e| *e == 4)); + } + + #[test] + fn test_map() { + let mut b = HistoryBufferWithDropCounter::<_, 2>::default(); + + b.write(1); + assert_eq!(b.drop_count(), 0); + { + let c = b.map(|i| i + 10); + assert_eq!(c.oldest_ordered().cloned().collect::>(), vec![11]); + assert_eq!(c.drop_count(), 0); + } + + b.write(2); + assert_eq!(b.drop_count(), 0); + { + let c = b.map(|i| i + 10); + assert_eq!( + c.oldest_ordered().cloned().collect::>(), + vec![11, 12] + ); + assert_eq!(c.drop_count(), 0); + } + + b.write(3); + assert_eq!(b.drop_count(), 1); + { + let c = b.map(|i| i + 10); + assert_eq!( + c.oldest_ordered().cloned().collect::>(), + vec![12, 13] + ); + assert_eq!(c.drop_count(), 1); + } + } +} diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 6d35fd9f7b..9ddd702c72 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -52,6 +52,8 @@ pub mod signals; pub mod fs_ext; +pub mod history_buffer; + /// use with fail::cfg("$name", "return(2000)") #[macro_export] macro_rules! failpoint_sleep_millis_async { diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 66c25e8576..f3ad2c5de6 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -67,6 +67,10 @@ utils.workspace = true workspace_hack.workspace = true reqwest.workspace = true rpds.workspace = true +enum-map.workspace = true +enumset.workspace = true +strum.workspace = true +strum_macros.workspace = true [dev-dependencies] criterion.workspace = true diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 7279ce0f63..229cf96ee3 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -20,6 +20,7 @@ use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; use crate::tenant::config::TenantConfOpt; use crate::tenant::mgr::TenantMapInsertError; +use crate::tenant::storage_layer::LayerAccessStatsReset; use crate::tenant::{PageReconstructError, Timeline}; use crate::{config::PageServerConf, tenant::mgr}; use utils::{ @@ -534,10 +535,13 @@ async fn tenant_size_handler(request: Request) -> Result, A async fn layer_map_info_handler(request: Request) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + let reset: LayerAccessStatsReset = + parse_query_param(&request, "reset")?.unwrap_or(LayerAccessStatsReset::NoReset); + check_permission(&request, Some(tenant_id))?; let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; - let layer_map_info = timeline.layer_map_info(); + let layer_map_info = timeline.layer_map_info(reset); json_response(StatusCode::OK, layer_map_info) } diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 8e0149046d..c4f213e755 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -169,7 +169,14 @@ task_local! { /// Note that we don't try to limit how many task of a certain kind can be running /// at the same time. /// -#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[derive( + Debug, + // NB: enumset::EnumSetType derives PartialEq, Eq, Clone, Copy + enumset::EnumSetType, + serde::Serialize, + serde::Deserialize, + strum_macros::IntoStaticStr, +)] pub enum TaskKind { // Pageserver startup, i.e., `main` Startup, diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 1867d489c8..e85359af16 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -9,13 +9,21 @@ mod remote_layer; use crate::config::PageServerConf; use crate::context::RequestContext; use crate::repository::{Key, Value}; +use crate::task_mgr::TaskKind; use crate::walrecord::NeonWalRecord; use anyhow::Result; use bytes::Bytes; -use pageserver_api::models::HistoricLayerInfo; +use enum_map::EnumMap; +use enumset::EnumSet; +use pageserver_api::models::LayerAccessKind; +use pageserver_api::models::{ + HistoricLayerInfo, LayerResidenceEvent, LayerResidenceEventReason, LayerResidenceStatus, +}; use std::ops::Range; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; +use utils::history_buffer::HistoryBufferWithDropCounter; use utils::{ id::{TenantId, TimelineId}, @@ -83,6 +91,149 @@ pub enum ValueReconstructResult { Missing, } +#[derive(Debug)] +pub struct LayerAccessStats(Mutex); + +#[derive(Debug, Default, Clone)] +struct LayerAccessStatsInner { + first_access: Option, + count_by_access_kind: EnumMap, + task_kind_flag: EnumSet, + last_accesses: HistoryBufferWithDropCounter, + last_residence_changes: HistoryBufferWithDropCounter, +} + +#[derive(Debug, Clone)] +struct LayerAccessStatFullDetails { + when: SystemTime, + task_kind: TaskKind, + access_kind: LayerAccessKind, +} + +#[derive(Clone, Copy, strum_macros::EnumString)] +pub enum LayerAccessStatsReset { + NoReset, + JustTaskKindFlags, + AllStats, +} + +fn system_time_to_millis_since_epoch(ts: &SystemTime) -> u64 { + ts.duration_since(UNIX_EPOCH) + .expect("better to die in this unlikely case than report false stats") + .as_millis() + .try_into() + .expect("64 bits is enough for few more years") +} + +impl LayerAccessStatFullDetails { + fn to_api_model(&self) -> pageserver_api::models::LayerAccessStatFullDetails { + let Self { + when, + task_kind, + access_kind, + } = self; + pageserver_api::models::LayerAccessStatFullDetails { + when_millis_since_epoch: system_time_to_millis_since_epoch(when), + task_kind: task_kind.into(), // into static str, powered by strum_macros + access_kind: *access_kind, + } + } +} + +impl LayerAccessStats { + pub(crate) fn for_loading_layer(status: LayerResidenceStatus) -> Self { + let new = LayerAccessStats(Mutex::new(LayerAccessStatsInner::default())); + new.record_residence_event(status, LayerResidenceEventReason::LayerLoad); + new + } + + pub(crate) fn for_new_layer_file() -> Self { + let new = LayerAccessStats(Mutex::new(LayerAccessStatsInner::default())); + new.record_residence_event( + LayerResidenceStatus::Resident, + LayerResidenceEventReason::LayerCreate, + ); + new + } + + /// Creates a clone of `self` and records `new_status` in the clone. + /// The `new_status` is not recorded in `self` + pub(crate) fn clone_for_residence_change( + &self, + new_status: LayerResidenceStatus, + ) -> LayerAccessStats { + let clone = { + let inner = self.0.lock().unwrap(); + inner.clone() + }; + let new = LayerAccessStats(Mutex::new(clone)); + new.record_residence_event(new_status, LayerResidenceEventReason::ResidenceChange); + new + } + + fn record_residence_event( + &self, + status: LayerResidenceStatus, + reason: LayerResidenceEventReason, + ) { + let mut inner = self.0.lock().unwrap(); + inner + .last_residence_changes + .write(LayerResidenceEvent::new(status, reason)); + } + + fn record_access(&self, access_kind: LayerAccessKind, task_kind: TaskKind) { + let mut inner = self.0.lock().unwrap(); + let this_access = LayerAccessStatFullDetails { + when: SystemTime::now(), + task_kind, + access_kind, + }; + inner + .first_access + .get_or_insert_with(|| this_access.clone()); + inner.count_by_access_kind[access_kind] += 1; + inner.task_kind_flag |= task_kind; + inner.last_accesses.write(this_access); + } + fn to_api_model( + &self, + reset: LayerAccessStatsReset, + ) -> pageserver_api::models::LayerAccessStats { + let mut inner = self.0.lock().unwrap(); + let LayerAccessStatsInner { + first_access, + count_by_access_kind, + task_kind_flag, + last_accesses, + last_residence_changes, + } = &*inner; + let ret = pageserver_api::models::LayerAccessStats { + access_count_by_access_kind: count_by_access_kind + .iter() + .map(|(kind, count)| (kind, *count)) + .collect(), + task_kind_access_flag: task_kind_flag + .iter() + .map(|task_kind| task_kind.into()) // into static str, powered by strum_macros + .collect(), + first: first_access.as_ref().map(|a| a.to_api_model()), + accesses_history: last_accesses.map(|m| m.to_api_model()), + residence_events_history: last_residence_changes.clone(), + }; + match reset { + LayerAccessStatsReset::NoReset => (), + LayerAccessStatsReset::JustTaskKindFlags => { + inner.task_kind_flag.clear(); + } + LayerAccessStatsReset::AllStats => { + *inner = LayerAccessStatsInner::default(); + } + } + ret + } +} + /// Supertrait of the [`Layer`] trait that captures the bare minimum interface /// required by [`LayerMap`]. /// @@ -193,7 +344,9 @@ pub trait PersistentLayer: Layer { /// current_physical_size is computed as the som of this value. fn file_size(&self) -> Option; - fn info(&self) -> HistoricLayerInfo; + fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo; + + fn access_stats(&self) -> &LayerAccessStats; } pub fn downcast_remote_layer( diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 142a8ce913..9b322faa65 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -37,7 +37,7 @@ use crate::virtual_file::VirtualFile; use crate::{walrecord, TEMP_FILE_SUFFIX}; use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION}; use anyhow::{bail, ensure, Context, Result}; -use pageserver_api::models::HistoricLayerInfo; +use pageserver_api::models::{HistoricLayerInfo, LayerAccessKind}; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; use std::fs::{self, File}; @@ -55,7 +55,10 @@ use utils::{ lsn::Lsn, }; -use super::{DeltaFileName, Layer, LayerFileName, LayerIter, LayerKeyIter, PathOrConf}; +use super::{ + DeltaFileName, Layer, LayerAccessStats, LayerAccessStatsReset, LayerFileName, LayerIter, + LayerKeyIter, LayerResidenceStatus, PathOrConf, +}; /// /// Header stored in the beginning of the file @@ -184,6 +187,8 @@ pub struct DeltaLayer { pub file_size: u64, + access_stats: LayerAccessStats, + inner: RwLock, } @@ -251,7 +256,7 @@ impl Layer for DeltaLayer { return Ok(()); } - let inner = self.load(ctx)?; + let inner = self.load(LayerAccessKind::Dump, ctx)?; println!( "index_start_blk: {}, root {}", @@ -324,7 +329,7 @@ impl Layer for DeltaLayer { { // Open the file and lock the metadata in memory - let inner = self.load(ctx)?; + let inner = self.load(LayerAccessKind::GetValueReconstructData, ctx)?; // Scan the page versions backwards, starting from `lsn`. let file = inner.file.as_ref().unwrap(); @@ -415,7 +420,9 @@ impl PersistentLayer for DeltaLayer { } fn iter(&self, ctx: &RequestContext) -> Result> { - let inner = self.load(ctx).context("load delta layer")?; + let inner = self + .load(LayerAccessKind::KeyIter, ctx) + .context("load delta layer")?; Ok(match DeltaValueIter::new(inner) { Ok(iter) => Box::new(iter), Err(err) => Box::new(std::iter::once(Err(err))), @@ -423,7 +430,7 @@ impl PersistentLayer for DeltaLayer { } fn key_iter(&self, ctx: &RequestContext) -> Result> { - let inner = self.load(ctx)?; + let inner = self.load(LayerAccessKind::KeyIter, ctx)?; Ok(Box::new( DeltaKeyIter::new(inner).context("Layer index is corrupted")?, )) @@ -439,18 +446,25 @@ impl PersistentLayer for DeltaLayer { Some(self.file_size) } - fn info(&self) -> HistoricLayerInfo { + fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.filename().file_name(); let lsn_range = self.get_lsn_range(); + let access_stats = self.access_stats.to_api_model(reset); + HistoricLayerInfo::Delta { layer_file_name, layer_file_size: Some(self.file_size), lsn_start: lsn_range.start, lsn_end: lsn_range.end, remote: false, + access_stats, } } + + fn access_stats(&self) -> &LayerAccessStats { + &self.access_stats + } } impl DeltaLayer { @@ -495,7 +509,13 @@ impl DeltaLayer { /// Open the underlying file and read the metadata into memory, if it's /// not loaded already. /// - fn load(&self, _ctx: &RequestContext) -> Result> { + fn load( + &self, + access_kind: LayerAccessKind, + ctx: &RequestContext, + ) -> Result> { + self.access_stats + .record_access(access_kind, ctx.task_kind()); loop { // Quick exit if already loaded let inner = self.inner.read().unwrap(); @@ -576,6 +596,7 @@ impl DeltaLayer { tenant_id: TenantId, filename: &DeltaFileName, file_size: u64, + access_stats: LayerAccessStats, ) -> DeltaLayer { DeltaLayer { path_or_conf: PathOrConf::Conf(conf), @@ -584,6 +605,7 @@ impl DeltaLayer { key_range: filename.key_range.clone(), lsn_range: filename.lsn_range.clone(), file_size, + access_stats, inner: RwLock::new(DeltaLayerInner { loaded: false, file: None, @@ -613,6 +635,7 @@ impl DeltaLayer { key_range: summary.key_range, lsn_range: summary.lsn_range, file_size: metadata.len(), + access_stats: LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident), inner: RwLock::new(DeltaLayerInner { loaded: false, file: None, @@ -783,6 +806,7 @@ impl DeltaLayerWriterInner { key_range: self.key_start..key_end, lsn_range: self.lsn_range.clone(), file_size: metadata.len(), + access_stats: LayerAccessStats::for_new_layer_file(), inner: RwLock::new(DeltaLayerInner { loaded: false, file: None, diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 8eacaeb936..86c1aee619 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -27,14 +27,14 @@ use crate::tenant::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter}; use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader}; use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection}; use crate::tenant::storage_layer::{ - PersistentLayer, ValueReconstructResult, ValueReconstructState, + LayerAccessStats, PersistentLayer, ValueReconstructResult, ValueReconstructState, }; use crate::virtual_file::VirtualFile; use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION, TEMP_FILE_SUFFIX}; use anyhow::{bail, ensure, Context, Result}; use bytes::Bytes; use hex; -use pageserver_api::models::HistoricLayerInfo; +use pageserver_api::models::{HistoricLayerInfo, LayerAccessKind}; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; use std::fs::{self, File}; @@ -53,7 +53,7 @@ use utils::{ }; use super::filename::{ImageFileName, LayerFileName}; -use super::{Layer, LayerIter, PathOrConf}; +use super::{Layer, LayerAccessStatsReset, LayerIter, LayerResidenceStatus, PathOrConf}; /// /// Header stored in the beginning of the file @@ -112,6 +112,8 @@ pub struct ImageLayer { // This entry contains an image of all pages as of this LSN pub lsn: Lsn, + access_stats: LayerAccessStats, + inner: RwLock, } @@ -176,7 +178,7 @@ impl Layer for ImageLayer { return Ok(()); } - let inner = self.load(ctx)?; + let inner = self.load(LayerAccessKind::Dump, ctx)?; let file = inner.file.as_ref().unwrap(); let tree_reader = DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file); @@ -203,7 +205,7 @@ impl Layer for ImageLayer { assert!(lsn_range.start >= self.lsn); assert!(lsn_range.end >= self.lsn); - let inner = self.load(ctx)?; + let inner = self.load(LayerAccessKind::GetValueReconstructData, ctx)?; let file = inner.file.as_ref().unwrap(); let tree_reader = DiskBtreeReader::new(inner.index_start_blk, inner.index_root_blk, file); @@ -258,7 +260,7 @@ impl PersistentLayer for ImageLayer { Some(self.file_size) } - fn info(&self) -> HistoricLayerInfo { + fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.filename().file_name(); let lsn_range = self.get_lsn_range(); @@ -267,8 +269,13 @@ impl PersistentLayer for ImageLayer { layer_file_size: Some(self.file_size), lsn_start: lsn_range.start, remote: false, + access_stats: self.access_stats.to_api_model(reset), } } + + fn access_stats(&self) -> &LayerAccessStats { + &self.access_stats + } } impl ImageLayer { @@ -306,7 +313,13 @@ impl ImageLayer { /// Open the underlying file and read the metadata into memory, if it's /// not loaded already. /// - fn load(&self, _ctx: &RequestContext) -> Result> { + fn load( + &self, + access_kind: LayerAccessKind, + ctx: &RequestContext, + ) -> Result> { + self.access_stats + .record_access(access_kind, ctx.task_kind()); loop { // Quick exit if already loaded let inner = self.inner.read().unwrap(); @@ -386,6 +399,7 @@ impl ImageLayer { tenant_id: TenantId, filename: &ImageFileName, file_size: u64, + access_stats: LayerAccessStats, ) -> ImageLayer { ImageLayer { path_or_conf: PathOrConf::Conf(conf), @@ -394,6 +408,7 @@ impl ImageLayer { key_range: filename.key_range.clone(), lsn: filename.lsn, file_size, + access_stats, inner: RwLock::new(ImageLayerInner { loaded: false, file: None, @@ -421,6 +436,7 @@ impl ImageLayer { key_range: summary.key_range, lsn: summary.lsn, file_size: metadata.len(), + access_stats: LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident), inner: RwLock::new(ImageLayerInner { file: None, loaded: false, @@ -580,6 +596,7 @@ impl ImageLayerWriterInner { key_range: self.key_range.clone(), lsn: self.lsn, file_size: metadata.len(), + access_stats: LayerAccessStats::for_new_layer_file(), inner: RwLock::new(ImageLayerInner { loaded: false, file: None, diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 8a12d67c49..7391875d0c 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -19,7 +19,10 @@ use utils::{ use super::filename::{DeltaFileName, ImageFileName, LayerFileName}; use super::image_layer::ImageLayer; -use super::{DeltaLayer, LayerIter, LayerKeyIter, PersistentLayer}; +use super::{ + DeltaLayer, LayerAccessStats, LayerAccessStatsReset, LayerIter, LayerKeyIter, + LayerResidenceStatus, PersistentLayer, +}; /// RemoteLayer is a not yet downloaded [`ImageLayer`] or /// [`crate::storage_layer::DeltaLayer`]. @@ -43,6 +46,8 @@ pub struct RemoteLayer { is_incremental: bool, + access_stats: LayerAccessStats, + pub(crate) ongoing_download: Arc, } @@ -155,7 +160,7 @@ impl PersistentLayer for RemoteLayer { self.layer_metadata.file_size() } - fn info(&self) -> HistoricLayerInfo { + fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.filename().file_name(); let lsn_range = self.get_lsn_range(); @@ -166,6 +171,7 @@ impl PersistentLayer for RemoteLayer { lsn_start: lsn_range.start, lsn_end: lsn_range.end, remote: true, + access_stats: self.access_stats.to_api_model(reset), } } else { HistoricLayerInfo::Image { @@ -173,9 +179,14 @@ impl PersistentLayer for RemoteLayer { layer_file_size: self.layer_metadata.file_size(), lsn_start: lsn_range.start, remote: true, + access_stats: self.access_stats.to_api_model(reset), } } } + + fn access_stats(&self) -> &LayerAccessStats { + &self.access_stats + } } impl RemoteLayer { @@ -184,6 +195,7 @@ impl RemoteLayer { timelineid: TimelineId, fname: &ImageFileName, layer_metadata: &LayerFileMetadata, + access_stats: LayerAccessStats, ) -> RemoteLayer { RemoteLayer { tenantid, @@ -195,6 +207,7 @@ impl RemoteLayer { file_name: fname.to_owned().into(), layer_metadata: layer_metadata.clone(), ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), + access_stats, } } @@ -203,6 +216,7 @@ impl RemoteLayer { timelineid: TimelineId, fname: &DeltaFileName, layer_metadata: &LayerFileMetadata, + access_stats: LayerAccessStats, ) -> RemoteLayer { RemoteLayer { tenantid, @@ -214,6 +228,7 @@ impl RemoteLayer { file_name: fname.to_owned().into(), layer_metadata: layer_metadata.clone(), ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), + access_stats, } } @@ -234,6 +249,8 @@ impl RemoteLayer { self.tenantid, &fname, file_size, + self.access_stats + .clone_for_residence_change(LayerResidenceStatus::Resident), )) } else { let fname = ImageFileName { @@ -246,6 +263,8 @@ impl RemoteLayer { self.tenantid, &fname, file_size, + self.access_stats + .clone_for_residence_change(LayerResidenceStatus::Resident), )) } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index fc6f7ad96c..6ae23c584b 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -10,7 +10,7 @@ use itertools::Itertools; use once_cell::sync::OnceCell; use pageserver_api::models::{ DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, - DownloadRemoteLayersTaskState, LayerMapInfo, TimelineState, + DownloadRemoteLayersTaskState, LayerMapInfo, LayerResidenceStatus, TimelineState, }; use tokio::sync::{oneshot, watch, Semaphore, TryAcquireError}; use tokio_util::sync::CancellationToken; @@ -30,8 +30,8 @@ use crate::broker_client::is_broker_client_initialized; use crate::context::{DownloadBehavior, RequestContext}; use crate::tenant::remote_timeline_client::{self, index::LayerFileMetadata}; use crate::tenant::storage_layer::{ - DeltaFileName, DeltaLayerWriter, ImageFileName, ImageLayerWriter, InMemoryLayer, LayerFileName, - RemoteLayer, + DeltaFileName, DeltaLayerWriter, ImageFileName, ImageLayerWriter, InMemoryLayer, + LayerAccessStats, LayerFileName, RemoteLayer, }; use crate::tenant::{ ephemeral_file::is_ephemeral_file, @@ -72,7 +72,7 @@ use walreceiver::spawn_connection_manager_task; use super::layer_map::BatchedUpdates; use super::remote_timeline_client::index::IndexPart; use super::remote_timeline_client::RemoteTimelineClient; -use super::storage_layer::{DeltaLayer, ImageLayer, Layer}; +use super::storage_layer::{DeltaLayer, ImageLayer, Layer, LayerAccessStatsReset}; #[derive(Debug, PartialEq, Eq, Clone, Copy)] enum FlushLoopState { @@ -835,7 +835,7 @@ impl Timeline { self.state.subscribe() } - pub fn layer_map_info(&self) -> LayerMapInfo { + pub fn layer_map_info(&self, reset: LayerAccessStatsReset) -> LayerMapInfo { let layer_map = self.layers.read().unwrap(); let mut in_memory_layers = Vec::with_capacity(layer_map.frozen_layers.len() + 1); if let Some(open_layer) = &layer_map.open_layer { @@ -847,7 +847,7 @@ impl Timeline { let mut historic_layers = Vec::new(); for historic_layer in layer_map.iter_historic_layers() { - historic_layers.push(historic_layer.info()); + historic_layers.push(historic_layer.info(reset)); } LayerMapInfo { @@ -891,12 +891,18 @@ impl Timeline { self.timeline_id, &image_name, &layer_metadata, + local_layer + .access_stats() + .clone_for_residence_change(LayerResidenceStatus::Evicted), ), LayerFileName::Delta(delta_name) => RemoteLayer::new_delta( self.tenant_id, self.timeline_id, &delta_name, &layer_metadata, + local_layer + .access_stats() + .clone_for_residence_change(LayerResidenceStatus::Evicted), ), }); @@ -1172,6 +1178,7 @@ impl Timeline { self.tenant_id, &imgfilename, file_size, + LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident), ); trace!("found layer {}", layer.path().display()); @@ -1203,6 +1210,7 @@ impl Timeline { self.tenant_id, &deltafilename, file_size, + LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident), ); trace!("found layer {}", layer.path().display()); @@ -1340,6 +1348,7 @@ impl Timeline { self.timeline_id, imgfilename, &remote_layer_metadata, + LayerAccessStats::for_loading_layer(LayerResidenceStatus::Evicted), ); let remote_layer = Arc::new(remote_layer); @@ -1364,6 +1373,7 @@ impl Timeline { self.timeline_id, deltafilename, &remote_layer_metadata, + LayerAccessStats::for_loading_layer(LayerResidenceStatus::Evicted), ); let remote_layer = Arc::new(remote_layer); updates.insert_historic(remote_layer); From fcb905f519464e657f367218e13fadbc9082ea5a Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 7 Feb 2023 11:08:55 +0200 Subject: [PATCH 102/102] Use LayerMap::replace in eviction (#3544) Follow-up to #3536, to actually use the new `Debug` in replacing the layers, and use replacement with manual eviction endpoint. Turns out the two paths share a lot of handling of `Replacement` but didn't unify the two (need 3). There are also upcoming refactorings from other PRs to this. --- pageserver/src/tenant/timeline.rs | 52 ++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 6ae23c584b..ff7b8c932c 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -867,15 +867,26 @@ impl Timeline { Ok(Some(true)) } + /// Evicts one layer as in replaces a downloaded layer with a remote layer + /// + /// Returns: + /// - `Ok(Some(true))` when the layer was replaced + /// - `Ok(Some(false))` when the layer was found, but no changes were made + /// - evictee was not yet downloaded + /// - layermap replacement failed + /// - `Ok(None)` when the layer is not found pub async fn evict_layer(&self, layer_file_name: &str) -> anyhow::Result> { + use super::layer_map::Replacement; + let Some(local_layer) = self.find_layer(layer_file_name) else { return Ok(None) }; if local_layer.is_remote_layer() { return Ok(Some(false)); } - let Some(remote_client) = &self.remote_client else { return Ok(Some(false)) }; // ensure the current layer is uploaded for sure - remote_client + self.remote_client + .as_ref() + .ok_or_else(|| anyhow::anyhow!("remote storage not configured; cannot evict"))? .wait_completion() .await .context("wait for layer upload ops to complete")?; @@ -909,13 +920,43 @@ impl Timeline { let gc_lock = self.layer_removal_cs.lock().await; let mut layers = self.layers.write().unwrap(); let mut updates = layers.batch_update(); - self.delete_historic_layer(&gc_lock, local_layer, &mut updates)?; - updates.insert_historic(new_remote_layer); + + let replaced = match updates.replace_historic(&local_layer, new_remote_layer)? { + Replacement::Replaced { .. } => { + let layer_size = local_layer.file_size(); + + if let Err(e) = local_layer.delete() { + error!("failed to remove layer file on evict after replacement: {e:#?}"); + } + + if let Some(layer_size) = layer_size { + self.metrics.resident_physical_size_gauge.sub(layer_size); + } + + true + } + Replacement::NotFound => { + debug!(evicted=?local_layer, "layer was no longer in layer map"); + false + } + Replacement::RemovalBuffered => { + unreachable!("not doing anything else in this batch") + } + Replacement::Unexpected(other) => { + error!( + local_layer.ptr=?Arc::as_ptr(&local_layer), + other.ptr=?Arc::as_ptr(&other), + ?other, + "failed to replace"); + false + } + }; + updates.flush(); drop(layers); drop(gc_lock); - Ok(Some(true)) + Ok(Some(replaced)) } } @@ -3422,6 +3463,7 @@ impl Timeline { error!( expected.ptr = ?Arc::as_ptr(&l), other.ptr = ?Arc::as_ptr(&other), + ?other, "replacing downloaded layer into layermap failed because another layer was found instead of expected" ); }