From 5ceccdc7def3d4bdd33b32d5b10a8f4e148ad9af Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 3 Nov 2023 18:40:27 +0200 Subject: [PATCH 01/63] Logical replication startup fixes (#5750) ## Problem See https://neondb.slack.com/archives/C04DGM6SMTM/p1698226491736459 ## Summary of changes Update WAL affected buffers when restoring WAL from safekeeper ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik Co-authored-by: Arseny Sher --- pgxn/neon/walproposer_pg.c | 19 +++- .../regress/test_logical_replication.py | 89 +++++++++++++++++++ vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/revisions.json | 6 +- 6 files changed, 111 insertions(+), 9 deletions(-) diff --git a/pgxn/neon/walproposer_pg.c b/pgxn/neon/walproposer_pg.c index 865f91165b..f83a08d407 100644 --- a/pgxn/neon/walproposer_pg.c +++ b/pgxn/neon/walproposer_pg.c @@ -88,7 +88,7 @@ static void StartProposerReplication(WalProposer *wp, StartReplicationCmd *cmd); static void WalSndLoop(WalProposer *wp); static void XLogBroadcastWalProposer(WalProposer *wp); -static void XLogWalPropWrite(char *buf, Size nbytes, XLogRecPtr recptr); +static void XLogWalPropWrite(WalProposer *wp, char *buf, Size nbytes, XLogRecPtr recptr); static void XLogWalPropClose(XLogRecPtr recptr); static void @@ -1241,7 +1241,7 @@ WalProposerRecovery(Safekeeper *sk, TimeLineID timeline, XLogRecPtr startpos, XL rec_end_lsn = rec_start_lsn + len - XLOG_HDR_SIZE; /* write WAL to disk */ - XLogWalPropWrite(&buf[XLOG_HDR_SIZE], len - XLOG_HDR_SIZE, rec_start_lsn); + XLogWalPropWrite(sk->wp, &buf[XLOG_HDR_SIZE], len - XLOG_HDR_SIZE, rec_start_lsn); ereport(DEBUG1, (errmsg("Recover message %X/%X length %d", @@ -1283,11 +1283,24 @@ static XLogSegNo walpropSegNo = 0; * Write XLOG data to disk. */ static void -XLogWalPropWrite(char *buf, Size nbytes, XLogRecPtr recptr) +XLogWalPropWrite(WalProposer *wp, char *buf, Size nbytes, XLogRecPtr recptr) { int startoff; int byteswritten; + /* + * Apart from walproposer, basebackup LSN page is also written out by + * postgres itself which writes WAL only in pages, and in basebackup it is + * inherently dummy (only safekeepers have historic WAL). Update WAL buffers + * here to avoid dummy page overwriting correct one we download here. Ugly, + * but alternatives are about the same ugly. We won't need that if we switch + * to on-demand WAL download from safekeepers, without writing to disk. + * + * https://github.com/neondatabase/neon/issues/5749 + */ + if (!wp->config->syncSafekeepers) + XLogUpdateWalBuffers(buf, recptr, nbytes); + while (nbytes > 0) { int segbytes; diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index 726e5e5def..d2d8d71e3f 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -1,11 +1,14 @@ import time +import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnv, logical_replication_sync, wait_for_last_flush_lsn, ) +from fixtures.types import Lsn +from fixtures.utils import query_scalar def test_logical_replication(neon_simple_env: NeonEnv, vanilla_pg): @@ -147,3 +150,89 @@ COMMIT; endpoint.start() # it must be gone (but walproposer slot still exists, hence 1) assert endpoint.safe_psql("select count(*) from pg_replication_slots")[0][0] == 1 + + +# Test compute start at LSN page of which starts with contrecord +# https://github.com/neondatabase/neon/issues/5749 +def test_wal_page_boundary_start(neon_simple_env: NeonEnv, vanilla_pg): + env = neon_simple_env + + env.neon_cli.create_branch("init") + endpoint = env.endpoints.create_start("init") + tenant_id = endpoint.safe_psql("show neon.tenant_id")[0][0] + timeline_id = endpoint.safe_psql("show neon.timeline_id")[0][0] + + cur = endpoint.connect().cursor() + cur.execute("create table t(key int, value text)") + cur.execute("CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int);") + cur.execute("insert into replication_example values (1, 2)") + cur.execute("create publication pub1 for table replication_example") + + # now start subscriber + vanilla_pg.start() + vanilla_pg.safe_psql("create table t(pk integer primary key, value text)") + vanilla_pg.safe_psql("CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int);") + + log.info(f"ep connstr is {endpoint.connstr()}, subscriber connstr {vanilla_pg.connstr()}") + connstr = endpoint.connstr().replace("'", "''") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + logical_replication_sync(vanilla_pg, endpoint) + vanilla_pg.stop() + + with endpoint.cursor() as cur: + # measure how much space logical message takes. Sometimes first attempt + # creates huge message and then it stabilizes, have no idea why. + for _ in range(3): + lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"current_lsn={lsn_before}") + # Non-transactional logical message doesn't write WAL, only XLogInsert's + # it, so use transactional. Which is a bit problematic as transactional + # necessitates commit record. Alternatively we can do smth like + # select neon_xlogflush(pg_current_wal_insert_lsn()); + # but isn't much better + that particular call complains on 'xlog flush + # request 0/282C018 is not satisfied' as pg_current_wal_insert_lsn skips + # page headers. + payload = "blahblah" + cur.execute(f"select pg_logical_emit_message(true, 'pref', '{payload}')") + lsn_after_by_curr_wal_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + lsn_diff = lsn_after_by_curr_wal_lsn - lsn_before + logical_message_base = lsn_after_by_curr_wal_lsn - lsn_before - len(payload) + log.info( + f"before {lsn_before}, after {lsn_after_by_curr_wal_lsn}, lsn diff is {lsn_diff}, base {logical_message_base}" + ) + + # and write logical message spanning exactly as we want + lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"current_lsn={lsn_before}") + curr_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + offs = int(curr_lsn) % 8192 + till_page = 8192 - offs + payload_len = ( + till_page - logical_message_base - 8 + ) # not sure why 8 is here, it is deduced from experiments + log.info(f"current_lsn={curr_lsn}, offs {offs}, till_page {till_page}") + + # payload_len above would go exactly till the page boundary; but we want contrecord, so make it slightly longer + payload_len += 8 + + cur.execute(f"select pg_logical_emit_message(true, 'pref', 'f{'a' * payload_len}')") + supposedly_contrecord_end = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"supposedly_page_boundary={supposedly_contrecord_end}") + # The calculations to hit the page boundary are very fuzzy, so just + # ignore test if we fail to reach it. + if not (int(supposedly_contrecord_end) % 8192 == 32): + pytest.skip("missed page boundary, bad luck") + + cur.execute("insert into replication_example values (2, 3)") + + wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop().start() + + cur = endpoint.connect().cursor() + # this should flush current wal page + cur.execute("insert into replication_example values (3, 4)") + vanilla_pg.start() + logical_replication_sync(vanilla_pg, endpoint) + assert vanilla_pg.safe_psql( + "select sum(somedata) from replication_example" + ) == endpoint.safe_psql("select sum(somedata) from replication_example") diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 6669a672ee..dd067cf656 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 6669a672ee14ab2c09d44c4552f9a13fad3afc10 +Subproject commit dd067cf656f6810a25aca6025633d32d02c5085a diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index ab67ab9635..bc88f53931 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit ab67ab96355d61e9d0218630be4aa7db53bf83e7 +Subproject commit bc88f539312fcc4bb292ce94ae9db09ab6656e8a diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 550ffa6495..763000f1d0 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 550ffa6495a5dc62fccc3a8b449386633758680b +Subproject commit 763000f1d0873b827829c41f2f6f799ffc0de55c diff --git a/vendor/revisions.json b/vendor/revisions.json index 012fb14035..377357e131 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "postgres-v16": "550ffa6495a5dc62fccc3a8b449386633758680b", - "postgres-v15": "ab67ab96355d61e9d0218630be4aa7db53bf83e7", - "postgres-v14": "6669a672ee14ab2c09d44c4552f9a13fad3afc10" + "postgres-v16": "763000f1d0873b827829c41f2f6f799ffc0de55c", + "postgres-v15": "bc88f539312fcc4bb292ce94ae9db09ab6656e8a", + "postgres-v14": "dd067cf656f6810a25aca6025633d32d02c5085a" } From 306c4f99678d5d256c9ad237dd502e9dbe4bb63e Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 3 Nov 2023 17:36:02 +0000 Subject: [PATCH 02/63] s3_scrubber: prepare for scrubbing buckets with generation-aware content (#5700) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The scrubber didn't know how to find the latest index_part when generations were in use. ## Summary of changes - Teach the scrubber to do the same dance that pageserver does when finding the latest index_part.json - Teach the scrubber how to understand layer files with generation suffixes. - General improvement to testability: scan_metadata has a machine readable output that the testing `S3Scrubber` wrapper can read. - Existing test coverage of scrubber was false-passing because it just didn't see any data due to prefixing of data in the bucket. Fix that. This is incremental improvement: the more confidence we can have in the scrubber, the more we can use it in integration tests to validate the state of remote storage. --------- Co-authored-by: Arpad Müller --- Cargo.lock | 1 + libs/utils/src/generation.rs | 2 +- .../src/tenant/remote_timeline_client.rs | 2 +- .../tenant/remote_timeline_client/index.rs | 2 +- s3_scrubber/Cargo.toml | 1 + s3_scrubber/src/checks.rs | 173 ++++++++++++------ s3_scrubber/src/lib.rs | 39 ++-- s3_scrubber/src/main.rs | 13 +- s3_scrubber/src/metadata_stream.rs | 4 +- s3_scrubber/src/scan_metadata.rs | 5 + test_runner/fixtures/neon_fixtures.py | 31 +++- test_runner/fixtures/utils.py | 3 +- .../regress/test_pageserver_generations.py | 15 ++ 13 files changed, 209 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d9b20dae4..3e9a7198a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4419,6 +4419,7 @@ dependencies = [ "itertools", "pageserver", "rand 0.8.5", + "remote_storage", "reqwest", "serde", "serde_json", diff --git a/libs/utils/src/generation.rs b/libs/utils/src/generation.rs index 88d50905c6..49e290dab8 100644 --- a/libs/utils/src/generation.rs +++ b/libs/utils/src/generation.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; /// /// See docs/rfcs/025-generation-numbers.md for detail on how generation /// numbers are used. -#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)] pub enum Generation { // Generations with this magic value will not add a suffix to S3 keys, and will not // be included in persisted index_part.json. This value is only to be used diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 76e75253fb..bbf6a0c5c5 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1542,7 +1542,7 @@ pub fn remote_index_path( } /// Given the key of an index, parse out the generation part of the name -pub(crate) fn parse_remote_index_path(path: RemotePath) -> Option { +pub fn parse_remote_index_path(path: RemotePath) -> Option { let file_name = match path.get_path().file_name() { Some(f) => f, None => { diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 7cf963ca9d..fdab74a8be 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -155,7 +155,7 @@ pub struct IndexLayerMetadata { #[serde(default = "Generation::none")] #[serde(skip_serializing_if = "Generation::is_none")] - pub(super) generation: Generation, + pub generation: Generation, } impl From for IndexLayerMetadata { diff --git a/s3_scrubber/Cargo.toml b/s3_scrubber/Cargo.toml index f3ea6e222c..0f3e5630e8 100644 --- a/s3_scrubber/Cargo.toml +++ b/s3_scrubber/Cargo.toml @@ -33,6 +33,7 @@ reqwest = { workspace = true, default-features = false, features = ["rustls-tls" aws-config = { workspace = true, default-features = false, features = ["rustls", "credentials-sso"] } pageserver = { path = "../pageserver" } +remote_storage = { path = "../libs/remote_storage" } tracing.workspace = true tracing-subscriber.workspace = true diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs index c11f1f9779..64702fca3d 100644 --- a/s3_scrubber/src/checks.rs +++ b/s3_scrubber/src/checks.rs @@ -1,13 +1,18 @@ use std::collections::HashSet; use anyhow::Context; -use aws_sdk_s3::Client; +use aws_sdk_s3::{types::ObjectIdentifier, Client}; use tracing::{error, info, warn}; +use utils::generation::Generation; use crate::cloud_admin_api::BranchData; -use crate::{download_object_with_retries, list_objects_with_retries, RootTarget}; +use crate::metadata_stream::stream_listing; +use crate::{download_object_with_retries, RootTarget}; +use futures_util::{pin_mut, StreamExt}; +use pageserver::tenant::remote_timeline_client::parse_remote_index_path; use pageserver::tenant::storage_layer::LayerFileName; use pageserver::tenant::IndexPart; +use remote_storage::RemotePath; use utils::id::TenantTimelineId; pub(crate) struct TimelineAnalysis { @@ -68,6 +73,7 @@ pub(crate) async fn branch_cleanup_and_check_errors( match s3_data.blob_data { BlobDataParseResult::Parsed { index_part, + index_part_generation, mut s3_layers, } => { if !IndexPart::KNOWN_VERSIONS.contains(&index_part.get_version()) { @@ -107,33 +113,62 @@ pub(crate) async fn branch_cleanup_and_check_errors( )) } - if !s3_layers.remove(&layer) { + let layer_map_key = (layer, metadata.generation); + if !s3_layers.remove(&layer_map_key) { + // FIXME: this will emit false positives if an index was + // uploaded concurrently with our scan. To make this check + // correct, we need to try sending a HEAD request for the + // layer we think is missing. result.errors.push(format!( - "index_part.json contains a layer {} that is not present in S3", - layer.file_name(), + "index_part.json contains a layer {}{} that is not present in remote storage", + layer_map_key.0.file_name(), + layer_map_key.1.get_suffix() )) } } - if !s3_layers.is_empty() { + let orphan_layers: Vec<(LayerFileName, Generation)> = s3_layers + .into_iter() + .filter(|(_layer_name, gen)| + // A layer is only considered orphaned if it has a generation below + // the index. If the generation is >= the index, then the layer may + // be an upload from a running pageserver, or even an upload from + // a new generation that didn't upload an index yet. + // + // Even so, a layer that is not referenced by the index could just + // be something enqueued for deletion, so while this check is valid + // for indicating that a layer is garbage, it is not an indicator + // of a problem. + gen < &index_part_generation) + .collect(); + + if !orphan_layers.is_empty() { result.errors.push(format!( "index_part.json does not contain layers from S3: {:?}", - s3_layers + orphan_layers .iter() - .map(|layer_name| layer_name.file_name()) + .map(|(layer_name, gen)| format!( + "{}{}", + layer_name.file_name(), + gen.get_suffix() + )) .collect::>(), )); - result - .garbage_keys - .extend(s3_layers.iter().map(|layer_name| { + result.garbage_keys.extend(orphan_layers.iter().map( + |(layer_name, layer_gen)| { let mut key = s3_root.timeline_root(id).prefix_in_bucket; let delimiter = s3_root.delimiter(); if !key.ends_with(delimiter) { key.push_str(delimiter); } - key.push_str(&layer_name.file_name()); + key.push_str(&format!( + "{}{}", + &layer_name.file_name(), + layer_gen.get_suffix() + )); key - })); + }, + )); } } BlobDataParseResult::Incorrect(parse_errors) => result.errors.extend( @@ -178,70 +213,97 @@ pub(crate) struct S3TimelineBlobData { pub(crate) enum BlobDataParseResult { Parsed { index_part: IndexPart, - s3_layers: HashSet, + index_part_generation: Generation, + s3_layers: HashSet<(LayerFileName, Generation)>, }, Incorrect(Vec), } +fn parse_layer_object_name(name: &str) -> Result<(LayerFileName, Generation), String> { + match name.rsplit_once('-') { + // FIXME: this is gross, just use a regex? + Some((layer_filename, gen)) if gen.len() == 8 => { + let layer = layer_filename.parse::()?; + let gen = + Generation::parse_suffix(gen).ok_or("Malformed generation suffix".to_string())?; + Ok((layer, gen)) + } + _ => Ok((name.parse::()?, Generation::none())), + } +} + pub(crate) async fn list_timeline_blobs( s3_client: &Client, id: TenantTimelineId, s3_root: &RootTarget, ) -> anyhow::Result { let mut s3_layers = HashSet::new(); - let mut index_part_object = None; - - let timeline_dir_target = s3_root.timeline_root(&id); - let mut continuation_token = None; let mut errors = Vec::new(); let mut keys_to_remove = Vec::new(); - loop { - let fetch_response = - list_objects_with_retries(s3_client, &timeline_dir_target, continuation_token.clone()) - .await?; + let mut timeline_dir_target = s3_root.timeline_root(&id); + timeline_dir_target.delimiter = String::new(); - let subdirectories = fetch_response.common_prefixes().unwrap_or_default(); - if !subdirectories.is_empty() { - errors.push(format!( - "S3 list response should not contain any subdirectories, but got {subdirectories:?}" - )); - } + let mut index_parts: Vec = Vec::new(); - for (object, key) in fetch_response - .contents() - .unwrap_or_default() - .iter() - .filter_map(|object| Some((object, object.key()?))) - { - let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); - match blob_name { - Some("index_part.json") => index_part_object = Some(object.clone()), - Some(maybe_layer_name) => match maybe_layer_name.parse::() { - Ok(new_layer) => { - s3_layers.insert(new_layer); - } - Err(e) => { - errors.push( - format!("S3 list response got an object with key {key} that is not a layer name: {e}"), - ); - keys_to_remove.push(key.to_string()); - } - }, - None => { - errors.push(format!("S3 list response got an object with odd key {key}")); + let stream = stream_listing(s3_client, &timeline_dir_target); + pin_mut!(stream); + while let Some(obj) = stream.next().await { + let obj = obj?; + let key = match obj.key() { + Some(k) => k, + None => continue, + }; + + let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); + match blob_name { + Some(name) if name.starts_with("index_part.json") => { + tracing::info!("Index key {key}"); + index_parts.push(obj) + } + Some(maybe_layer_name) => match parse_layer_object_name(maybe_layer_name) { + Ok((new_layer, gen)) => { + tracing::info!("Parsed layer key: {} {:?}", new_layer, gen); + s3_layers.insert((new_layer, gen)); + } + Err(e) => { + tracing::info!("Error parsing key {maybe_layer_name}"); + errors.push( + format!("S3 list response got an object with key {key} that is not a layer name: {e}"), + ); keys_to_remove.push(key.to_string()); } + }, + None => { + tracing::info!("Peculiar key {}", key); + errors.push(format!("S3 list response got an object with odd key {key}")); + keys_to_remove.push(key.to_string()); } } - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } } + // Choose the index_part with the highest generation + let (index_part_object, index_part_generation) = match index_parts + .iter() + .filter_map(|k| { + let key = k.key().unwrap(); + // Stripping the index key to the last part, because RemotePath doesn't + // like absolute paths, and depending on prefix_in_bucket it's possible + // for the keys we read back to start with a slash. + let basename = key.rsplit_once('/').unwrap().1; + parse_remote_index_path(RemotePath::from_string(basename).unwrap()).map(|g| (k, g)) + }) + .max_by_key(|i| i.1) + .map(|(k, g)| (k.clone(), g)) + { + Some((key, gen)) => (Some(key), gen), + None => { + // Legacy/missing case: one or zero index parts, which did not have a generation + (index_parts.pop(), Generation::none()) + } + }; + if index_part_object.is_none() { errors.push("S3 list response got no index_part.json file".to_string()); } @@ -261,6 +323,7 @@ pub(crate) async fn list_timeline_blobs( return Ok(S3TimelineBlobData { blob_data: BlobDataParseResult::Parsed { index_part, + index_part_generation, s3_layers, }, keys_to_remove, diff --git a/s3_scrubber/src/lib.rs b/s3_scrubber/src/lib.rs index 98e7ed5334..d892438633 100644 --- a/s3_scrubber/src/lib.rs +++ b/s3_scrubber/src/lib.rs @@ -34,6 +34,9 @@ const CLOUD_ADMIN_API_TOKEN_ENV_VAR: &str = "CLOUD_ADMIN_API_TOKEN"; #[derive(Debug, Clone)] pub struct S3Target { pub bucket_name: String, + /// This `prefix_in_bucket` is only equal to the PS/SK config of the same + /// name for the RootTarget: other instances of S3Target will have prefix_in_bucket + /// with extra parts. pub prefix_in_bucket: String, pub delimiter: String, } @@ -77,9 +80,13 @@ impl Display for NodeKind { impl S3Target { pub fn with_sub_segment(&self, new_segment: &str) -> Self { let mut new_self = self.clone(); - let _ = new_self.prefix_in_bucket.pop(); - new_self.prefix_in_bucket = - [&new_self.prefix_in_bucket, new_segment, ""].join(&new_self.delimiter); + if new_self.prefix_in_bucket.is_empty() { + new_self.prefix_in_bucket = format!("/{}/", new_segment); + } else { + let _ = new_self.prefix_in_bucket.pop(); + new_self.prefix_in_bucket = + [&new_self.prefix_in_bucket, new_segment, ""].join(&new_self.delimiter); + } new_self } } @@ -91,10 +98,10 @@ pub enum RootTarget { } impl RootTarget { - pub fn tenants_root(&self) -> &S3Target { + pub fn tenants_root(&self) -> S3Target { match self { - Self::Pageserver(root) => root, - Self::Safekeeper(root) => root, + Self::Pageserver(root) => root.with_sub_segment(TENANTS_SEGMENT_NAME), + Self::Safekeeper(root) => root.with_sub_segment("wal"), } } @@ -133,6 +140,7 @@ impl RootTarget { pub struct BucketConfig { pub region: String, pub bucket: String, + pub prefix_in_bucket: Option, /// Use SSO if this is set, else rely on AWS_* environment vars pub sso_account_id: Option, @@ -155,10 +163,12 @@ impl BucketConfig { let sso_account_id = env::var("SSO_ACCOUNT_ID").ok(); let region = env::var("REGION").context("'REGION' param retrieval")?; let bucket = env::var("BUCKET").context("'BUCKET' param retrieval")?; + let prefix_in_bucket = env::var("BUCKET_PREFIX").ok(); Ok(Self { region, bucket, + prefix_in_bucket, sso_account_id, }) } @@ -191,14 +201,14 @@ pub fn init_logging(file_name: &str) -> WorkerGuard { .with_target(false) .with_ansi(false) .with_writer(file_writer); - let stdout_logs = fmt::Layer::new() - .with_ansi(std::io::stdout().is_terminal()) + let stderr_logs = fmt::Layer::new() + .with_ansi(std::io::stderr().is_terminal()) .with_target(false) - .with_writer(std::io::stdout); + .with_writer(std::io::stderr); tracing_subscriber::registry() .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) .with(file_logs) - .with(stdout_logs) + .with(stderr_logs) .init(); guard @@ -250,15 +260,20 @@ fn init_remote( let bucket_region = Region::new(bucket_config.region); let delimiter = "/".to_string(); let s3_client = Arc::new(init_s3_client(bucket_config.sso_account_id, bucket_region)); + let s3_root = match node_kind { NodeKind::Pageserver => RootTarget::Pageserver(S3Target { bucket_name: bucket_config.bucket, - prefix_in_bucket: ["pageserver", "v1", TENANTS_SEGMENT_NAME, ""].join(&delimiter), + prefix_in_bucket: bucket_config + .prefix_in_bucket + .unwrap_or("pageserver/v1".to_string()), delimiter, }), NodeKind::Safekeeper => RootTarget::Safekeeper(S3Target { bucket_name: bucket_config.bucket, - prefix_in_bucket: ["safekeeper", "v1", "wal", ""].join(&delimiter), + prefix_in_bucket: bucket_config + .prefix_in_bucket + .unwrap_or("safekeeper/v1".to_string()), delimiter, }), }; diff --git a/s3_scrubber/src/main.rs b/s3_scrubber/src/main.rs index 9a0d6c9ae8..1f0ceebdaf 100644 --- a/s3_scrubber/src/main.rs +++ b/s3_scrubber/src/main.rs @@ -31,7 +31,10 @@ enum Command { #[arg(short, long, default_value_t = PurgeMode::DeletedOnly)] mode: PurgeMode, }, - ScanMetadata {}, + ScanMetadata { + #[arg(short, long, default_value_t = false)] + json: bool, + }, } #[tokio::main] @@ -54,13 +57,17 @@ async fn main() -> anyhow::Result<()> { )); match cli.command { - Command::ScanMetadata {} => match scan_metadata(bucket_config).await { + Command::ScanMetadata { json } => match scan_metadata(bucket_config).await { Err(e) => { tracing::error!("Failed: {e}"); Err(e) } Ok(summary) => { - println!("{}", summary.summary_string()); + if json { + println!("{}", serde_json::to_string(&summary).unwrap()) + } else { + println!("{}", summary.summary_string()); + } if summary.is_fatal() { Err(anyhow::anyhow!("Fatal scrub errors detected")) } else { diff --git a/s3_scrubber/src/metadata_stream.rs b/s3_scrubber/src/metadata_stream.rs index 125a370e90..8095071c1f 100644 --- a/s3_scrubber/src/metadata_stream.rs +++ b/s3_scrubber/src/metadata_stream.rs @@ -13,10 +13,10 @@ pub fn stream_tenants<'a>( ) -> impl Stream> + 'a { try_stream! { let mut continuation_token = None; + let tenants_target = target.tenants_root(); loop { - let tenants_target = target.tenants_root(); let fetch_response = - list_objects_with_retries(s3_client, tenants_target, continuation_token.clone()).await?; + list_objects_with_retries(s3_client, &tenants_target, continuation_token.clone()).await?; let new_entry_ids = fetch_response .common_prefixes() diff --git a/s3_scrubber/src/scan_metadata.rs b/s3_scrubber/src/scan_metadata.rs index 33acd41a5b..ad82db1e76 100644 --- a/s3_scrubber/src/scan_metadata.rs +++ b/s3_scrubber/src/scan_metadata.rs @@ -10,8 +10,10 @@ use aws_sdk_s3::Client; use futures_util::{pin_mut, StreamExt, TryStreamExt}; use histogram::Histogram; use pageserver::tenant::IndexPart; +use serde::Serialize; use utils::id::TenantTimelineId; +#[derive(Serialize)] pub struct MetadataSummary { count: usize, with_errors: HashSet, @@ -25,7 +27,9 @@ pub struct MetadataSummary { } /// A histogram plus minimum and maximum tracking +#[derive(Serialize)] struct MinMaxHisto { + #[serde(skip)] histo: Histogram, min: u64, max: u64, @@ -109,6 +113,7 @@ impl MetadataSummary { self.count += 1; if let BlobDataParseResult::Parsed { index_part, + index_part_generation: _, s3_layers: _, } = &data.blob_data { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 81a7b0750d..02faf715da 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2968,24 +2968,33 @@ class S3Scrubber: self.env = env self.log_dir = log_dir - def scrubber_cli(self, args, timeout): + def scrubber_cli(self, args: list[str], timeout) -> str: assert isinstance(self.env.pageserver_remote_storage, S3Storage) s3_storage = self.env.pageserver_remote_storage env = { "REGION": s3_storage.bucket_region, "BUCKET": s3_storage.bucket_name, + "BUCKET_PREFIX": s3_storage.prefix_in_bucket, + "RUST_LOG": "DEBUG", } env.update(s3_storage.access_env_vars()) if s3_storage.endpoint is not None: env.update({"AWS_ENDPOINT_URL": s3_storage.endpoint}) - base_args = [self.env.neon_binpath / "s3_scrubber"] + base_args = [str(self.env.neon_binpath / "s3_scrubber")] args = base_args + args - (output_path, _, status_code) = subprocess_capture( - self.log_dir, args, echo_stderr=True, echo_stdout=True, env=env, check=False + (output_path, stdout, status_code) = subprocess_capture( + self.log_dir, + args, + echo_stderr=True, + echo_stdout=True, + env=env, + check=False, + capture_stdout=True, + timeout=timeout, ) if status_code: log.warning(f"Scrub command {args} failed") @@ -2994,8 +3003,18 @@ class S3Scrubber: raise RuntimeError("Remote storage scrub failed") - def scan_metadata(self): - self.scrubber_cli(["scan-metadata"], timeout=30) + assert stdout is not None + return stdout + + def scan_metadata(self) -> Any: + stdout = self.scrubber_cli(["scan-metadata", "--json"], timeout=30) + + try: + return json.loads(stdout) + except: + log.error("Failed to decode JSON output from `scan-metadata`. Dumping stdout:") + log.error(stdout) + raise def get_test_output_dir(request: FixtureRequest, top_output_dir: Path) -> Path: diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index e54b82dfb4..ba8d70d5a9 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -35,6 +35,7 @@ def subprocess_capture( echo_stderr=False, echo_stdout=False, capture_stdout=False, + timeout=None, **kwargs: Any, ) -> Tuple[str, Optional[str], int]: """Run a process and bifurcate its output to files and the `log` logger @@ -104,7 +105,7 @@ def subprocess_capture( stderr_handler = OutputHandler(p.stderr, stderr_f, echo=echo_stderr, capture=False) stderr_handler.start() - r = p.wait() + r = p.wait(timeout=timeout) stdout_handler.join() stderr_handler.join() diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 78ef6b9165..3e5021ae06 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -21,6 +21,7 @@ from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, PgBin, + S3Scrubber, last_flush_lsn_upload, wait_for_last_flush_lsn, ) @@ -234,8 +235,22 @@ def test_generations_upgrade(neon_env_builder: NeonEnvBuilder): assert len(suffixed_objects) > 0 assert len(legacy_objects) > 0 + # Flush through deletions to get a clean state for scrub: we are implicitly validating + # that our generations-enabled pageserver was able to do deletions of layers + # from earlier which don't have a generation. + env.pageserver.http_client().deletion_queue_flush(execute=True) + assert get_deletion_queue_unexpected_errors(env.pageserver.http_client()) == 0 + # Having written a mixture of generation-aware and legacy index_part.json, + # ensure the scrubber handles the situation as expected. + metadata_summary = S3Scrubber( + neon_env_builder.test_output_dir, neon_env_builder + ).scan_metadata() + assert metadata_summary["count"] == 1 # Scrubber should have seen our timeline + assert not metadata_summary["with_errors"] + assert not metadata_summary["with_warnings"] + def test_deferred_deletion(neon_env_builder: NeonEnvBuilder): neon_env_builder.enable_generations = True From 09b5954526d5192fc54f181c719b129f8db08249 Mon Sep 17 00:00:00 2001 From: duguorong009 <80258679+duguorong009@users.noreply.github.com> Date: Sun, 5 Nov 2023 05:16:54 -0500 Subject: [PATCH 03/63] refactor: use streaming in safekeeper `/v1/debug_dump` http response (#5731) - Update the handler for `/v1/debug_dump` http response in safekeeper - Update the `debug_dump::build()` to use the streaming in JSON build process --- Cargo.lock | 1 + libs/utils/src/http/endpoint.rs | 170 +++++++++++++------------- safekeeper/Cargo.toml | 1 + safekeeper/src/debug_dump.rs | 111 +++++++++++------ safekeeper/src/http/routes.rs | 55 ++++++++- safekeeper/src/pull_timeline.rs | 13 +- test_runner/fixtures/neon_fixtures.py | 2 +- 7 files changed, 227 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e9a7198a9..2203ad462c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4478,6 +4478,7 @@ dependencies = [ "tokio", "tokio-io-timeout", "tokio-postgres", + "tokio-stream", "toml_edit", "tracing", "url", diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index f3f5e95d0b..a9ee2425a4 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -14,6 +14,11 @@ use tracing::{self, debug, info, info_span, warn, Instrument}; use std::future::Future; use std::str::FromStr; +use bytes::{Bytes, BytesMut}; +use std::io::Write as _; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + static SERVE_METRICS_COUNT: Lazy = Lazy::new(|| { register_int_counter!( "libmetrics_metric_handler_requests_total", @@ -146,94 +151,89 @@ impl Drop for RequestCancelled { } } +/// An [`std::io::Write`] implementation on top of a channel sending [`bytes::Bytes`] chunks. +pub struct ChannelWriter { + buffer: BytesMut, + pub tx: mpsc::Sender>, + written: usize, +} + +impl ChannelWriter { + pub fn new(buf_len: usize, tx: mpsc::Sender>) -> Self { + assert_ne!(buf_len, 0); + ChannelWriter { + // split about half off the buffer from the start, because we flush depending on + // capacity. first flush will come sooner than without this, but now resizes will + // have better chance of picking up the "other" half. not guaranteed of course. + buffer: BytesMut::with_capacity(buf_len).split_off(buf_len / 2), + tx, + written: 0, + } + } + + pub fn flush0(&mut self) -> std::io::Result { + let n = self.buffer.len(); + if n == 0 { + return Ok(0); + } + + tracing::trace!(n, "flushing"); + let ready = self.buffer.split().freeze(); + + // not ideal to call from blocking code to block_on, but we are sure that this + // operation does not spawn_blocking other tasks + let res: Result<(), ()> = tokio::runtime::Handle::current().block_on(async { + self.tx.send(Ok(ready)).await.map_err(|_| ())?; + + // throttle sending to allow reuse of our buffer in `write`. + self.tx.reserve().await.map_err(|_| ())?; + + // now the response task has picked up the buffer and hopefully started + // sending it to the client. + Ok(()) + }); + if res.is_err() { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + self.written += n; + Ok(n) + } + + pub fn flushed_bytes(&self) -> usize { + self.written + } +} + +impl std::io::Write for ChannelWriter { + fn write(&mut self, mut buf: &[u8]) -> std::io::Result { + let remaining = self.buffer.capacity() - self.buffer.len(); + + let out_of_space = remaining < buf.len(); + + let original_len = buf.len(); + + if out_of_space { + let can_still_fit = buf.len() - remaining; + self.buffer.extend_from_slice(&buf[..can_still_fit]); + buf = &buf[can_still_fit..]; + self.flush0()?; + } + + // assume that this will often under normal operation just move the pointer back to the + // beginning of allocation, because previous split off parts are already sent and + // dropped. + self.buffer.extend_from_slice(buf); + Ok(original_len) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.flush0().map(|_| ()) + } +} + async fn prometheus_metrics_handler(_req: Request) -> Result, ApiError> { - use bytes::{Bytes, BytesMut}; - use std::io::Write as _; - use tokio::sync::mpsc; - use tokio_stream::wrappers::ReceiverStream; - SERVE_METRICS_COUNT.inc(); - /// An [`std::io::Write`] implementation on top of a channel sending [`bytes::Bytes`] chunks. - struct ChannelWriter { - buffer: BytesMut, - tx: mpsc::Sender>, - written: usize, - } - - impl ChannelWriter { - fn new(buf_len: usize, tx: mpsc::Sender>) -> Self { - assert_ne!(buf_len, 0); - ChannelWriter { - // split about half off the buffer from the start, because we flush depending on - // capacity. first flush will come sooner than without this, but now resizes will - // have better chance of picking up the "other" half. not guaranteed of course. - buffer: BytesMut::with_capacity(buf_len).split_off(buf_len / 2), - tx, - written: 0, - } - } - - fn flush0(&mut self) -> std::io::Result { - let n = self.buffer.len(); - if n == 0 { - return Ok(0); - } - - tracing::trace!(n, "flushing"); - let ready = self.buffer.split().freeze(); - - // not ideal to call from blocking code to block_on, but we are sure that this - // operation does not spawn_blocking other tasks - let res: Result<(), ()> = tokio::runtime::Handle::current().block_on(async { - self.tx.send(Ok(ready)).await.map_err(|_| ())?; - - // throttle sending to allow reuse of our buffer in `write`. - self.tx.reserve().await.map_err(|_| ())?; - - // now the response task has picked up the buffer and hopefully started - // sending it to the client. - Ok(()) - }); - if res.is_err() { - return Err(std::io::ErrorKind::BrokenPipe.into()); - } - self.written += n; - Ok(n) - } - - fn flushed_bytes(&self) -> usize { - self.written - } - } - - impl std::io::Write for ChannelWriter { - fn write(&mut self, mut buf: &[u8]) -> std::io::Result { - let remaining = self.buffer.capacity() - self.buffer.len(); - - let out_of_space = remaining < buf.len(); - - let original_len = buf.len(); - - if out_of_space { - let can_still_fit = buf.len() - remaining; - self.buffer.extend_from_slice(&buf[..can_still_fit]); - buf = &buf[can_still_fit..]; - self.flush0()?; - } - - // assume that this will often under normal operation just move the pointer back to the - // beginning of allocation, because previous split off parts are already sent and - // dropped. - self.buffer.extend_from_slice(buf); - Ok(original_len) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.flush0().map(|_| ()) - } - } - let started_at = std::time::Instant::now(); let (tx, rx) = mpsc::channel(1); diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index 64ef9f6997..30516c0763 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -47,6 +47,7 @@ pq_proto.workspace = true remote_storage.workspace = true safekeeper_api.workspace = true storage_broker.workspace = true +tokio-stream.workspace = true utils.workspace = true workspace_hack.workspace = true diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index ee9d7118c6..8cbc0aa47f 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -5,6 +5,7 @@ use std::fs::DirEntry; use std::io::BufReader; use std::io::Read; use std::path::PathBuf; +use std::sync::Arc; use anyhow::Result; use camino::Utf8Path; @@ -28,7 +29,7 @@ use crate::send_wal::WalSenderState; use crate::GlobalTimelines; /// Various filters that influence the resulting JSON output. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct Args { /// Dump all available safekeeper state. False by default. pub dump_all: bool, @@ -53,15 +54,76 @@ pub struct Args { } /// Response for debug dump request. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize)] pub struct Response { pub start_time: DateTime, pub finish_time: DateTime, - pub timelines: Vec, + pub timelines: Vec, pub timelines_count: usize, pub config: Config, } +pub struct TimelineDumpSer { + pub tli: Arc, + pub args: Args, + pub runtime: Arc, +} + +impl std::fmt::Debug for TimelineDumpSer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TimelineDumpSer") + .field("tli", &self.tli.ttid) + .field("args", &self.args) + .finish() + } +} + +impl Serialize for TimelineDumpSer { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let dump = self + .runtime + .block_on(build_from_tli_dump(self.tli.clone(), self.args.clone())); + dump.serialize(serializer) + } +} + +async fn build_from_tli_dump(timeline: Arc, args: Args) -> Timeline { + let control_file = if args.dump_control_file { + let mut state = timeline.get_state().await.1; + if !args.dump_term_history { + state.acceptor_state.term_history = TermHistory(vec![]); + } + Some(state) + } else { + None + }; + + let memory = if args.dump_memory { + Some(timeline.memory_dump().await) + } else { + None + }; + + let disk_content = if args.dump_disk_content { + // build_disk_content can fail, but we don't want to fail the whole + // request because of that. + build_disk_content(&timeline.timeline_dir).ok() + } else { + None + }; + + Timeline { + tenant_id: timeline.ttid.tenant_id, + timeline_id: timeline.ttid.timeline_id, + control_file, + memory, + disk_content, + } +} + /// Safekeeper configuration. #[derive(Debug, Serialize, Deserialize)] pub struct Config { @@ -140,8 +202,12 @@ pub async fn build(args: Args) -> Result { GlobalTimelines::get_all() }; - // TODO: return Stream instead of Vec let mut timelines = Vec::new(); + let runtime = Arc::new( + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(), + ); for tli in ptrs_snapshot { let ttid = tli.ttid; if let Some(tenant_id) = args.tenant_id { @@ -155,38 +221,11 @@ pub async fn build(args: Args) -> Result { } } - let control_file = if args.dump_control_file { - let mut state = tli.get_state().await.1; - if !args.dump_term_history { - state.acceptor_state.term_history = TermHistory(vec![]); - } - Some(state) - } else { - None - }; - - let memory = if args.dump_memory { - Some(tli.memory_dump().await) - } else { - None - }; - - let disk_content = if args.dump_disk_content { - // build_disk_content can fail, but we don't want to fail the whole - // request because of that. - build_disk_content(&tli.timeline_dir).ok() - } else { - None - }; - - let timeline = Timeline { - tenant_id: ttid.tenant_id, - timeline_id: ttid.timeline_id, - control_file, - memory, - disk_content, - }; - timelines.push(timeline); + timelines.push(TimelineDumpSer { + tli, + args: args.clone(), + runtime: runtime.clone(), + }); } let config = GlobalTimelines::get_global_config(); diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 940ac82df6..474d636441 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -13,7 +13,12 @@ use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use tokio::fs::File; use tokio::io::AsyncReadExt; -use utils::http::endpoint::request_span; + +use std::io::Write as _; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tracing::info_span; +use utils::http::endpoint::{request_span, ChannelWriter}; use crate::receive_wal::WalReceiverState; use crate::safekeeper::Term; @@ -373,8 +378,52 @@ async fn dump_debug_handler(mut request: Request) -> Result .await .map_err(ApiError::InternalServerError)?; - // TODO: use streaming response - json_response(StatusCode::OK, resp) + let started_at = std::time::Instant::now(); + + let (tx, rx) = mpsc::channel(1); + + let body = Body::wrap_stream(ReceiverStream::new(rx)); + + let mut writer = ChannelWriter::new(128 * 1024, tx); + + let response = Response::builder() + .status(200) + .header(hyper::header::CONTENT_TYPE, "application/octet-stream") + .body(body) + .unwrap(); + + let span = info_span!("blocking"); + tokio::task::spawn_blocking(move || { + let _span = span.entered(); + + let res = serde_json::to_writer(&mut writer, &resp) + .map_err(std::io::Error::from) + .and_then(|_| writer.flush()); + + match res { + Ok(()) => { + tracing::info!( + bytes = writer.flushed_bytes(), + elapsed_ms = started_at.elapsed().as_millis(), + "responded /v1/debug_dump" + ); + } + Err(e) => { + tracing::warn!("failed to write out /v1/debug_dump response: {e:#}"); + // semantics of this error are quite... unclear. we want to error the stream out to + // abort the response to somehow notify the client that we failed. + // + // though, most likely the reason for failure is that the receiver is already gone. + drop( + writer + .tx + .blocking_send(Err(std::io::ErrorKind::BrokenPipe.into())), + ); + } + } + }); + + Ok(response) } /// Safekeeper http router. diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 1343bba5cc..e2f1b9fcff 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use anyhow::{bail, Context, Result}; @@ -32,6 +33,16 @@ pub struct Response { // TODO: add more fields? } +/// Response for debug dump request. +#[derive(Debug, Serialize, Deserialize)] +pub struct DebugDumpResponse { + pub start_time: DateTime, + pub finish_time: DateTime, + pub timelines: Vec, + pub timelines_count: usize, + pub config: debug_dump::Config, +} + /// Find the most advanced safekeeper and pull timeline from it. pub async fn handle_request(request: Request) -> Result { let existing_tli = GlobalTimelines::get(TenantTimelineId::new( @@ -103,7 +114,7 @@ async fn pull_timeline(status: TimelineStatus, host: String) -> Result // Implementing our own scp over HTTP. // At first, we need to fetch list of files from safekeeper. - let dump: debug_dump::Response = client + let dump: DebugDumpResponse = client .get(format!( "{}/v1/debug_dump?dump_all=true&tenant_id={}&timeline_id={}", host, status.tenant_id, status.timeline_id diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 02faf715da..740c05dcea 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2868,7 +2868,7 @@ class SafekeeperHttpClient(requests.Session): params = params or {} res = self.get(f"http://localhost:{self.port}/v1/debug_dump", params=params) res.raise_for_status() - res_json = res.json() + res_json = json.loads(res.text) assert isinstance(res_json, dict) return res_json From b85fc39bdb49cd19fea4e6bab6d3cec52d9ca471 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 6 Nov 2023 09:26:09 +0200 Subject: [PATCH 04/63] Update control plane API path for getting compute spec. (#5357) We changed the path in the control plane. The old path is still accepted for compatibility with existing computes, but we'd like to phase it out. --- compute_tools/src/spec.rs | 2 +- test_runner/regress/test_ddl_forwarding.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 3fa3cf055f..a85d6287b1 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -68,7 +68,7 @@ pub fn get_spec_from_control_plane( base_uri: &str, compute_id: &str, ) -> Result> { - let cp_uri = format!("{base_uri}/management/api/v2/computes/{compute_id}/spec"); + let cp_uri = format!("{base_uri}/compute/api/v2/computes/{compute_id}/spec"); let jwt: String = match std::env::var("NEON_CONTROL_PLANE_TOKEN") { Ok(v) => v, Err(_) => "".to_string(), diff --git a/test_runner/regress/test_ddl_forwarding.py b/test_runner/regress/test_ddl_forwarding.py index 6bd09c7030..07ee8b405e 100644 --- a/test_runner/regress/test_ddl_forwarding.py +++ b/test_runner/regress/test_ddl_forwarding.py @@ -72,7 +72,7 @@ class DdlForwardingContext: self.dbs: Dict[str, str] = {} self.roles: Dict[str, str] = {} self.fail = False - endpoint = "/management/api/v2/roles_and_databases" + endpoint = "/test/roles_and_databases" ddl_url = f"http://{host}:{port}{endpoint}" self.pg.configure( [ From b3d3a2587d80bc541403dc47d9b0729ab6e61b5d Mon Sep 17 00:00:00 2001 From: duguorong009 <80258679+duguorong009@users.noreply.github.com> Date: Mon, 6 Nov 2023 04:40:03 -0500 Subject: [PATCH 05/63] feat: improve the serde impl for several types(`Lsn`, `TenantId`, `TimelineId` ...) (#5335) Improve the serde impl for several types (`Lsn`, `TenantId`, `TimelineId`) by making them sensitive to `Serializer::is_human_readadable` (true for json, false for bincode). Fixes #3511 by: - Implement the custom serde for `Lsn` - Implement the custom serde for `Id` - Add the helper module `serde_as_u64` in `libs/utils/src/lsn.rs` - Remove the unnecessary attr `#[serde_as(as = "DisplayFromStr")]` in all possible structs Additionally some safekeeper types gained serde tests. --------- Co-authored-by: Joonas Koivunen --- Cargo.lock | 11 + Cargo.toml | 1 + control_plane/src/attachment_service.rs | 3 - control_plane/src/endpoint.rs | 4 - control_plane/src/local_env.rs | 4 - libs/compute_api/src/spec.rs | 11 +- libs/pageserver_api/src/control_api.rs | 7 - libs/pageserver_api/src/models.rs | 47 +--- libs/safekeeper_api/src/models.rs | 12 - libs/utils/Cargo.toml | 1 + libs/utils/src/auth.rs | 3 - libs/utils/src/hex.rs | 41 +++ libs/utils/src/id.rs | 180 ++++++++++++- libs/utils/src/lib.rs | 4 + libs/utils/src/lsn.rs | 206 +++++++++++++- libs/utils/src/pageserver_feedback.rs | 5 - pageserver/src/consumption_metrics/metrics.rs | 4 - pageserver/src/consumption_metrics/upload.rs | 4 - pageserver/src/deletion_queue.rs | 3 - pageserver/src/http/routes.rs | 5 - pageserver/src/tenant/metadata.rs | 119 +++++++++ .../tenant/remote_timeline_client/index.rs | 3 - pageserver/src/tenant/size.rs | 13 - pageserver/src/tenant/timeline.rs | 3 - s3_scrubber/src/cloud_admin_api.rs | 21 +- safekeeper/src/control_file_upgrade.rs | 252 +++++++++++++++++- safekeeper/src/debug_dump.rs | 4 - safekeeper/src/http/routes.rs | 13 - safekeeper/src/json_ctrl.rs | 3 + safekeeper/src/pull_timeline.rs | 5 - safekeeper/src/safekeeper.rs | 114 +++++++- safekeeper/src/send_wal.rs | 3 - safekeeper/src/timeline.rs | 6 - 33 files changed, 930 insertions(+), 185 deletions(-) create mode 100644 libs/utils/src/hex.rs diff --git a/Cargo.lock b/Cargo.lock index 2203ad462c..a7e88688ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4681,6 +4681,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_assert" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda563240c1288b044209be1f0d38bb4d15044fb3e00dc354fbc922ab4733e80" +dependencies = [ + "hashbrown 0.13.2", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.183" @@ -5967,6 +5977,7 @@ dependencies = [ "routerify", "sentry", "serde", + "serde_assert", "serde_json", "serde_with", "signal-hook", diff --git a/Cargo.toml b/Cargo.toml index 6f538941f9..d992db4858 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ sentry = { version = "0.31", default-features = false, features = ["backtrace", serde = { version = "1.0", features = ["derive"] } serde_json = "1" serde_with = "2.0" +serde_assert = "0.5.0" sha2 = "0.10.2" signal-hook = "0.3" smallvec = "1.11" diff --git a/control_plane/src/attachment_service.rs b/control_plane/src/attachment_service.rs index ca632c5eb6..fcefe0e431 100644 --- a/control_plane/src/attachment_service.rs +++ b/control_plane/src/attachment_service.rs @@ -2,7 +2,6 @@ use crate::{background_process, local_env::LocalEnv}; use anyhow::anyhow; use camino::Utf8PathBuf; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::{path::PathBuf, process::Child}; use utils::id::{NodeId, TenantId}; @@ -14,10 +13,8 @@ pub struct AttachmentService { const COMMAND: &str = "attachment_service"; -#[serde_as] #[derive(Serialize, Deserialize)] pub struct AttachHookRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, pub node_id: Option, } diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index cb16f48829..4443fd8704 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -46,7 +46,6 @@ use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{NodeId, TenantId, TimelineId}; use crate::local_env::LocalEnv; @@ -57,13 +56,10 @@ use compute_api::responses::{ComputeState, ComputeStatus}; use compute_api::spec::{Cluster, ComputeMode, ComputeSpec}; // contents of a endpoint.json file -#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct EndpointConf { endpoint_id: String, - #[serde_as(as = "DisplayFromStr")] tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] timeline_id: TimelineId, mode: ComputeMode, pg_port: u16, diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 45a7469787..b9c8aeddcb 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -8,7 +8,6 @@ use anyhow::{bail, ensure, Context}; use postgres_backend::AuthType; use reqwest::Url; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::collections::HashMap; use std::env; use std::fs; @@ -33,7 +32,6 @@ pub const DEFAULT_PG_VERSION: u32 = 15; // to 'neon_local init --config=' option. See control_plane/simple.conf for // an example. // -#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct LocalEnv { // Base directory for all the nodes (the pageserver, safekeepers and @@ -59,7 +57,6 @@ pub struct LocalEnv { // Default tenant ID to use with the 'neon_local' command line utility, when // --tenant_id is not explicitly specified. #[serde(default)] - #[serde_as(as = "Option")] pub default_tenant_id: Option, // used to issue tokens during e.g pg start @@ -84,7 +81,6 @@ pub struct LocalEnv { // A `HashMap>` would be more appropriate here, // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table". - #[serde_as(as = "HashMap<_, Vec<(DisplayFromStr, DisplayFromStr)>>")] branch_name_mappings: HashMap>, } diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index c16deceebb..175b4461ac 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{TenantId, TimelineId}; use utils::lsn::Lsn; @@ -19,7 +18,6 @@ pub type PgIdent = String; /// Cluster spec or configuration represented as an optional number of /// delta operations + final cluster state description. -#[serde_as] #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ComputeSpec { pub format_version: f32, @@ -50,12 +48,12 @@ pub struct ComputeSpec { // these, and instead set the "neon.tenant_id", "neon.timeline_id", // etc. GUCs in cluster.settings. TODO: Once the control plane has been // updated to fill these fields, we can make these non optional. - #[serde_as(as = "Option")] pub tenant_id: Option, - #[serde_as(as = "Option")] + pub timeline_id: Option, - #[serde_as(as = "Option")] + pub pageserver_connstring: Option, + #[serde(default)] pub safekeeper_connstrings: Vec, @@ -140,14 +138,13 @@ impl RemoteExtSpec { } } -#[serde_as] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] pub enum ComputeMode { /// A read-write node #[default] Primary, /// A read-only node, pinned at a particular LSN - Static(#[serde_as(as = "DisplayFromStr")] Lsn), + Static(Lsn), /// A read-only node that follows the tip of the branch in hot standby mode /// /// Future versions may want to distinguish between replicas with hot standby diff --git a/libs/pageserver_api/src/control_api.rs b/libs/pageserver_api/src/control_api.rs index 3819111a5b..8232e81b98 100644 --- a/libs/pageserver_api/src/control_api.rs +++ b/libs/pageserver_api/src/control_api.rs @@ -4,7 +4,6 @@ //! See docs/rfcs/025-generation-numbers.md use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{NodeId, TenantId}; #[derive(Serialize, Deserialize)] @@ -12,10 +11,8 @@ pub struct ReAttachRequest { pub node_id: NodeId, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ReAttachResponseTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub gen: u32, } @@ -25,10 +22,8 @@ pub struct ReAttachResponse { pub tenants: Vec, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ValidateRequestTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub gen: u32, } @@ -43,10 +38,8 @@ pub struct ValidateResponse { pub tenants: Vec, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ValidateResponseTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub valid: bool, } diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index e667645c0a..cb99dc0a55 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -6,7 +6,7 @@ use std::{ use byteorder::{BigEndian, ReadBytesExt}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; +use serde_with::serde_as; use strum_macros; use utils::{ completion, @@ -174,25 +174,19 @@ pub enum TimelineState { Broken { reason: String, backtrace: String }, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub new_timeline_id: TimelineId, #[serde(default)] - #[serde_as(as = "Option")] pub ancestor_timeline_id: Option, #[serde(default)] - #[serde_as(as = "Option")] pub ancestor_start_lsn: Option, pub pg_version: Option, } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub new_tenant_id: TenantId, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] @@ -201,7 +195,6 @@ pub struct TenantCreateRequest { pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it } -#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantLoadRequest { @@ -278,31 +271,26 @@ pub struct LocationConfig { pub tenant_conf: TenantConfig, } -#[serde_as] #[derive(Serialize, Deserialize)] #[serde(transparent)] -pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub TenantId); +pub struct TenantCreateResponse(pub TenantId); #[derive(Serialize)] pub struct StatusResponse { pub id: NodeId, } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantLocationConfigRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, #[serde(flatten)] pub config: LocationConfig, // as we have a flattened field, we should reject all unknown fields in it } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantConfigRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, #[serde(flatten)] pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it @@ -374,10 +362,8 @@ pub enum TenantAttachmentStatus { Failed { reason: String }, } -#[serde_as] #[derive(Serialize, Deserialize, Clone)] pub struct TenantInfo { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, // NB: intentionally not part of OpenAPI, we don't want to commit to a specific set of TenantState's pub state: TenantState, @@ -388,33 +374,22 @@ pub struct TenantInfo { } /// This represents the output of the "timeline_detail" and "timeline_list" API calls. -#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimelineInfo { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, - #[serde_as(as = "Option")] pub ancestor_timeline_id: Option, - #[serde_as(as = "Option")] pub ancestor_lsn: Option, - #[serde_as(as = "DisplayFromStr")] pub last_record_lsn: Lsn, - #[serde_as(as = "Option")] pub prev_record_lsn: Option, - #[serde_as(as = "DisplayFromStr")] pub latest_gc_cutoff_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, /// The LSN that we have succesfully uploaded to remote storage - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, /// The LSN that we are advertizing to safekeepers - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn_visible: Lsn, pub current_logical_size: Option, // is None when timeline is Unloaded @@ -426,7 +401,6 @@ pub struct TimelineInfo { pub timeline_dir_layer_file_size_sum: Option, pub wal_source_connstr: Option, - #[serde_as(as = "Option")] pub last_received_msg_lsn: Option, /// the timestamp (in microseconds) of the last received message pub last_received_msg_ts: Option, @@ -523,23 +497,13 @@ pub struct LayerAccessStats { pub residence_events_history: HistoryBufferWithDropCounter, } -#[serde_as] #[derive(Debug, Clone, Serialize)] #[serde(tag = "kind")] pub enum InMemoryLayerInfo { - Open { - #[serde_as(as = "DisplayFromStr")] - lsn_start: Lsn, - }, - Frozen { - #[serde_as(as = "DisplayFromStr")] - lsn_start: Lsn, - #[serde_as(as = "DisplayFromStr")] - lsn_end: Lsn, - }, + Open { lsn_start: Lsn }, + Frozen { lsn_start: Lsn, lsn_end: Lsn }, } -#[serde_as] #[derive(Debug, Clone, Serialize)] #[serde(tag = "kind")] pub enum HistoricLayerInfo { @@ -547,9 +511,7 @@ pub enum HistoricLayerInfo { layer_file_name: String, layer_file_size: u64, - #[serde_as(as = "DisplayFromStr")] lsn_start: Lsn, - #[serde_as(as = "DisplayFromStr")] lsn_end: Lsn, remote: bool, access_stats: LayerAccessStats, @@ -558,7 +520,6 @@ pub enum HistoricLayerInfo { layer_file_name: String, layer_file_size: u64, - #[serde_as(as = "DisplayFromStr")] lsn_start: Lsn, remote: bool, access_stats: LayerAccessStats, diff --git a/libs/safekeeper_api/src/models.rs b/libs/safekeeper_api/src/models.rs index eaea266f32..786712deb1 100644 --- a/libs/safekeeper_api/src/models.rs +++ b/libs/safekeeper_api/src/models.rs @@ -1,23 +1,18 @@ use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::{ id::{NodeId, TenantId, TimelineId}, lsn::Lsn, }; -#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub peer_ids: Option>, pub pg_version: u32, pub system_id: Option, pub wal_seg_size: Option, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, // If not passed, it is assigned to the beginning of commit_lsn segment. pub local_start_lsn: Option, @@ -28,7 +23,6 @@ fn lsn_invalid() -> Lsn { } /// Data about safekeeper's timeline, mirrors broker.proto. -#[serde_as] #[derive(Debug, Clone, Deserialize, Serialize)] pub struct SkTimelineInfo { /// Term. @@ -36,25 +30,19 @@ pub struct SkTimelineInfo { /// Term of the last entry. pub last_log_term: Option, /// LSN of the last record. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub flush_lsn: Lsn, /// Up to which LSN safekeeper regards its WAL as committed. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub commit_lsn: Lsn, /// LSN up to which safekeeper has backed WAL. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub backup_lsn: Lsn, /// LSN of last checkpoint uploaded by pageserver. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub remote_consistent_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub peer_horizon_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub local_start_lsn: Lsn, /// A connection string to use for WAL receiving. diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 27df0265b4..3f4ef2abeb 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -55,6 +55,7 @@ bytes.workspace = true criterion.workspace = true hex-literal.workspace = true camino-tempfile.workspace = true +serde_assert.workspace = true [[bench]] name = "benchmarks" diff --git a/libs/utils/src/auth.rs b/libs/utils/src/auth.rs index 54b90fa070..37299e4e7f 100644 --- a/libs/utils/src/auth.rs +++ b/libs/utils/src/auth.rs @@ -9,7 +9,6 @@ use jsonwebtoken::{ decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, }; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use crate::id::TenantId; @@ -32,11 +31,9 @@ pub enum Scope { } /// JWT payload. See docs/authentication.md for the format -#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct Claims { #[serde(default)] - #[serde_as(as = "Option")] pub tenant_id: Option, pub scope: Scope, } diff --git a/libs/utils/src/hex.rs b/libs/utils/src/hex.rs new file mode 100644 index 0000000000..fc0bb7e4a2 --- /dev/null +++ b/libs/utils/src/hex.rs @@ -0,0 +1,41 @@ +/// Useful type for asserting that expected bytes match reporting the bytes more readable +/// array-syntax compatible hex bytes. +/// +/// # Usage +/// +/// ``` +/// use utils::Hex; +/// +/// let actual = serialize_something(); +/// let expected = [0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64]; +/// +/// // the type implements PartialEq and on mismatch, both sides are printed in 16 wide multiline +/// // output suffixed with an array style length for easier comparisons. +/// assert_eq!(Hex(&actual), Hex(&expected)); +/// +/// // with `let expected = [0x68];` the error would had been: +/// // assertion `left == right` failed +/// // left: [0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64; 11] +/// // right: [0x68; 1] +/// # fn serialize_something() -> Vec { "hello world".as_bytes().to_vec() } +/// ``` +#[derive(PartialEq)] +pub struct Hex<'a>(pub &'a [u8]); + +impl std::fmt::Debug for Hex<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for (i, c) in self.0.chunks(16).enumerate() { + if i > 0 && !c.is_empty() { + writeln!(f, ", ")?; + } + for (j, b) in c.iter().enumerate() { + if j > 0 { + write!(f, ", ")?; + } + write!(f, "0x{b:02x}")?; + } + } + write!(f, "; {}]", self.0.len()) + } +} diff --git a/libs/utils/src/id.rs b/libs/utils/src/id.rs index ec13c2f96f..eef6a358e6 100644 --- a/libs/utils/src/id.rs +++ b/libs/utils/src/id.rs @@ -3,6 +3,7 @@ use std::{fmt, str::FromStr}; use anyhow::Context; use hex::FromHex; use rand::Rng; +use serde::de::Visitor; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -17,12 +18,74 @@ pub enum IdError { /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// -/// Use `#[serde_as(as = "DisplayFromStr")]` to (de)serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. -/// Check the `serde_with::serde_as` documentation for options for more complex types. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] struct Id([u8; 16]); +impl Serialize for Id { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + self.0.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for Id { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> Visitor<'de> for IdVisitor { + type Value = Id; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 16])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 16] = Deserialize::deserialize(s)?; + Ok(Id::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Id::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 16, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} + impl Id { pub fn get_from_buf(buf: &mut impl bytes::Buf) -> Id { let mut arr = [0u8; 16]; @@ -308,3 +371,112 @@ impl fmt::Display for NodeId { write!(f, "{}", self.0) } } + +#[cfg(test)] +mod tests { + use serde_assert::{Deserializer, Serializer, Token, Tokens}; + + use crate::bin_ser::BeSer; + + use super::*; + + #[test] + fn test_id_serde_non_human_readable() { + let original_id = Id([ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]); + let expected_tokens = Tokens(vec![ + Token::Tuple { len: 16 }, + Token::U8(173), + Token::U8(80), + Token::U8(132), + Token::U8(115), + Token::U8(129), + Token::U8(226), + Token::U8(72), + Token::U8(254), + Token::U8(170), + Token::U8(201), + Token::U8(135), + Token::U8(108), + Token::U8(199), + Token::U8(26), + Token::U8(228), + Token::U8(24), + Token::TupleEnd, + ]); + + let serializer = Serializer::builder().is_human_readable(false).build(); + let serialized_tokens = original_id.serialize(&serializer).unwrap(); + assert_eq!(serialized_tokens, expected_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(serialized_tokens) + .build(); + let deserialized_id = Id::deserialize(&mut deserializer).unwrap(); + assert_eq!(deserialized_id, original_id); + } + + #[test] + fn test_id_serde_human_readable() { + let original_id = Id([ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]); + let expected_tokens = Tokens(vec![Token::Str(String::from( + "ad50847381e248feaac9876cc71ae418", + ))]); + + let serializer = Serializer::builder().is_human_readable(true).build(); + let serialized_tokens = original_id.serialize(&serializer).unwrap(); + assert_eq!(serialized_tokens, expected_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(Tokens(vec![Token::Str(String::from( + "ad50847381e248feaac9876cc71ae418", + ))])) + .build(); + assert_eq!(Id::deserialize(&mut deserializer).unwrap(), original_id); + } + + macro_rules! roundtrip_type { + ($type:ty, $expected_bytes:expr) => {{ + let expected_bytes: [u8; 16] = $expected_bytes; + let original_id = <$type>::from(expected_bytes); + + let ser_bytes = original_id.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_id = <$type>::des(&ser_bytes).unwrap(); + assert_eq!(des_id, original_id); + }}; + } + + #[test] + fn test_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(Id, expected_bytes); + } + + #[test] + fn test_tenant_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(TenantId, expected_bytes); + } + + #[test] + fn test_timeline_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(TimelineId, expected_bytes); + } +} diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 88cefd516d..1e34034023 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -24,6 +24,10 @@ pub mod auth; // utility functions and helper traits for unified unique id generation/serialization etc. pub mod id; + +mod hex; +pub use hex::Hex; + // http endpoint utils pub mod http; diff --git a/libs/utils/src/lsn.rs b/libs/utils/src/lsn.rs index 7d9baf7d49..262dcb8a8a 100644 --- a/libs/utils/src/lsn.rs +++ b/libs/utils/src/lsn.rs @@ -1,7 +1,7 @@ #![warn(missing_docs)] use camino::Utf8Path; -use serde::{Deserialize, Serialize}; +use serde::{de::Visitor, Deserialize, Serialize}; use std::fmt; use std::ops::{Add, AddAssign}; use std::str::FromStr; @@ -13,10 +13,114 @@ use crate::seqwait::MonotonicCounter; pub const XLOG_BLCKSZ: u32 = 8192; /// A Postgres LSN (Log Sequence Number), also known as an XLogRecPtr -#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, Serialize, Deserialize)] -#[serde(transparent)] +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash)] pub struct Lsn(pub u64); +impl Serialize for Lsn { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + self.0.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for Lsn { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct LsnVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> Visitor<'de> for LsnVisitor { + type Value = Lsn; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str( + "value in form of hex string({upper_u32_hex}/{lower_u32_hex}) representing u64 integer", + ) + } else { + formatter.write_str("value in form of integer(u64)") + } + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + Ok(Lsn(v)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Lsn::from_str(v).map_err(|e| E::custom(e)) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(LsnVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_u64(LsnVisitor { + is_human_readable_deserializer: false, + }) + } + } +} + +/// Allows (de)serialization of an `Lsn` always as `u64`. +/// +/// ### Example +/// +/// ```rust +/// # use serde::{Serialize, Deserialize}; +/// use utils::lsn::Lsn; +/// +/// #[derive(PartialEq, Serialize, Deserialize, Debug)] +/// struct Foo { +/// #[serde(with = "utils::lsn::serde_as_u64")] +/// always_u64: Lsn, +/// } +/// +/// let orig = Foo { always_u64: Lsn(1234) }; +/// +/// let res = serde_json::to_string(&orig).unwrap(); +/// assert_eq!(res, r#"{"always_u64":1234}"#); +/// +/// let foo = serde_json::from_str::(&res).unwrap(); +/// assert_eq!(foo, orig); +/// ``` +/// +pub mod serde_as_u64 { + use super::Lsn; + + /// Serializes the Lsn as u64 disregarding the human readability of the format. + /// + /// Meant to be used via `#[serde(with = "...")]` or `#[serde(serialize_with = "...")]`. + pub fn serialize(lsn: &Lsn, serializer: S) -> Result { + use serde::Serialize; + lsn.0.serialize(serializer) + } + + /// Deserializes the Lsn as u64 disregarding the human readability of the format. + /// + /// Meant to be used via `#[serde(with = "...")]` or `#[serde(deserialize_with = "...")]`. + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + use serde::Deserialize; + u64::deserialize(deserializer).map(Lsn) + } +} + /// We tried to parse an LSN from a string, but failed #[derive(Debug, PartialEq, Eq, thiserror::Error)] #[error("LsnParseError")] @@ -264,8 +368,13 @@ impl MonotonicCounter for RecordLsn { #[cfg(test)] mod tests { + use crate::bin_ser::BeSer; + use super::*; + use serde::ser::Serialize; + use serde_assert::{Deserializer, Serializer, Token, Tokens}; + #[test] fn test_lsn_strings() { assert_eq!("12345678/AAAA5555".parse(), Ok(Lsn(0x12345678AAAA5555))); @@ -341,4 +450,95 @@ mod tests { assert_eq!(lsn.fetch_max(Lsn(6000)), Lsn(5678)); assert_eq!(lsn.fetch_max(Lsn(5000)), Lsn(6000)); } + + #[test] + fn test_lsn_serde() { + let original_lsn = Lsn(0x0123456789abcdef); + let expected_readable_tokens = Tokens(vec![Token::U64(0x0123456789abcdef)]); + let expected_non_readable_tokens = + Tokens(vec![Token::Str(String::from("1234567/89ABCDEF"))]); + + // Testing human_readable ser/de + let serializer = Serializer::builder().is_human_readable(false).build(); + let readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + assert_eq!(readable_ser_tokens, expected_readable_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(readable_ser_tokens) + .build(); + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + + // Testing NON human_readable ser/de + let serializer = Serializer::builder().is_human_readable(true).build(); + let non_readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + assert_eq!(non_readable_ser_tokens, expected_non_readable_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(non_readable_ser_tokens) + .build(); + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + + // Testing mismatching ser/de + let serializer = Serializer::builder().is_human_readable(false).build(); + let non_readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(non_readable_ser_tokens) + .build(); + Lsn::deserialize(&mut deserializer).unwrap_err(); + + let serializer = Serializer::builder().is_human_readable(true).build(); + let readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(readable_ser_tokens) + .build(); + Lsn::deserialize(&mut deserializer).unwrap_err(); + } + + #[test] + fn test_lsn_ensure_roundtrip() { + let original_lsn = Lsn(0xaaaabbbb); + + let serializer = Serializer::builder().is_human_readable(false).build(); + let ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(ser_tokens) + .build(); + + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + } + + #[test] + fn test_lsn_bincode_serde() { + let lsn = Lsn(0x0123456789abcdef); + let expected_bytes = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + + let ser_bytes = lsn.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_lsn = Lsn::des(&ser_bytes).unwrap(); + assert_eq!(des_lsn, lsn); + } + + #[test] + fn test_lsn_bincode_ensure_roundtrip() { + let original_lsn = Lsn(0x01_02_03_04_05_06_07_08); + let expected_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + + let ser_bytes = original_lsn.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_lsn = Lsn::des(&ser_bytes).unwrap(); + assert_eq!(des_lsn, original_lsn); + } } diff --git a/libs/utils/src/pageserver_feedback.rs b/libs/utils/src/pageserver_feedback.rs index a3b53201d3..c9fbdde928 100644 --- a/libs/utils/src/pageserver_feedback.rs +++ b/libs/utils/src/pageserver_feedback.rs @@ -3,7 +3,6 @@ use std::time::{Duration, SystemTime}; use bytes::{Buf, BufMut, Bytes, BytesMut}; use pq_proto::{read_cstr, PG_EPOCH}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use tracing::{trace, warn}; use crate::lsn::Lsn; @@ -15,21 +14,17 @@ use crate::lsn::Lsn; /// /// serde Serialize is used only for human readable dump to json (e.g. in /// safekeepers debug_dump). -#[serde_as] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct PageserverFeedback { /// Last known size of the timeline. Used to enforce timeline size limit. pub current_timeline_size: u64, /// LSN last received and ingested by the pageserver. Controls backpressure. - #[serde_as(as = "DisplayFromStr")] pub last_received_lsn: Lsn, /// LSN up to which data is persisted by the pageserver to its local disc. /// Controls backpressure. - #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, /// LSN up to which data is persisted by the pageserver on s3; safekeepers /// consider WAL before it can be removed. - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, // Serialize with RFC3339 format. #[serde(with = "serde_systemtime")] diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index 652dd98683..c22f218976 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -3,7 +3,6 @@ use anyhow::Context; use chrono::{DateTime, Utc}; use consumption_metrics::EventType; use futures::stream::StreamExt; -use serde_with::serde_as; use std::{sync::Arc, time::SystemTime}; use utils::{ id::{TenantId, TimelineId}, @@ -42,13 +41,10 @@ pub(super) enum Name { /// /// This is a denormalization done at the MetricsKey const methods; these should not be constructed /// elsewhere. -#[serde_with::serde_as] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub(crate) struct MetricsKey { - #[serde_as(as = "serde_with::DisplayFromStr")] pub(super) tenant_id: TenantId, - #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] pub(super) timeline_id: Option, diff --git a/pageserver/src/consumption_metrics/upload.rs b/pageserver/src/consumption_metrics/upload.rs index d69d43a2a8..322ed95cc8 100644 --- a/pageserver/src/consumption_metrics/upload.rs +++ b/pageserver/src/consumption_metrics/upload.rs @@ -1,5 +1,4 @@ use consumption_metrics::{Event, EventChunk, IdempotencyKey, CHUNK_SIZE}; -use serde_with::serde_as; use tokio_util::sync::CancellationToken; use tracing::Instrument; @@ -7,12 +6,9 @@ use super::{metrics::Name, Cache, MetricsKey, RawMetric}; use utils::id::{TenantId, TimelineId}; /// How the metrics from pageserver are identified. -#[serde_with::serde_as] #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq)] struct Ids { - #[serde_as(as = "serde_with::DisplayFromStr")] pub(super) tenant_id: TenantId, - #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] pub(super) timeline_id: Option, } diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 7b2db929fa..b6f889c682 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -18,7 +18,6 @@ use hex::FromHex; use remote_storage::{GenericRemoteStorage, RemotePath}; use serde::Deserialize; use serde::Serialize; -use serde_with::serde_as; use thiserror::Error; use tokio; use tokio_util::sync::CancellationToken; @@ -215,7 +214,6 @@ where /// during recovery as startup. const TEMP_SUFFIX: &str = "tmp"; -#[serde_as] #[derive(Debug, Serialize, Deserialize)] struct DeletionList { /// Serialization version, for future use @@ -244,7 +242,6 @@ struct DeletionList { validated: bool, } -#[serde_as] #[derive(Debug, Serialize, Deserialize)] struct DeletionHeader { /// Serialization version, for future use diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 2c46d733d6..a6fb26b298 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -17,7 +17,6 @@ use pageserver_api::models::{ TenantLoadRequest, TenantLocationConfigRequest, }; use remote_storage::GenericRemoteStorage; -use serde_with::{serde_as, DisplayFromStr}; use tenant_size_model::{SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; use tracing::*; @@ -499,10 +498,8 @@ async fn get_lsn_by_timestamp_handler( let result = timeline.find_lsn_for_timestamp(timestamp_pg, &ctx).await?; if version.unwrap_or(0) > 1 { - #[serde_as] #[derive(serde::Serialize)] struct Result { - #[serde_as(as = "DisplayFromStr")] lsn: Lsn, kind: &'static str, } @@ -811,10 +808,8 @@ async fn tenant_size_handler( } /// The type resides in the pageserver not to expose `ModelInputs`. - #[serde_with::serde_as] #[derive(serde::Serialize)] struct TenantHistorySize { - #[serde_as(as = "serde_with::DisplayFromStr")] id: TenantId, /// Size is a mixture of WAL and logical size, so the unit is bytes. /// diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 75ffe09696..38fd426746 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -406,4 +406,123 @@ mod tests { METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION ); } + + #[test] + fn test_metadata_bincode_serde() { + let original_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + // Any version will do here, so use the default + crate::DEFAULT_PG_VERSION, + ); + let metadata_bytes = original_metadata + .to_bytes() + .expect("Cannot create bytes array from metadata"); + + let metadata_bincode_be_bytes = original_metadata + .ser() + .expect("Cannot serialize the metadata"); + + // 8 bytes for the length of the vector + assert_eq!(metadata_bincode_be_bytes.len(), 8 + metadata_bytes.len()); + + let expected_bincode_bytes = { + let mut temp = vec![]; + let len_bytes = metadata_bytes.len().to_be_bytes(); + temp.extend_from_slice(&len_bytes); + temp.extend_from_slice(&metadata_bytes); + temp + }; + assert_eq!(metadata_bincode_be_bytes, expected_bincode_bytes); + + let deserialized_metadata = TimelineMetadata::des(&metadata_bincode_be_bytes).unwrap(); + // Deserialized metadata has the metadata header, which is different from the serialized one. + // Reference: TimelineMetaData::to_bytes() + let expected_metadata = { + let mut temp_metadata = original_metadata; + let body_bytes = temp_metadata + .body + .ser() + .expect("Cannot serialize the metadata body"); + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_FORMAT_VERSION, + checksum: crc32c::crc32c(&body_bytes), + }; + temp_metadata.hdr = hdr; + temp_metadata + }; + assert_eq!(deserialized_metadata, expected_metadata); + } + + #[test] + fn test_metadata_bincode_serde_ensure_roundtrip() { + let original_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + // Any version will do here, so use the default + crate::DEFAULT_PG_VERSION, + ); + let expected_bytes = vec![ + /* bincode length encoding bytes */ + 0, 0, 0, 0, 0, 0, 2, 0, // 8 bytes for the length of the serialized vector + /* TimelineMetadataHeader */ + 4, 37, 101, 34, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2) + /* TimelineMetadataBodyV2 */ + 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes) + 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes) + 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119, + 136, // ancestor_timeline (17 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes) + 0, 0, 0, 15, // pg_version (4 bytes) + /* padding bytes */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + ]; + let metadata_ser_bytes = original_metadata.ser().unwrap(); + assert_eq!(metadata_ser_bytes, expected_bytes); + + let expected_metadata = { + let mut temp_metadata = original_metadata; + let body_bytes = temp_metadata + .body + .ser() + .expect("Cannot serialize the metadata body"); + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_FORMAT_VERSION, + checksum: crc32c::crc32c(&body_bytes), + }; + temp_metadata.hdr = hdr; + temp_metadata + }; + let des_metadata = TimelineMetadata::des(&metadata_ser_bytes).unwrap(); + assert_eq!(des_metadata, expected_metadata); + } } diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index fdab74a8be..fa0679c7a2 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::bin_ser::SerializeError; use crate::tenant::metadata::TimelineMetadata; @@ -58,7 +57,6 @@ impl LayerFileMetadata { /// /// This type needs to be backwards and forwards compatible. When changing the fields, /// remember to add a test case for the changed version. -#[serde_as] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct IndexPart { /// Debugging aid describing the version of this type. @@ -78,7 +76,6 @@ pub struct IndexPart { // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata. // It's duplicated for convenience when reading the serialized structure, but is // private because internally we would read from metadata instead. - #[serde_as(as = "DisplayFromStr")] disk_consistent_lsn: Lsn, #[serde(rename = "metadata_bytes")] diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index e737d3f59c..5e8ee2b99e 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -29,7 +29,6 @@ use tenant_size_model::{Segment, StorageModel}; /// needs. We will convert this into a StorageModel when it's time to perform /// the calculation. /// -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct ModelInputs { pub segments: Vec, @@ -37,11 +36,9 @@ pub struct ModelInputs { } /// A [`Segment`], with some extra information for display purposes -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct SegmentMeta { pub segment: Segment, - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, pub kind: LsnKind, } @@ -77,32 +74,22 @@ pub enum LsnKind { /// Collect all relevant LSNs to the inputs. These will only be helpful in the serialized form as /// part of [`ModelInputs`] from the HTTP api, explaining the inputs. -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct TimelineInputs { - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, - #[serde_as(as = "Option")] pub ancestor_id: Option, - #[serde_as(as = "serde_with::DisplayFromStr")] ancestor_lsn: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] last_record: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] latest_gc_cutoff: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] horizon_cutoff: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] pitr_cutoff: Lsn, /// Cutoff point based on GC settings - #[serde_as(as = "serde_with::DisplayFromStr")] next_gc_cutoff: Lsn, /// Cutoff point calculated from the user-supplied 'max_retention_period' - #[serde_as(as = "Option")] retention_param_cutoff: Option, } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 4608683c4a..baa97b6cf9 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -2936,13 +2936,10 @@ struct CompactLevel0Phase1StatsBuilder { new_deltas_size: Option, } -#[serde_as] #[derive(serde::Serialize)] struct CompactLevel0Phase1Stats { version: u64, - #[serde_as(as = "serde_with::DisplayFromStr")] tenant_id: TenantId, - #[serde_as(as = "serde_with::DisplayFromStr")] timeline_id: TimelineId, read_lock_acquisition_micros: RecordedDuration, read_lock_held_spawn_blocking_startup_micros: RecordedDuration, diff --git a/s3_scrubber/src/cloud_admin_api.rs b/s3_scrubber/src/cloud_admin_api.rs index 1b28dc4dff..151421c84f 100644 --- a/s3_scrubber/src/cloud_admin_api.rs +++ b/s3_scrubber/src/cloud_admin_api.rs @@ -5,6 +5,7 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use hex::FromHex; +use pageserver::tenant::Tenant; use reqwest::{header, Client, StatusCode, Url}; use serde::Deserialize; use tokio::sync::Semaphore; @@ -118,13 +119,18 @@ fn from_nullable_id<'de, D>(deserializer: D) -> Result where D: serde::de::Deserializer<'de>, { - let id_str = String::deserialize(deserializer)?; - if id_str.is_empty() { - // This is a bogus value, but for the purposes of the scrubber all that - // matters is that it doesn't collide with any real IDs. - Ok(TenantId::from([0u8; 16])) + if deserializer.is_human_readable() { + let id_str = String::deserialize(deserializer)?; + if id_str.is_empty() { + // This is a bogus value, but for the purposes of the scrubber all that + // matters is that it doesn't collide with any real IDs. + Ok(TenantId::from([0u8; 16])) + } else { + TenantId::from_hex(&id_str).map_err(|e| serde::de::Error::custom(format!("{e}"))) + } } else { - TenantId::from_hex(&id_str).map_err(|e| serde::de::Error::custom(format!("{e}"))) + let id_arr = <[u8; 16]>::deserialize(deserializer)?; + Ok(TenantId::from(id_arr)) } } @@ -153,7 +159,6 @@ pub struct ProjectData { pub maintenance_set: Option, } -#[serde_with::serde_as] #[derive(Debug, serde::Deserialize)] pub struct BranchData { pub id: BranchId, @@ -161,12 +166,10 @@ pub struct BranchData { pub updated_at: DateTime, pub name: String, pub project_id: ProjectId, - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, #[serde(default)] pub parent_id: Option, #[serde(default)] - #[serde_as(as = "Option")] pub parent_lsn: Option, pub default: bool, pub deleted: bool, diff --git a/safekeeper/src/control_file_upgrade.rs b/safekeeper/src/control_file_upgrade.rs index a7f17df797..a0be2b2054 100644 --- a/safekeeper/src/control_file_upgrade.rs +++ b/safekeeper/src/control_file_upgrade.rs @@ -13,7 +13,7 @@ use utils::{ }; /// Persistent consensus state of the acceptor. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct AcceptorStateV1 { /// acceptor's last term it voted for (advanced in 1 phase) term: Term, @@ -21,7 +21,7 @@ struct AcceptorStateV1 { epoch: Term, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct SafeKeeperStateV1 { /// persistent acceptor state acceptor_state: AcceptorStateV1, @@ -50,7 +50,7 @@ pub struct ServerInfoV2 { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV2 { /// persistent acceptor state pub acceptor_state: AcceptorState, @@ -81,7 +81,7 @@ pub struct ServerInfoV3 { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV3 { /// persistent acceptor state pub acceptor_state: AcceptorState, @@ -101,7 +101,7 @@ pub struct SafeKeeperStateV3 { pub wal_start_lsn: Lsn, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV4 { #[serde(with = "hex")] pub tenant_id: TenantId, @@ -264,3 +264,245 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result } bail!("unsupported safekeeper control file version {}", version) } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use utils::{id::NodeId, Hex}; + + use crate::safekeeper::PersistedPeerInfo; + + use super::*; + + #[test] + fn roundtrip_v1() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV1 { + acceptor_state: AcceptorStateV1 { + term: 42, + epoch: 43, + }, + server: ServerInfoV2 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + #[rustfmt::skip] + let expected = [ + // term + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // epoch + 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // pg_version + 0x0e, 0x00, 0x00, 0x00, + // system_id + 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, + // tenant_id + 0xcf, 0x04, 0x80, 0x92, 0x97, 0x07, 0xee, 0x75, 0x37, 0x23, 0x37, 0xef, 0xaa, 0x5e, 0xcf, 0x96, + // timeline_id + 0x11, 0x2d, 0xed, 0x66, 0x42, 0x2a, 0xa5, 0xe9, 0x53, 0xe5, 0x44, 0x0f, 0xa5, 0x42, 0x7a, 0xc4, + // wal_seg_size + 0x78, 0x56, 0x34, 0x12, + // proposer_uuid + 0xc4, 0x7a, 0x42, 0xa5, 0x0f, 0x44, 0xe5, 0x53, 0xe9, 0xa5, 0x2a, 0x42, 0x66, 0xed, 0x2d, 0x11, + // commit_lsn + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + // truncate_lsn + 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, 0x00, 0x00, + // wal_start_lsn + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV1::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v2() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV2 { + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfoV2 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, + 0x34, 0x12, 0xcf, 0x04, 0x80, 0x92, 0x97, 0x07, 0xee, 0x75, 0x37, 0x23, 0x37, 0xef, + 0xaa, 0x5e, 0xcf, 0x96, 0x11, 0x2d, 0xed, 0x66, 0x42, 0x2a, 0xa5, 0xe9, 0x53, 0xe5, + 0x44, 0x0f, 0xa5, 0x42, 0x7a, 0xc4, 0x78, 0x56, 0x34, 0x12, 0xc4, 0x7a, 0x42, 0xa5, + 0x0f, 0x44, 0xe5, 0x53, 0xe9, 0xa5, 0x2a, 0x42, 0x66, 0xed, 0x2d, 0x11, 0x78, 0x02, + 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV2::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v3() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV3 { + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfoV3 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x66, 0x30, 0x34, + 0x38, 0x30, 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, + 0x32, 0x33, 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, + 0x36, 0x36, 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, + 0x34, 0x34, 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x34, 0x37, 0x61, + 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, + 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV3::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v4() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV4 { + tenant_id, + timeline_id, + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfo { + pg_version: 14, + system_id: 0x1234567887654321, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + peers: PersistedPeers(vec![( + NodeId(1), + PersistedPeerInfo { + backup_lsn: Lsn(1234567000), + term: 42, + flush_lsn: Lsn(1234567800 - 8), + commit_lsn: Lsn(1234567600), + }, + )]), + commit_lsn: Lsn(1234567800), + s3_wal_lsn: Lsn(1234567300), + peer_horizon_lsn: Lsn(9999999), + remote_consistent_lsn: Lsn(1234560000), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x66, 0x30, 0x34, 0x38, 0x30, + 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, 0x32, 0x33, + 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, 0x20, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, 0x36, 0x36, + 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, 0x34, 0x34, + 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, 0x2a, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x34, 0x37, 0x61, + 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, + 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x84, 0x00, 0x96, 0x49, 0x00, 0x00, + 0x00, 0x00, 0x7f, 0x96, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x95, 0x49, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0xff, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, + 0x00, 0x00, 0xb0, 0x01, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV4::des(&ser).unwrap(); + + assert_eq!(state, deser); + } +} diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index 8cbc0aa47f..daf9255ecb 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -14,7 +14,6 @@ use postgres_ffi::XLogSegNo; use serde::Deserialize; use serde::Serialize; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::NodeId; use utils::id::TenantTimelineId; use utils::id::{TenantId, TimelineId}; @@ -136,12 +135,9 @@ pub struct Config { pub wal_backup_enabled: bool, } -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct Timeline { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub control_file: Option, pub memory: Option, diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 474d636441..06b903719e 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -4,7 +4,6 @@ use once_cell::sync::Lazy; use postgres_ffi::WAL_SEGMENT_SIZE; use safekeeper_api::models::SkTimelineInfo; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::collections::{HashMap, HashSet}; use std::fmt; use std::str::FromStr; @@ -67,11 +66,9 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { /// Same as TermLsn, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. -#[serde_as] #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct TermSwitchApiEntry { pub term: Term, - #[serde_as(as = "DisplayFromStr")] pub lsn: Lsn, } @@ -93,28 +90,18 @@ pub struct AcceptorStateStatus { } /// Info about timeline on safekeeper ready for reporting. -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct TimelineStatus { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub acceptor_state: AcceptorStateStatus, pub pg_info: ServerInfo, - #[serde_as(as = "DisplayFromStr")] pub flush_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub timeline_start_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub local_start_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub backup_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub peer_horizon_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, pub peers: Vec, pub walsenders: Vec, diff --git a/safekeeper/src/json_ctrl.rs b/safekeeper/src/json_ctrl.rs index 2ad2ca7706..303bdd67fe 100644 --- a/safekeeper/src/json_ctrl.rs +++ b/safekeeper/src/json_ctrl.rs @@ -44,8 +44,11 @@ pub struct AppendLogicalMessage { // fields from AppendRequestHeader pub term: Term, + #[serde(with = "utils::lsn::serde_as_u64")] pub epoch_start_lsn: Lsn, + #[serde(with = "utils::lsn::serde_as_u64")] pub begin_lsn: Lsn, + #[serde(with = "utils::lsn::serde_as_u64")] pub truncate_lsn: Lsn, pub pg_version: u32, } diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index e2f1b9fcff..ad3a18a536 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -6,8 +6,6 @@ use tokio::io::AsyncWriteExt; use tracing::info; use utils::id::{TenantId, TenantTimelineId, TimelineId}; -use serde_with::{serde_as, DisplayFromStr}; - use crate::{ control_file, debug_dump, http::routes::TimelineStatus, @@ -16,12 +14,9 @@ use crate::{ }; /// Info about timeline on safekeeper ready for reporting. -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct Request { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub http_hosts: Vec, } diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 1437bdb50e..47a624281d 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -52,7 +52,7 @@ impl From<(Term, Lsn)> for TermLsn { } } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq)] pub struct TermHistory(pub Vec); impl TermHistory { @@ -178,7 +178,7 @@ impl fmt::Debug for TermHistory { pub type PgUuid = [u8; 16]; /// Persistent consensus state of the acceptor. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct AcceptorState { /// acceptor's last term it voted for (advanced in 1 phase) pub term: Term, @@ -209,16 +209,16 @@ pub struct ServerInfo { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct PersistedPeerInfo { /// LSN up to which safekeeper offloaded WAL to s3. - backup_lsn: Lsn, + pub backup_lsn: Lsn, /// Term of the last entry. - term: Term, + pub term: Term, /// LSN of the last record. - flush_lsn: Lsn, + pub flush_lsn: Lsn, /// Up to which LSN safekeeper regards its WAL as committed. - commit_lsn: Lsn, + pub commit_lsn: Lsn, } impl PersistedPeerInfo { @@ -232,12 +232,12 @@ impl PersistedPeerInfo { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct PersistedPeers(pub Vec<(NodeId, PersistedPeerInfo)>); /// Persistent information stored on safekeeper node /// On disk data is prefixed by magic and format version and followed by checksum. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperState { #[serde(with = "hex")] pub tenant_id: TenantId, @@ -1096,7 +1096,7 @@ mod tests { use super::*; use crate::wal_storage::Storage; - use std::{ops::Deref, time::Instant}; + use std::{ops::Deref, str::FromStr, time::Instant}; // fake storage for tests struct InMemoryState { @@ -1314,4 +1314,98 @@ mod tests { }) ); } + + #[test] + fn test_sk_state_bincode_serde_roundtrip() { + use utils::Hex; + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperState { + tenant_id, + timeline_id, + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfo { + pg_version: 14, + system_id: 0x1234567887654321, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + timeline_start_lsn: Lsn(0x12345600), + local_start_lsn: Lsn(0x12), + commit_lsn: Lsn(1234567800), + backup_lsn: Lsn(1234567300), + peer_horizon_lsn: Lsn(9999999), + remote_consistent_lsn: Lsn(1234560000), + peers: PersistedPeers(vec![( + NodeId(1), + PersistedPeerInfo { + backup_lsn: Lsn(1234567000), + term: 42, + flush_lsn: Lsn(1234567800 - 8), + commit_lsn: Lsn(1234567600), + }, + )]), + }; + + let ser = state.ser().unwrap(); + + #[rustfmt::skip] + let expected = [ + // tenant_id as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x66, 0x30, 0x34, 0x38, 0x30, 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, 0x32, 0x33, 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, + // timeline_id as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, 0x36, 0x36, 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, 0x34, 0x34, 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, + // term + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // length prefix + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // unsure why this order is swapped + 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // pg_version + 0x0e, 0x00, 0x00, 0x00, + // systemid + 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, + // wal_seg_size + 0x78, 0x56, 0x34, 0x12, + // pguuid as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x34, 0x37, 0x61, 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + + // timeline_start_lsn + 0x00, 0x56, 0x34, 0x12, 0x00, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x00, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0x96, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe4, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + // length prefix for persistentpeers + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // nodeid + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // backuplsn + 0x58, 0xff, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0xb0, 0x01, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperState::des(&ser).unwrap(); + + assert_eq!(deser, state); + } } diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index cd765dcbce..44f14f8c7e 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -16,7 +16,6 @@ use postgres_ffi::get_current_timestamp; use postgres_ffi::{TimestampTz, MAX_SEND_SIZE}; use pq_proto::{BeMessage, WalSndKeepAlive, XLogDataBody}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use tokio::io::{AsyncRead, AsyncWrite}; use utils::id::TenantTimelineId; use utils::lsn::AtomicLsn; @@ -313,10 +312,8 @@ impl WalSendersShared { } // Serialized is used only for pretty printing in json. -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WalSenderState { - #[serde_as(as = "DisplayFromStr")] ttid: TenantTimelineId, addr: SocketAddr, conn_id: ConnectionId, diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 96d844fa7a..2ba871207e 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -5,10 +5,8 @@ use anyhow::{anyhow, bail, Result}; use camino::Utf8PathBuf; use postgres_ffi::XLogSegNo; use serde::{Deserialize, Serialize}; -use serde_with::serde_as; use tokio::fs; -use serde_with::DisplayFromStr; use std::cmp::max; use std::sync::Arc; use std::time::Duration; @@ -42,7 +40,6 @@ use crate::SafeKeeperConf; use crate::{debug_dump, wal_storage}; /// Things safekeeper should know about timeline state on peers. -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerInfo { pub sk_id: NodeId, @@ -50,13 +47,10 @@ pub struct PeerInfo { /// Term of the last entry. pub last_log_term: Term, /// LSN of the last record. - #[serde_as(as = "DisplayFromStr")] pub flush_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, /// Since which LSN safekeeper has WAL. TODO: remove this once we fill new /// sk since backup_lsn. - #[serde_as(as = "DisplayFromStr")] pub local_start_lsn: Lsn, /// When info was received. Serde annotations are not very useful but make /// the code compile -- we don't rely on this field externally. From 6defa2b5d551dbc4c45e47a28c38e66d9ff33cf5 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 6 Nov 2023 12:39:20 +0000 Subject: [PATCH 06/63] pageserver: add `Gate` as a partner to CancellationToken for safe shutdown of `Tenant` & `Timeline` (#5711) ## Problem When shutting down a Tenant, it isn't just important to cause any background tasks to stop. It's also important to wait until they have stopped before declaring shutdown complete, in cases where we may re-use the tenant's local storage for something else, such as running in secondary mode, or creating a new tenant with the same ID. ## Summary of changes A `Gate` class is added, inspired by [seastar::gate](https://docs.seastar.io/master/classseastar_1_1gate.html). For types that have an important lifetime that corresponds to some physical resource, use of a Gate as well as a CancellationToken provides a robust pattern for async requests & shutdown: - Requests must always acquire the gate as long as they are using the object - Shutdown must set the cancellation token, and then `close()` the gate to wait for requests in progress before returning. This is not for memory safety: it's for expressing the difference between "Arc exists", and "This tenant's files on disk are eligible to be read/written". - Both Tenant and Timeline get a Gate & CancellationToken. - The Timeline gate is held during eviction of layers, and during page_service requests. - Existing cancellation support in page_service is refined to use the timeline-scope cancellation token instead of a process-scope cancellation token. This replaces the use of `task_mgr::associate_with`: tasks no longer change their tenant/timelineidentity after being spawned. The Tenant's Gate is not yet used, but will be important for Tenant-scoped operations in secondary mode, where we must ensure that our secondary-mode downloads for a tenant are gated wrt the activity of an attached Tenant. This is part of a broader move away from using the global-state driven `task_mgr` shutdown tokens: - less global state where we rely on implicit knowledge of what task a given function is running in, and more explicit references to the cancellation token that a particular function/type will respect, making shutdown easier to reason about. - eventually avoid the big global TASKS mutex. --------- Co-authored-by: Joonas Koivunen --- libs/postgres_backend/src/lib.rs | 17 +- libs/utils/src/sync.rs | 2 + libs/utils/src/sync/gate.rs | 151 ++++++++++++++++++ pageserver/src/disk_usage_eviction_task.rs | 7 +- pageserver/src/http/routes.rs | 3 + pageserver/src/lib.rs | 17 +- pageserver/src/page_service.rs | 69 ++++---- pageserver/src/pgdatadir_mapping.rs | 18 ++- pageserver/src/task_mgr.rs | 47 ++---- pageserver/src/tenant.rs | 30 ++++ pageserver/src/tenant/size.rs | 10 +- pageserver/src/tenant/timeline.rs | 76 ++++++--- pageserver/src/tenant/timeline/delete.rs | 11 ++ .../src/tenant/timeline/eviction_task.rs | 5 +- .../walreceiver/connection_manager.rs | 11 +- ...test_pageserver_restarts_under_workload.py | 4 + 16 files changed, 358 insertions(+), 120 deletions(-) create mode 100644 libs/utils/src/sync/gate.rs diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index ad3af4e794..455fe7a481 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -728,12 +728,17 @@ impl PostgresBackend { trace!("got query {query_string:?}"); if let Err(e) = handler.process_query(self, query_string).await { - log_query_error(query_string, &e); - let short_error = short_error(&e); - self.write_message_noflush(&BeMessage::ErrorResponse( - &short_error, - Some(e.pg_error_code()), - ))?; + match e { + QueryError::Shutdown => return Ok(ProcessMsgResult::Break), + e => { + log_query_error(query_string, &e); + let short_error = short_error(&e); + self.write_message_noflush(&BeMessage::ErrorResponse( + &short_error, + Some(e.pg_error_code()), + ))?; + } + } } self.write_message_noflush(&BeMessage::ReadyForQuery)?; } diff --git a/libs/utils/src/sync.rs b/libs/utils/src/sync.rs index 125eeca129..2ee8f35449 100644 --- a/libs/utils/src/sync.rs +++ b/libs/utils/src/sync.rs @@ -1 +1,3 @@ pub mod heavier_once_cell; + +pub mod gate; diff --git a/libs/utils/src/sync/gate.rs b/libs/utils/src/sync/gate.rs new file mode 100644 index 0000000000..1391d238e6 --- /dev/null +++ b/libs/utils/src/sync/gate.rs @@ -0,0 +1,151 @@ +use std::{sync::Arc, time::Duration}; + +/// Gates are a concurrency helper, primarily used for implementing safe shutdown. +/// +/// Users of a resource call `enter()` to acquire a GateGuard, and the owner of +/// the resource calls `close()` when they want to ensure that all holders of guards +/// have released them, and that no future guards will be issued. +pub struct Gate { + /// Each caller of enter() takes one unit from the semaphore. In close(), we + /// take all the units to ensure all GateGuards are destroyed. + sem: Arc, + + /// For observability only: a name that will be used to log warnings if a particular + /// gate is holding up shutdown + name: String, +} + +/// RAII guard for a [`Gate`]: as long as this exists, calls to [`Gate::close`] will +/// not complete. +#[derive(Debug)] +pub struct GateGuard(tokio::sync::OwnedSemaphorePermit); + +/// Observability helper: every `warn_period`, emit a log warning that we're still waiting on this gate +async fn warn_if_stuck( + fut: Fut, + name: &str, + warn_period: std::time::Duration, +) -> ::Output { + let started = std::time::Instant::now(); + + let mut fut = std::pin::pin!(fut); + + loop { + match tokio::time::timeout(warn_period, &mut fut).await { + Ok(ret) => return ret, + Err(_) => { + tracing::warn!( + gate = name, + elapsed_ms = started.elapsed().as_millis(), + "still waiting, taking longer than expected..." + ); + } + } + } +} + +#[derive(Debug)] +pub enum GateError { + GateClosed, +} + +impl Gate { + const MAX_UNITS: u32 = u32::MAX; + + pub fn new(name: String) -> Self { + Self { + sem: Arc::new(tokio::sync::Semaphore::new(Self::MAX_UNITS as usize)), + name, + } + } + + /// Acquire a guard that will prevent close() calls from completing. If close() + /// was already called, this will return an error which should be interpreted + /// as "shutting down". + /// + /// This function would typically be used from e.g. request handlers. While holding + /// the guard returned from this function, it is important to respect a CancellationToken + /// to avoid blocking close() indefinitely: typically types that contain a Gate will + /// also contain a CancellationToken. + pub fn enter(&self) -> Result { + self.sem + .clone() + .try_acquire_owned() + .map(GateGuard) + .map_err(|_| GateError::GateClosed) + } + + /// Types with a shutdown() method and a gate should call this method at the + /// end of shutdown, to ensure that all GateGuard holders are done. + /// + /// This will wait for all guards to be destroyed. For this to complete promptly, it is + /// important that the holders of such guards are respecting a CancellationToken which has + /// been cancelled before entering this function. + pub async fn close(&self) { + warn_if_stuck(self.do_close(), &self.name, Duration::from_millis(1000)).await + } + + async fn do_close(&self) { + tracing::debug!(gate = self.name, "Closing Gate..."); + match self.sem.acquire_many(Self::MAX_UNITS).await { + Ok(_units) => { + // While holding all units, close the semaphore. All subsequent calls to enter() will fail. + self.sem.close(); + } + Err(_) => { + // Semaphore closed: we are the only function that can do this, so it indicates a double-call. + // This is legal. Timeline::shutdown for example is not protected from being called more than + // once. + tracing::debug!(gate = self.name, "Double close") + } + } + tracing::debug!(gate = self.name, "Closed Gate.") + } +} + +#[cfg(test)] +mod tests { + use futures::FutureExt; + + use super::*; + + #[tokio::test] + async fn test_idle_gate() { + // Having taken no gates, we should not be blocked in close + let gate = Gate::new("test".to_string()); + gate.close().await; + + // If a guard is dropped before entering, close should not be blocked + let gate = Gate::new("test".to_string()); + let guard = gate.enter().unwrap(); + drop(guard); + gate.close().await; + + // Entering a closed guard fails + gate.enter().expect_err("enter should fail after close"); + } + + #[tokio::test] + async fn test_busy_gate() { + let gate = Gate::new("test".to_string()); + + let guard = gate.enter().unwrap(); + + let mut close_fut = std::pin::pin!(gate.close()); + + // Close should be blocked + assert!(close_fut.as_mut().now_or_never().is_none()); + + // Attempting to enter() should fail, even though close isn't done yet. + gate.enter() + .expect_err("enter should fail after entering close"); + + drop(guard); + + // Guard is gone, close should finish + assert!(close_fut.as_mut().now_or_never().is_some()); + + // Attempting to enter() is still forbidden + gate.enter().expect_err("enter should fail finishing close"); + } +} diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index 413c941bc4..bc4bf51862 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -403,7 +403,7 @@ pub async fn disk_usage_eviction_task_iteration_impl( return (evicted_bytes, evictions_failed); }; - let results = timeline.evict_layers(&batch, &cancel).await; + let results = timeline.evict_layers(&batch).await; match results { Ok(results) => { @@ -554,6 +554,11 @@ async fn collect_eviction_candidates( } }; + if tenant.cancel.is_cancelled() { + info!(%tenant_id, "Skipping tenant for eviction, it is shutting down"); + continue; + } + // collect layers from all timelines in this tenant // // If one of the timelines becomes `!is_active()` during the iteration, diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index a6fb26b298..db728f243e 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -396,6 +396,9 @@ async fn timeline_create_handler( Err(e @ tenant::CreateTimelineError::AncestorNotActive) => { json_response(StatusCode::SERVICE_UNAVAILABLE, HttpErrorBody::from_msg(e.to_string())) } + Err(tenant::CreateTimelineError::ShuttingDown) => { + json_response(StatusCode::SERVICE_UNAVAILABLE,HttpErrorBody::from_msg("tenant shutting down".to_string())) + } Err(tenant::CreateTimelineError::Other(err)) => Err(ApiError::InternalServerError(err)), } } diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index e71130e8af..04c3ae9746 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -61,14 +61,6 @@ pub async fn shutdown_pageserver(deletion_queue: Option, exit_cod ) .await; - // Shut down any page service tasks. - timed( - task_mgr::shutdown_tasks(Some(TaskKind::PageRequestHandler), None, None), - "shutdown PageRequestHandlers", - Duration::from_secs(1), - ) - .await; - // Shut down all the tenants. This flushes everything to disk and kills // the checkpoint and GC tasks. timed( @@ -78,6 +70,15 @@ pub async fn shutdown_pageserver(deletion_queue: Option, exit_cod ) .await; + // Shut down any page service tasks: any in-progress work for particular timelines or tenants + // should already have been canclled via mgr::shutdown_all_tenants + timed( + task_mgr::shutdown_tasks(Some(TaskKind::PageRequestHandler), None, None), + "shutdown PageRequestHandlers", + Duration::from_secs(1), + ) + .await; + // Best effort to persist any outstanding deletions, to avoid leaking objects if let Some(mut deletion_queue) = deletion_queue { deletion_queue.shutdown(Duration::from_secs(5)).await; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 536334d051..334aee3dd1 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -223,13 +223,7 @@ async fn page_service_conn_main( // and create a child per-query context when it invokes process_query. // But it's in a shared crate, so, we store connection_ctx inside PageServerHandler // and create the per-query context in process_query ourselves. - let mut conn_handler = PageServerHandler::new( - conf, - broker_client, - auth, - connection_ctx, - task_mgr::shutdown_token(), - ); + let mut conn_handler = PageServerHandler::new(conf, broker_client, auth, connection_ctx); let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; match pgbackend @@ -263,10 +257,6 @@ struct PageServerHandler { /// For each query received over the connection, /// `process_query` creates a child context from this one. connection_ctx: RequestContext, - - /// A token that should fire when the tenant transitions from - /// attached state, or when the pageserver is shutting down. - cancel: CancellationToken, } impl PageServerHandler { @@ -275,7 +265,6 @@ impl PageServerHandler { broker_client: storage_broker::BrokerClientChannel, auth: Option>, connection_ctx: RequestContext, - cancel: CancellationToken, ) -> Self { PageServerHandler { _conf: conf, @@ -283,7 +272,6 @@ impl PageServerHandler { auth, claims: None, connection_ctx, - cancel, } } @@ -291,7 +279,11 @@ impl PageServerHandler { /// this rather than naked flush() in order to shut down promptly. Without this, we would /// block shutdown of a tenant if a postgres client was failing to consume bytes we send /// in the flush. - async fn flush_cancellable(&self, pgb: &mut PostgresBackend) -> Result<(), QueryError> + async fn flush_cancellable( + &self, + pgb: &mut PostgresBackend, + cancel: &CancellationToken, + ) -> Result<(), QueryError> where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { @@ -299,7 +291,7 @@ impl PageServerHandler { flush_r = pgb.flush() => { Ok(flush_r?) }, - _ = self.cancel.cancelled() => { + _ = cancel.cancelled() => { Err(QueryError::Shutdown) } ) @@ -308,6 +300,7 @@ impl PageServerHandler { fn copyin_stream<'a, IO>( &'a self, pgb: &'a mut PostgresBackend, + cancel: &'a CancellationToken, ) -> impl Stream> + 'a where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, @@ -317,7 +310,7 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = self.cancel.cancelled() => { + _ = cancel.cancelled() => { // We were requested to shut down. let msg = "pageserver is shutting down"; let _ = pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, None)); @@ -357,7 +350,7 @@ impl PageServerHandler { let query_error = QueryError::Disconnected(ConnectionError::Io(io::Error::new(io::ErrorKind::ConnectionReset, msg))); // error can't happen here, ErrorResponse serialization should be always ok pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, Some(query_error.pg_error_code()))).map_err(|e| e.into_io_error())?; - self.flush_cancellable(pgb).await.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + self.flush_cancellable(pgb, cancel).await.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?; } Err(QueryError::Disconnected(ConnectionError::Io(io_error))) => { @@ -384,10 +377,6 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - // NOTE: pagerequests handler exits when connection is closed, - // so there is no need to reset the association - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); - // Make request tracer if needed let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; let mut tracer = if tenant.get_trace_read_requests() { @@ -405,9 +394,14 @@ impl PageServerHandler { .get_timeline(timeline_id, true) .map_err(|e| anyhow::anyhow!(e))?; + // Avoid starting new requests if the timeline has already started shutting down, + // and block timeline shutdown until this request is complete, or drops out due + // to cancellation. + let _timeline_guard = timeline.gate.enter().map_err(|_| QueryError::Shutdown)?; + // switch client to COPYBOTH pgb.write_message_noflush(&BeMessage::CopyBothResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; let metrics = metrics::SmgrQueryTimePerTimeline::new(&tenant_id, &timeline_id); @@ -415,7 +409,7 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = self.cancel.cancelled() => { + _ = timeline.cancel.cancelled() => { // We were requested to shut down. info!("shutdown request received in page handler"); return Err(QueryError::Shutdown) @@ -490,9 +484,20 @@ impl PageServerHandler { } }; + if let Err(e) = &response { + if timeline.cancel.is_cancelled() { + // If we fail to fulfil a request during shutdown, which may be _because_ of + // shutdown, then do not send the error to the client. Instead just drop the + // connection. + span.in_scope(|| info!("dropped response during shutdown: {e:#}")); + return Err(QueryError::Shutdown); + } + } + let response = response.unwrap_or_else(|e| { // print the all details to the log with {:#}, but for the client the - // error message is enough + // error message is enough. Do not log if shutting down, as the anyhow::Error + // here includes cancellation which is not an error. span.in_scope(|| error!("error reading relation or page version: {:#}", e)); PagestreamBeMessage::Error(PagestreamErrorResponse { message: e.to_string(), @@ -500,7 +505,7 @@ impl PageServerHandler { }); pgb.write_message_noflush(&BeMessage::CopyData(&response.serialize()))?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; } Ok(()) } @@ -522,7 +527,6 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); // Create empty timeline info!("creating new timeline"); let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; @@ -543,9 +547,9 @@ impl PageServerHandler { // Import basebackup provided via CopyData info!("importing basebackup"); pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &tenant.cancel).await?; - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb))); + let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &tenant.cancel))); timeline .import_basebackup_from_tar( &mut copyin_reader, @@ -582,7 +586,6 @@ impl PageServerHandler { IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { debug_assert_current_span_has_tenant_and_timeline_id(); - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; let last_record_lsn = timeline.get_last_record_lsn(); @@ -598,8 +601,8 @@ impl PageServerHandler { // Import wal provided via CopyData info!("importing wal"); pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb).await?; - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb))); + self.flush_cancellable(pgb, &timeline.cancel).await?; + let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &timeline.cancel))); import_wal_from_tar(&timeline, &mut copyin_reader, start_lsn, end_lsn, &ctx).await?; info!("wal import complete"); @@ -807,7 +810,7 @@ impl PageServerHandler { // switch client to COPYOUT pgb.write_message_noflush(&BeMessage::CopyOutResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; // Send a tarball of the latest layer on the timeline. Compress if not // fullbackup. TODO Compress in that case too (tests need to be updated) @@ -859,7 +862,7 @@ impl PageServerHandler { } pgb.write_message_noflush(&BeMessage::CopyDone)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; let basebackup_after = started .elapsed() diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index daadf6abd4..88974588d4 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -44,6 +44,17 @@ pub enum CalculateLogicalSizeError { Other(#[from] anyhow::Error), } +impl From for CalculateLogicalSizeError { + fn from(pre: PageReconstructError) -> Self { + match pre { + PageReconstructError::AncestorStopping(_) | PageReconstructError::Cancelled => { + Self::Cancelled + } + _ => Self::Other(pre.into()), + } + } +} + #[derive(Debug, thiserror::Error)] pub enum RelationError { #[error("Relation Already Exists")] @@ -573,7 +584,7 @@ impl Timeline { crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id(); // Fetch list of database dirs and iterate them - let buf = self.get(DBDIR_KEY, lsn, ctx).await.context("read dbdir")?; + let buf = self.get(DBDIR_KEY, lsn, ctx).await?; let dbdir = DbDirectory::des(&buf).context("deserialize db directory")?; let mut total_size: u64 = 0; @@ -587,10 +598,7 @@ impl Timeline { return Err(CalculateLogicalSizeError::Cancelled); } let relsize_key = rel_size_to_key(rel); - let mut buf = self - .get(relsize_key, lsn, ctx) - .await - .with_context(|| format!("read relation size of {rel:?}"))?; + let mut buf = self.get(relsize_key, lsn, ctx).await?; let relsize = buf.get_u32_le(); total_size += relsize as u64; diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 017322ffb2..4270b6edb0 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -299,10 +299,6 @@ pub enum TaskKind { #[derive(Default)] struct MutableTaskState { - /// Tenant and timeline that this task is associated with. - tenant_id: Option, - timeline_id: Option, - /// Handle for waiting for the task to exit. It can be None, if the /// the task has already exited. join_handle: Option>, @@ -319,6 +315,11 @@ struct PageServerTask { // To request task shutdown, just cancel this token. cancel: CancellationToken, + /// Tasks may optionally be launched for a particular tenant/timeline, enabling + /// later cancelling tasks for that tenant/timeline in [`shutdown_tasks`] + tenant_id: Option, + timeline_id: Option, + mutable: Mutex, } @@ -344,11 +345,9 @@ where kind, name: name.to_string(), cancel: cancel.clone(), - mutable: Mutex::new(MutableTaskState { - tenant_id, - timeline_id, - join_handle: None, - }), + tenant_id, + timeline_id, + mutable: Mutex::new(MutableTaskState { join_handle: None }), }); TASKS.lock().unwrap().insert(task_id, Arc::clone(&task)); @@ -418,8 +417,6 @@ async fn task_finish( let mut shutdown_process = false; { - let task_mut = task.mutable.lock().unwrap(); - match result { Ok(Ok(())) => { debug!("Task '{}' exited normally", task_name); @@ -428,13 +425,13 @@ async fn task_finish( if shutdown_process_on_error { error!( "Shutting down: task '{}' tenant_id: {:?}, timeline_id: {:?} exited with error: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); shutdown_process = true; } else { error!( "Task '{}' tenant_id: {:?}, timeline_id: {:?} exited with error: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); } } @@ -442,13 +439,13 @@ async fn task_finish( if shutdown_process_on_error { error!( "Shutting down: task '{}' tenant_id: {:?}, timeline_id: {:?} panicked: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); shutdown_process = true; } else { error!( "Task '{}' tenant_id: {:?}, timeline_id: {:?} panicked: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); } } @@ -460,17 +457,6 @@ async fn task_finish( } } -// expected to be called from the task of the given id. -pub fn associate_with(tenant_id: Option, timeline_id: Option) { - CURRENT_TASK.with(|ct| { - let mut task_mut = ct.mutable.lock().unwrap(); - task_mut.tenant_id = tenant_id; - task_mut.timeline_id = timeline_id; - }); -} - -/// Is there a task running that matches the criteria - /// Signal and wait for tasks to shut down. /// /// @@ -493,17 +479,16 @@ pub async fn shutdown_tasks( { let tasks = TASKS.lock().unwrap(); for task in tasks.values() { - let task_mut = task.mutable.lock().unwrap(); if (kind.is_none() || Some(task.kind) == kind) - && (tenant_id.is_none() || task_mut.tenant_id == tenant_id) - && (timeline_id.is_none() || task_mut.timeline_id == timeline_id) + && (tenant_id.is_none() || task.tenant_id == tenant_id) + && (timeline_id.is_none() || task.timeline_id == timeline_id) { task.cancel.cancel(); victim_tasks.push(( Arc::clone(task), task.kind, - task_mut.tenant_id, - task_mut.timeline_id, + task.tenant_id, + task.timeline_id, )); } } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 3a426ac87b..55815a4956 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -26,6 +26,7 @@ use tracing::*; use utils::completion; use utils::crashsafe::path_with_suffix_extension; use utils::fs_ext; +use utils::sync::gate::Gate; use std::cmp::min; use std::collections::hash_map::Entry; @@ -252,6 +253,14 @@ pub struct Tenant { eviction_task_tenant_state: tokio::sync::Mutex, pub(crate) delete_progress: Arc>, + + // Cancellation token fires when we have entered shutdown(). This is a parent of + // Timelines' cancellation token. + pub(crate) cancel: CancellationToken, + + // Users of the Tenant such as the page service must take this Gate to avoid + // trying to use a Tenant which is shutting down. + pub(crate) gate: Gate, } pub(crate) enum WalRedoManager { @@ -395,6 +404,8 @@ pub enum CreateTimelineError { AncestorLsn(anyhow::Error), #[error("ancestor timeline is not active")] AncestorNotActive, + #[error("tenant shutting down")] + ShuttingDown, #[error(transparent)] Other(#[from] anyhow::Error), } @@ -1524,6 +1535,11 @@ impl Tenant { ))); } + let _gate = self + .gate + .enter() + .map_err(|_| CreateTimelineError::ShuttingDown)?; + if let Ok(existing) = self.get_timeline(new_timeline_id, false) { debug!("timeline {new_timeline_id} already exists"); @@ -1808,6 +1824,7 @@ impl Tenant { freeze_and_flush: bool, ) -> Result<(), completion::Barrier> { span::debug_assert_current_span_has_tenant_id(); + // Set tenant (and its timlines) to Stoppping state. // // Since we can only transition into Stopping state after activation is complete, @@ -1846,6 +1863,7 @@ impl Tenant { js.spawn(async move { timeline.shutdown(freeze_and_flush).instrument(span).await }); }) }; + tracing::info!("Waiting for timelines..."); while let Some(res) = js.join_next().await { match res { Ok(()) => {} @@ -1855,12 +1873,21 @@ impl Tenant { } } + // We cancel the Tenant's cancellation token _after_ the timelines have all shut down. This permits + // them to continue to do work during their shutdown methods, e.g. flushing data. + tracing::debug!("Cancelling CancellationToken"); + self.cancel.cancel(); + // shutdown all tenant and timeline tasks: gc, compaction, page service // No new tasks will be started for this tenant because it's in `Stopping` state. // // this will additionally shutdown and await all timeline tasks. + tracing::debug!("Waiting for tasks..."); task_mgr::shutdown_tasks(None, Some(self.tenant_id), None).await; + // Wait for any in-flight operations to complete + self.gate.close().await; + Ok(()) } @@ -2267,6 +2294,7 @@ impl Tenant { initial_logical_size_can_start.cloned(), initial_logical_size_attempt.cloned().flatten(), state, + self.cancel.child_token(), ); Ok(timeline) @@ -2356,6 +2384,8 @@ impl Tenant { cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)), eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()), delete_progress: Arc::new(tokio::sync::Mutex::new(DeleteTenantFlow::default())), + cancel: CancellationToken::default(), + gate: Gate::new(format!("Tenant<{tenant_id}>")), } } diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index 5e8ee2b99e..e4df94b8e9 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -406,10 +406,12 @@ async fn fill_logical_sizes( have_any_error = true; } Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Err(error)))) => { - warn!( - timeline_id=%timeline.timeline_id, - "failed to calculate logical size at {lsn}: {error:#}" - ); + if !matches!(error, CalculateLogicalSizeError::Cancelled) { + warn!( + timeline_id=%timeline.timeline_id, + "failed to calculate logical size at {lsn}: {error:#}" + ); + } have_any_error = true; } Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Ok(size)))) => { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index baa97b6cf9..36629e0655 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -23,7 +23,7 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::id::TenantTimelineId; +use utils::{id::TenantTimelineId, sync::gate::Gate}; use std::cmp::{max, min, Ordering}; use std::collections::{BinaryHeap, HashMap, HashSet}; @@ -310,6 +310,13 @@ pub struct Timeline { /// Load or creation time information about the disk_consistent_lsn and when the loading /// happened. Used for consumption metrics. pub(crate) loaded_at: (Lsn, SystemTime), + + /// Gate to prevent shutdown completing while I/O is still happening to this timeline's data + pub(crate) gate: Gate, + + /// Cancellation token scoped to this timeline: anything doing long-running work relating + /// to the timeline should drop out when this token fires. + pub(crate) cancel: CancellationToken, } pub struct WalReceiverInfo { @@ -786,7 +793,11 @@ impl Timeline { // as an empty timeline. Also in unit tests, when we use the timeline // as a simple key-value store, ignoring the datadir layout. Log the // error but continue. - error!("could not compact, repartitioning keyspace failed: {err:?}"); + // + // Suppress error when it's due to cancellation + if !self.cancel.is_cancelled() { + error!("could not compact, repartitioning keyspace failed: {err:?}"); + } } }; @@ -884,7 +895,12 @@ impl Timeline { pub async fn shutdown(self: &Arc, freeze_and_flush: bool) { debug_assert_current_span_has_tenant_and_timeline_id(); + // Signal any subscribers to our cancellation token to drop out + tracing::debug!("Cancelling CancellationToken"); + self.cancel.cancel(); + // prevent writes to the InMemoryLayer + tracing::debug!("Waiting for WalReceiverManager..."); task_mgr::shutdown_tasks( Some(TaskKind::WalReceiverManager), Some(self.tenant_id), @@ -920,6 +936,16 @@ impl Timeline { warn!("failed to await for frozen and flushed uploads: {e:#}"); } } + + // Page request handlers might be waiting for LSN to advance: they do not respect Timeline::cancel + // while doing so. + self.last_record_lsn.shutdown(); + + tracing::debug!("Waiting for tasks..."); + task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(self.timeline_id)).await; + + // Finally wait until any gate-holders are complete + self.gate.close().await; } pub fn set_state(&self, new_state: TimelineState) { @@ -1048,6 +1074,11 @@ impl Timeline { /// Like [`evict_layer_batch`](Self::evict_layer_batch), but for just one layer. /// Additional case `Ok(None)` covers the case where the layer could not be found by its `layer_file_name`. pub async fn evict_layer(&self, layer_file_name: &str) -> anyhow::Result> { + let _gate = self + .gate + .enter() + .map_err(|_| anyhow::anyhow!("Shutting down"))?; + let Some(local_layer) = self.find_layer(layer_file_name).await else { return Ok(None); }; @@ -1063,9 +1094,8 @@ impl Timeline { .as_ref() .ok_or_else(|| anyhow::anyhow!("remote storage not configured; cannot evict"))?; - let cancel = CancellationToken::new(); let results = self - .evict_layer_batch(remote_client, &[local_layer], &cancel) + .evict_layer_batch(remote_client, &[local_layer]) .await?; assert_eq!(results.len(), 1); let result: Option> = results.into_iter().next().unwrap(); @@ -1080,15 +1110,18 @@ impl Timeline { pub(crate) async fn evict_layers( &self, layers_to_evict: &[Layer], - cancel: &CancellationToken, ) -> anyhow::Result>>> { + let _gate = self + .gate + .enter() + .map_err(|_| anyhow::anyhow!("Shutting down"))?; + let remote_client = self .remote_client .as_ref() .context("timeline must have RemoteTimelineClient")?; - self.evict_layer_batch(remote_client, layers_to_evict, cancel) - .await + self.evict_layer_batch(remote_client, layers_to_evict).await } /// Evict multiple layers at once, continuing through errors. @@ -1109,7 +1142,6 @@ impl Timeline { &self, remote_client: &Arc, layers_to_evict: &[Layer], - cancel: &CancellationToken, ) -> anyhow::Result>>> { // ensure that the layers have finished uploading // (don't hold the layer_removal_cs while we do it, we're not removing anything yet) @@ -1157,7 +1189,7 @@ impl Timeline { }; tokio::select! { - _ = cancel.cancelled() => {}, + _ = self.cancel.cancelled() => {}, _ = join => {} } @@ -1267,6 +1299,7 @@ impl Timeline { initial_logical_size_can_start: Option, initial_logical_size_attempt: Option, state: TimelineState, + cancel: CancellationToken, ) -> Arc { let disk_consistent_lsn = metadata.disk_consistent_lsn(); let (state, _) = watch::channel(state); @@ -1367,6 +1400,8 @@ impl Timeline { initial_logical_size_can_start, initial_logical_size_attempt: Mutex::new(initial_logical_size_attempt), + cancel, + gate: Gate::new(format!("Timeline<{tenant_id}/{timeline_id}>")), }; result.repartition_threshold = result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; @@ -2030,6 +2065,10 @@ impl Timeline { let mut cont_lsn = Lsn(request_lsn.0 + 1); 'outer: loop { + if self.cancel.is_cancelled() { + return Err(PageReconstructError::Cancelled); + } + // The function should have updated 'state' //info!("CALLED for {} at {}: {:?} with {} records, cached {}", key, cont_lsn, result, reconstruct_state.records.len(), cached_lsn); match result { @@ -4366,25 +4405,10 @@ mod tests { .expect("should had been resident") .drop_eviction_guard(); - let cancel = tokio_util::sync::CancellationToken::new(); let batch = [layer]; - let first = { - let cancel = cancel.child_token(); - async { - let cancel = cancel; - timeline - .evict_layer_batch(&rc, &batch, &cancel) - .await - .unwrap() - } - }; - let second = async { - timeline - .evict_layer_batch(&rc, &batch, &cancel) - .await - .unwrap() - }; + let first = async { timeline.evict_layer_batch(&rc, &batch).await.unwrap() }; + let second = async { timeline.evict_layer_batch(&rc, &batch).await.unwrap() }; let (first, second) = tokio::join!(first, second); diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 6d30664515..56a99a25cf 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -17,6 +17,7 @@ use crate::{ deletion_queue::DeletionQueueClient, task_mgr::{self, TaskKind}, tenant::{ + debug_assert_current_span_has_tenant_and_timeline_id, metadata::TimelineMetadata, remote_timeline_client::{ self, PersistIndexPartWithDeletedFlagError, RemoteTimelineClient, @@ -30,6 +31,11 @@ use super::{Timeline, TimelineResources}; /// Now that the Timeline is in Stopping state, request all the related tasks to shut down. async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { + debug_assert_current_span_has_tenant_and_timeline_id(); + // Notify any timeline work to drop out of loops/requests + tracing::debug!("Cancelling CancellationToken"); + timeline.cancel.cancel(); + // Stop the walreceiver first. debug!("waiting for wal receiver to shutdown"); let maybe_started_walreceiver = { timeline.walreceiver.lock().unwrap().take() }; @@ -74,6 +80,11 @@ async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { "failpoint: timeline-delete-before-index-deleted-at" ))? }); + + tracing::debug!("Waiting for gate..."); + timeline.gate.close().await; + tracing::debug!("Shutdown complete"); + Ok(()) } diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index dc5c71bbe1..52c53a5c3b 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -277,10 +277,7 @@ impl Timeline { Some(c) => c, }; - let results = match self - .evict_layer_batch(remote_client, &candidates, cancel) - .await - { + let results = match self.evict_layer_batch(remote_client, &candidates).await { Err(pre_err) => { stats.errors += candidates.len(); error!("could not do any evictions: {pre_err:#}"); diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index e28c4a5f69..3077712445 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -426,7 +426,7 @@ impl ConnectionManagerState { timeline, new_sk.wal_source_connconf, events_sender, - cancellation, + cancellation.clone(), connect_timeout, ctx, node_id, @@ -447,7 +447,14 @@ impl ConnectionManagerState { } WalReceiverError::Other(e) => { // give out an error to have task_mgr give it a really verbose logging - Err(e).context("walreceiver connection handling failure") + if cancellation.is_cancelled() { + // Ideally we would learn about this via some path other than Other, but + // that requires refactoring all the intermediate layers of ingest code + // that only emit anyhow::Error + Ok(()) + } else { + Err(e).context("walreceiver connection handling failure") + } } } } diff --git a/test_runner/regress/test_pageserver_restarts_under_workload.py b/test_runner/regress/test_pageserver_restarts_under_workload.py index 65569f3bac..d07b8dbe6b 100644 --- a/test_runner/regress/test_pageserver_restarts_under_workload.py +++ b/test_runner/regress/test_pageserver_restarts_under_workload.py @@ -17,6 +17,10 @@ def test_pageserver_restarts_under_worload(neon_simple_env: NeonEnv, pg_bin: PgB n_restarts = 10 scale = 10 + # Pageserver currently logs requests on non-active tenants at error level + # https://github.com/neondatabase/neon/issues/5784 + env.pageserver.allowed_errors.append(".* will not become active. Current state: Stopping.*") + def run_pgbench(connstr: str): log.info(f"Start a pgbench workload on pg {connstr}") pg_bin.run_capture(["pgbench", "-i", f"-s{scale}", connstr]) From dc725672882e6c0ae2e5afbc87ac00829a4d8ffe Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Mon, 6 Nov 2023 08:58:20 -0500 Subject: [PATCH 07/63] Layer flush minor speedup (#5765) Convert keys to `i128` before sorting --- pageserver/src/tenant.rs | 14 +- pageserver/src/tenant/disk_btree_test_data.rs | 4000 ++++++++--------- .../tenant/storage_layer/inmemory_layer.rs | 11 +- 3 files changed, 2015 insertions(+), 2010 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 55815a4956..ad538a82fd 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3722,7 +3722,7 @@ mod tests { use tokio_util::sync::CancellationToken; static TEST_KEY: Lazy = - Lazy::new(|| Key::from_slice(&hex!("112222222233333333444444445500000001"))); + Lazy::new(|| Key::from_slice(&hex!("010000000033333333444444445500000001"))); #[tokio::test] async fn test_basic() -> anyhow::Result<()> { @@ -3818,9 +3818,9 @@ mod tests { let writer = tline.writer().await; #[allow(non_snake_case)] - let TEST_KEY_A: Key = Key::from_hex("112222222233333333444444445500000001").unwrap(); + let TEST_KEY_A: Key = Key::from_hex("110000000033333333444444445500000001").unwrap(); #[allow(non_snake_case)] - let TEST_KEY_B: Key = Key::from_hex("112222222233333333444444445500000002").unwrap(); + let TEST_KEY_B: Key = Key::from_hex("110000000033333333444444445500000002").unwrap(); // Insert a value on the timeline writer @@ -4404,7 +4404,7 @@ mod tests { let mut keyspace = KeySpaceAccum::new(); - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut blknum = 0; for _ in 0..50 { for _ in 0..10000 { @@ -4450,7 +4450,7 @@ mod tests { const NUM_KEYS: usize = 1000; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut keyspace = KeySpaceAccum::new(); @@ -4531,7 +4531,7 @@ mod tests { const NUM_KEYS: usize = 1000; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut keyspace = KeySpaceAccum::new(); @@ -4622,7 +4622,7 @@ mod tests { const NUM_KEYS: usize = 100; const NUM_TLINES: usize = 50; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); // Track page mutation lsns across different timelines. let mut updated = [[Lsn(0); NUM_KEYS]; NUM_TLINES]; diff --git a/pageserver/src/tenant/disk_btree_test_data.rs b/pageserver/src/tenant/disk_btree_test_data.rs index 9462573f03..4f835f55af 100644 --- a/pageserver/src/tenant/disk_btree_test_data.rs +++ b/pageserver/src/tenant/disk_btree_test_data.rs @@ -10,2004 +10,2004 @@ use hex_literal::hex; /// by a delta layer, for evaluating how well the prefix compression works. #[rustfmt::skip] pub static TEST_DATA: [([u8; 26], u64); 2000] = [ - (hex!("0122222222333333334444444455000000000000000000000010"), 0x004001), - (hex!("0122222222333333334444444455000000000000000000007cb0"), 0x0040a1), - (hex!("0122222222333333334444444455000000010000000000000020"), 0x004141), - (hex!("0122222222333333334444444455000000020000000000000030"), 0x0041e1), - (hex!("01222222223333333344444444550000000200000000000051a0"), 0x004281), - (hex!("0122222222333333334444444455000000030000000000000040"), 0x004321), - (hex!("0122222222333333334444444455000000030000000000006cf0"), 0x0043c1), - (hex!("0122222222333333334444444455000000030000000000007140"), 0x004461), - (hex!("0122222222333333334444444455000000040000000000000050"), 0x004501), - (hex!("01222222223333333344444444550000000400000000000047f0"), 0x0045a1), - (hex!("01222222223333333344444444550000000400000000000072b0"), 0x004641), - (hex!("0122222222333333334444444455000000050000000000000060"), 0x0046e1), - (hex!("0122222222333333334444444455000000050000000000005550"), 0x004781), - (hex!("0122222222333333334444444455000000060000000000000070"), 0x004821), - (hex!("01222222223333333344444444550000000600000000000044a0"), 0x0048c1), - (hex!("0122222222333333334444444455000000060000000000006870"), 0x004961), - (hex!("0122222222333333334444444455000000070000000000000080"), 0x004a01), - (hex!("0122222222333333334444444455000000080000000000000090"), 0x004aa1), - (hex!("0122222222333333334444444455000000080000000000004150"), 0x004b41), - (hex!("01222222223333333344444444550000000900000000000000a0"), 0x004be1), - (hex!("01222222223333333344444444550000000a00000000000000b0"), 0x004c81), - (hex!("01222222223333333344444444550000000a0000000000006680"), 0x004d21), - (hex!("01222222223333333344444444550000000b00000000000000c0"), 0x004dc1), - (hex!("01222222223333333344444444550000000b0000000000006230"), 0x004e61), - (hex!("01222222223333333344444444550000000c00000000000000d0"), 0x004f01), - (hex!("01222222223333333344444444550000000d00000000000000e0"), 0x004fa1), - (hex!("01222222223333333344444444550000000e00000000000000f0"), 0x005041), - (hex!("01222222223333333344444444550000000e0000000000006000"), 0x0050e1), - (hex!("01222222223333333344444444550000000f0000000000000100"), 0x005181), - (hex!("01222222223333333344444444550000000f00000000000053c0"), 0x005221), - (hex!("01222222223333333344444444550000000f0000000000006580"), 0x0052c1), - (hex!("0122222222333333334444444455000000100000000000000110"), 0x005361), - (hex!("01222222223333333344444444550000001000000000000046c0"), 0x005401), - (hex!("0122222222333333334444444455000000100000000000004e40"), 0x0054a1), - (hex!("0122222222333333334444444455000000110000000000000120"), 0x005541), - (hex!("0122222222333333334444444455000000120000000000000130"), 0x0055e1), - (hex!("01222222223333333344444444550000001200000000000066d0"), 0x005681), - (hex!("0122222222333333334444444455000000130000000000000140"), 0x005721), - (hex!("0122222222333333334444444455000000130000000000007710"), 0x0057c1), - (hex!("0122222222333333334444444455000000140000000000000150"), 0x005861), - (hex!("0122222222333333334444444455000000140000000000006c40"), 0x005901), - (hex!("0122222222333333334444444455000000150000000000000160"), 0x0059a1), - (hex!("0122222222333333334444444455000000150000000000005990"), 0x005a41), - (hex!("0122222222333333334444444455000000160000000000000170"), 0x005ae1), - (hex!("0122222222333333334444444455000000160000000000005530"), 0x005b81), - (hex!("0122222222333333334444444455000000170000000000000180"), 0x005c21), - (hex!("0122222222333333334444444455000000170000000000004290"), 0x005cc1), - (hex!("0122222222333333334444444455000000180000000000000190"), 0x005d61), - (hex!("01222222223333333344444444550000001800000000000051c0"), 0x005e01), - (hex!("01222222223333333344444444550000001900000000000001a0"), 0x005ea1), - (hex!("0122222222333333334444444455000000190000000000005420"), 0x005f41), - (hex!("0122222222333333334444444455000000190000000000005770"), 0x005fe1), - (hex!("01222222223333333344444444550000001900000000000079d0"), 0x006081), - (hex!("01222222223333333344444444550000001a00000000000001b0"), 0x006121), - (hex!("01222222223333333344444444550000001a0000000000006f70"), 0x0061c1), - (hex!("01222222223333333344444444550000001a0000000000007150"), 0x006261), - (hex!("01222222223333333344444444550000001b00000000000001c0"), 0x006301), - (hex!("01222222223333333344444444550000001b0000000000005070"), 0x0063a1), - (hex!("01222222223333333344444444550000001c00000000000001d0"), 0x006441), - (hex!("01222222223333333344444444550000001d00000000000001e0"), 0x0064e1), - (hex!("01222222223333333344444444550000001e00000000000001f0"), 0x006581), - (hex!("01222222223333333344444444550000001e0000000000005650"), 0x006621), - (hex!("01222222223333333344444444550000001f0000000000000200"), 0x0066c1), - (hex!("01222222223333333344444444550000001f0000000000006ca0"), 0x006761), - (hex!("0122222222333333334444444455000000200000000000000210"), 0x006801), - (hex!("0122222222333333334444444455000000200000000000005fc0"), 0x0068a1), - (hex!("0122222222333333334444444455000000210000000000000220"), 0x006941), - (hex!("0122222222333333334444444455000000210000000000006430"), 0x0069e1), - (hex!("0122222222333333334444444455000000220000000000000230"), 0x006a81), - (hex!("01222222223333333344444444550000002200000000000040e0"), 0x006b21), - (hex!("0122222222333333334444444455000000230000000000000240"), 0x006bc1), - (hex!("01222222223333333344444444550000002300000000000042d0"), 0x006c61), - (hex!("0122222222333333334444444455000000240000000000000250"), 0x006d01), - (hex!("0122222222333333334444444455000000250000000000000260"), 0x006da1), - (hex!("01222222223333333344444444550000002500000000000058c0"), 0x006e41), - (hex!("0122222222333333334444444455000000260000000000000270"), 0x006ee1), - (hex!("0122222222333333334444444455000000260000000000004020"), 0x006f81), - (hex!("0122222222333333334444444455000000270000000000000280"), 0x007021), - (hex!("0122222222333333334444444455000000280000000000000290"), 0x0070c1), - (hex!("0122222222333333334444444455000000280000000000007c00"), 0x007161), - (hex!("01222222223333333344444444550000002900000000000002a0"), 0x007201), - (hex!("01222222223333333344444444550000002a00000000000002b0"), 0x0072a1), - (hex!("01222222223333333344444444550000002b00000000000002c0"), 0x007341), - (hex!("01222222223333333344444444550000002c00000000000002d0"), 0x0073e1), - (hex!("01222222223333333344444444550000002c00000000000041b0"), 0x007481), - (hex!("01222222223333333344444444550000002c0000000000004c30"), 0x007521), - (hex!("01222222223333333344444444550000002d00000000000002e0"), 0x0075c1), - (hex!("01222222223333333344444444550000002d0000000000005e40"), 0x007661), - (hex!("01222222223333333344444444550000002d0000000000006990"), 0x007701), - (hex!("01222222223333333344444444550000002e00000000000002f0"), 0x0077a1), - (hex!("01222222223333333344444444550000002f0000000000000300"), 0x007841), - (hex!("01222222223333333344444444550000002f0000000000004a70"), 0x0078e1), - (hex!("01222222223333333344444444550000002f0000000000006b40"), 0x007981), - (hex!("0122222222333333334444444455000000300000000000000310"), 0x007a21), - (hex!("0122222222333333334444444455000000310000000000000320"), 0x007ac1), - (hex!("0122222222333333334444444455000000320000000000000330"), 0x007b61), - (hex!("01222222223333333344444444550000003200000000000041a0"), 0x007c01), - (hex!("0122222222333333334444444455000000320000000000007340"), 0x007ca1), - (hex!("0122222222333333334444444455000000320000000000007730"), 0x007d41), - (hex!("0122222222333333334444444455000000330000000000000340"), 0x007de1), - (hex!("01222222223333333344444444550000003300000000000055a0"), 0x007e81), - (hex!("0122222222333333334444444455000000340000000000000350"), 0x007f21), - (hex!("0122222222333333334444444455000000350000000000000360"), 0x007fc1), - (hex!("01222222223333333344444444550000003500000000000077a0"), 0x008061), - (hex!("0122222222333333334444444455000000360000000000000370"), 0x008101), - (hex!("0122222222333333334444444455000000370000000000000380"), 0x0081a1), - (hex!("0122222222333333334444444455000000380000000000000390"), 0x008241), - (hex!("01222222223333333344444444550000003900000000000003a0"), 0x0082e1), - (hex!("01222222223333333344444444550000003a00000000000003b0"), 0x008381), - (hex!("01222222223333333344444444550000003a00000000000071c0"), 0x008421), - (hex!("01222222223333333344444444550000003b00000000000003c0"), 0x0084c1), - (hex!("01222222223333333344444444550000003c00000000000003d0"), 0x008561), - (hex!("01222222223333333344444444550000003d00000000000003e0"), 0x008601), - (hex!("01222222223333333344444444550000003e00000000000003f0"), 0x0086a1), - (hex!("01222222223333333344444444550000003e00000000000062e0"), 0x008741), - (hex!("01222222223333333344444444550000003f0000000000000400"), 0x0087e1), - (hex!("0122222222333333334444444455000000400000000000000410"), 0x008881), - (hex!("0122222222333333334444444455000000400000000000004460"), 0x008921), - (hex!("0122222222333333334444444455000000400000000000005b90"), 0x0089c1), - (hex!("01222222223333333344444444550000004000000000000079b0"), 0x008a61), - (hex!("0122222222333333334444444455000000410000000000000420"), 0x008b01), - (hex!("0122222222333333334444444455000000420000000000000430"), 0x008ba1), - (hex!("0122222222333333334444444455000000420000000000005640"), 0x008c41), - (hex!("0122222222333333334444444455000000430000000000000440"), 0x008ce1), - (hex!("01222222223333333344444444550000004300000000000072a0"), 0x008d81), - (hex!("0122222222333333334444444455000000440000000000000450"), 0x008e21), - (hex!("0122222222333333334444444455000000450000000000000460"), 0x008ec1), - (hex!("0122222222333333334444444455000000450000000000005750"), 0x008f61), - (hex!("01222222223333333344444444550000004500000000000077b0"), 0x009001), - (hex!("0122222222333333334444444455000000460000000000000470"), 0x0090a1), - (hex!("0122222222333333334444444455000000470000000000000480"), 0x009141), - (hex!("0122222222333333334444444455000000480000000000000490"), 0x0091e1), - (hex!("01222222223333333344444444550000004800000000000069e0"), 0x009281), - (hex!("01222222223333333344444444550000004900000000000004a0"), 0x009321), - (hex!("0122222222333333334444444455000000490000000000007370"), 0x0093c1), - (hex!("01222222223333333344444444550000004a00000000000004b0"), 0x009461), - (hex!("01222222223333333344444444550000004a0000000000005cb0"), 0x009501), - (hex!("01222222223333333344444444550000004b00000000000004c0"), 0x0095a1), - (hex!("01222222223333333344444444550000004c00000000000004d0"), 0x009641), - (hex!("01222222223333333344444444550000004c0000000000004880"), 0x0096e1), - (hex!("01222222223333333344444444550000004c0000000000007a40"), 0x009781), - (hex!("01222222223333333344444444550000004d00000000000004e0"), 0x009821), - (hex!("01222222223333333344444444550000004d0000000000006390"), 0x0098c1), - (hex!("01222222223333333344444444550000004e00000000000004f0"), 0x009961), - (hex!("01222222223333333344444444550000004e0000000000004db0"), 0x009a01), - (hex!("01222222223333333344444444550000004f0000000000000500"), 0x009aa1), - (hex!("0122222222333333334444444455000000500000000000000510"), 0x009b41), - (hex!("0122222222333333334444444455000000510000000000000520"), 0x009be1), - (hex!("01222222223333333344444444550000005100000000000069c0"), 0x009c81), - (hex!("0122222222333333334444444455000000520000000000000530"), 0x009d21), - (hex!("0122222222333333334444444455000000520000000000006e60"), 0x009dc1), - (hex!("01222222223333333344444444550000005200000000000070c0"), 0x009e61), - (hex!("0122222222333333334444444455000000530000000000000540"), 0x009f01), - (hex!("0122222222333333334444444455000000530000000000005840"), 0x009fa1), - (hex!("0122222222333333334444444455000000540000000000000550"), 0x00a041), - (hex!("01222222223333333344444444550000005400000000000043e0"), 0x00a0e1), - (hex!("01222222223333333344444444550000005400000000000074e0"), 0x00a181), - (hex!("0122222222333333334444444455000000550000000000000560"), 0x00a221), - (hex!("0122222222333333334444444455000000550000000000003ee0"), 0x00a2c1), - (hex!("0122222222333333334444444455000000560000000000000570"), 0x00a361), - (hex!("0122222222333333334444444455000000570000000000000580"), 0x00a401), - (hex!("0122222222333333334444444455000000570000000000007030"), 0x00a4a1), - (hex!("0122222222333333334444444455000000580000000000000590"), 0x00a541), - (hex!("0122222222333333334444444455000000580000000000005340"), 0x00a5e1), - (hex!("01222222223333333344444444550000005800000000000059f0"), 0x00a681), - (hex!("0122222222333333334444444455000000580000000000006930"), 0x00a721), - (hex!("01222222223333333344444444550000005900000000000005a0"), 0x00a7c1), - (hex!("0122222222333333334444444455000000590000000000003f90"), 0x00a861), - (hex!("01222222223333333344444444550000005a00000000000005b0"), 0x00a901), - (hex!("01222222223333333344444444550000005b00000000000005c0"), 0x00a9a1), - (hex!("01222222223333333344444444550000005b00000000000062c0"), 0x00aa41), - (hex!("01222222223333333344444444550000005c00000000000005d0"), 0x00aae1), - (hex!("01222222223333333344444444550000005c0000000000005a70"), 0x00ab81), - (hex!("01222222223333333344444444550000005c0000000000005dd0"), 0x00ac21), - (hex!("01222222223333333344444444550000005d00000000000005e0"), 0x00acc1), - (hex!("01222222223333333344444444550000005d0000000000005730"), 0x00ad61), - (hex!("01222222223333333344444444550000005e00000000000005f0"), 0x00ae01), - (hex!("01222222223333333344444444550000005e0000000000004f40"), 0x00aea1), - (hex!("01222222223333333344444444550000005f0000000000000600"), 0x00af41), - (hex!("0122222222333333334444444455000000600000000000000610"), 0x00afe1), - (hex!("0122222222333333334444444455000000600000000000007c40"), 0x00b081), - (hex!("0122222222333333334444444455000000610000000000000620"), 0x00b121), - (hex!("0122222222333333334444444455000000610000000000007860"), 0x00b1c1), - (hex!("0122222222333333334444444455000000620000000000000630"), 0x00b261), - (hex!("0122222222333333334444444455000000620000000000005050"), 0x00b301), - (hex!("0122222222333333334444444455000000630000000000000640"), 0x00b3a1), - (hex!("0122222222333333334444444455000000640000000000000650"), 0x00b441), - (hex!("0122222222333333334444444455000000650000000000000660"), 0x00b4e1), - (hex!("0122222222333333334444444455000000650000000000005330"), 0x00b581), - (hex!("0122222222333333334444444455000000660000000000000670"), 0x00b621), - (hex!("0122222222333333334444444455000000660000000000004e20"), 0x00b6c1), - (hex!("0122222222333333334444444455000000660000000000005ee0"), 0x00b761), - (hex!("0122222222333333334444444455000000660000000000006360"), 0x00b801), - (hex!("0122222222333333334444444455000000670000000000000680"), 0x00b8a1), - (hex!("0122222222333333334444444455000000670000000000004040"), 0x00b941), - (hex!("0122222222333333334444444455000000680000000000000690"), 0x00b9e1), - (hex!("0122222222333333334444444455000000680000000000003f80"), 0x00ba81), - (hex!("01222222223333333344444444550000006800000000000041e0"), 0x00bb21), - (hex!("01222222223333333344444444550000006900000000000006a0"), 0x00bbc1), - (hex!("0122222222333333334444444455000000690000000000006080"), 0x00bc61), - (hex!("01222222223333333344444444550000006a00000000000006b0"), 0x00bd01), - (hex!("01222222223333333344444444550000006a00000000000042f0"), 0x00bda1), - (hex!("01222222223333333344444444550000006b00000000000006c0"), 0x00be41), - (hex!("01222222223333333344444444550000006b00000000000052f0"), 0x00bee1), - (hex!("01222222223333333344444444550000006b0000000000005980"), 0x00bf81), - (hex!("01222222223333333344444444550000006b0000000000006170"), 0x00c021), - (hex!("01222222223333333344444444550000006c00000000000006d0"), 0x00c0c1), - (hex!("01222222223333333344444444550000006d00000000000006e0"), 0x00c161), - (hex!("01222222223333333344444444550000006d0000000000006fb0"), 0x00c201), - (hex!("01222222223333333344444444550000006e00000000000006f0"), 0x00c2a1), - (hex!("01222222223333333344444444550000006e00000000000065b0"), 0x00c341), - (hex!("01222222223333333344444444550000006e0000000000007970"), 0x00c3e1), - (hex!("01222222223333333344444444550000006f0000000000000700"), 0x00c481), - (hex!("01222222223333333344444444550000006f0000000000005900"), 0x00c521), - (hex!("01222222223333333344444444550000006f0000000000006d90"), 0x00c5c1), - (hex!("0122222222333333334444444455000000700000000000000710"), 0x00c661), - (hex!("01222222223333333344444444550000007000000000000045c0"), 0x00c701), - (hex!("0122222222333333334444444455000000700000000000004d40"), 0x00c7a1), - (hex!("0122222222333333334444444455000000710000000000000720"), 0x00c841), - (hex!("0122222222333333334444444455000000710000000000004dc0"), 0x00c8e1), - (hex!("0122222222333333334444444455000000710000000000007550"), 0x00c981), - (hex!("0122222222333333334444444455000000720000000000000730"), 0x00ca21), - (hex!("0122222222333333334444444455000000720000000000003ec0"), 0x00cac1), - (hex!("01222222223333333344444444550000007200000000000045a0"), 0x00cb61), - (hex!("0122222222333333334444444455000000720000000000006770"), 0x00cc01), - (hex!("0122222222333333334444444455000000720000000000006bc0"), 0x00cca1), - (hex!("0122222222333333334444444455000000730000000000000740"), 0x00cd41), - (hex!("0122222222333333334444444455000000730000000000005250"), 0x00cde1), - (hex!("01222222223333333344444444550000007300000000000075f0"), 0x00ce81), - (hex!("0122222222333333334444444455000000740000000000000750"), 0x00cf21), - (hex!("0122222222333333334444444455000000740000000000003ff0"), 0x00cfc1), - (hex!("01222222223333333344444444550000007400000000000079e0"), 0x00d061), - (hex!("0122222222333333334444444455000000750000000000000760"), 0x00d101), - (hex!("0122222222333333334444444455000000750000000000004310"), 0x00d1a1), - (hex!("0122222222333333334444444455000000760000000000000770"), 0x00d241), - (hex!("0122222222333333334444444455000000770000000000000780"), 0x00d2e1), - (hex!("01222222223333333344444444550000007700000000000062f0"), 0x00d381), - (hex!("0122222222333333334444444455000000770000000000006940"), 0x00d421), - (hex!("0122222222333333334444444455000000780000000000000790"), 0x00d4c1), - (hex!("01222222223333333344444444550000007900000000000007a0"), 0x00d561), - (hex!("0122222222333333334444444455000000790000000000007af0"), 0x00d601), - (hex!("01222222223333333344444444550000007a00000000000007b0"), 0x00d6a1), - (hex!("01222222223333333344444444550000007b00000000000007c0"), 0x00d741), - (hex!("01222222223333333344444444550000007b00000000000067e0"), 0x00d7e1), - (hex!("01222222223333333344444444550000007b0000000000007890"), 0x00d881), - (hex!("01222222223333333344444444550000007c00000000000007d0"), 0x00d921), - (hex!("01222222223333333344444444550000007d00000000000007e0"), 0x00d9c1), - (hex!("01222222223333333344444444550000007e00000000000007f0"), 0x00da61), - (hex!("01222222223333333344444444550000007f0000000000000800"), 0x00db01), - (hex!("01222222223333333344444444550000007f0000000000005be0"), 0x00dba1), - (hex!("0122222222333333334444444455000000800000000000000810"), 0x00dc41), - (hex!("0122222222333333334444444455000000810000000000000820"), 0x00dce1), - (hex!("0122222222333333334444444455000000810000000000007190"), 0x00dd81), - (hex!("0122222222333333334444444455000000820000000000000830"), 0x00de21), - (hex!("0122222222333333334444444455000000820000000000004ab0"), 0x00dec1), - (hex!("0122222222333333334444444455000000830000000000000840"), 0x00df61), - (hex!("0122222222333333334444444455000000830000000000006720"), 0x00e001), - (hex!("0122222222333333334444444455000000840000000000000850"), 0x00e0a1), - (hex!("0122222222333333334444444455000000850000000000000860"), 0x00e141), - (hex!("01222222223333333344444444550000008500000000000054f0"), 0x00e1e1), - (hex!("0122222222333333334444444455000000850000000000007920"), 0x00e281), - (hex!("0122222222333333334444444455000000860000000000000870"), 0x00e321), - (hex!("01222222223333333344444444550000008600000000000060e0"), 0x00e3c1), - (hex!("0122222222333333334444444455000000860000000000006be0"), 0x00e461), - (hex!("0122222222333333334444444455000000870000000000000880"), 0x00e501), - (hex!("0122222222333333334444444455000000870000000000006820"), 0x00e5a1), - (hex!("0122222222333333334444444455000000880000000000000890"), 0x00e641), - (hex!("01222222223333333344444444550000008900000000000008a0"), 0x00e6e1), - (hex!("0122222222333333334444444455000000890000000000007c30"), 0x00e781), - (hex!("01222222223333333344444444550000008a00000000000008b0"), 0x00e821), - (hex!("01222222223333333344444444550000008b00000000000008c0"), 0x00e8c1), - (hex!("01222222223333333344444444550000008b0000000000005910"), 0x00e961), - (hex!("01222222223333333344444444550000008b0000000000006fe0"), 0x00ea01), - (hex!("01222222223333333344444444550000008c00000000000008d0"), 0x00eaa1), - (hex!("01222222223333333344444444550000008c0000000000006800"), 0x00eb41), - (hex!("01222222223333333344444444550000008d00000000000008e0"), 0x00ebe1), - (hex!("01222222223333333344444444550000008d0000000000005810"), 0x00ec81), - (hex!("01222222223333333344444444550000008d0000000000007c90"), 0x00ed21), - (hex!("01222222223333333344444444550000008e00000000000008f0"), 0x00edc1), - (hex!("01222222223333333344444444550000008e00000000000058f0"), 0x00ee61), - (hex!("01222222223333333344444444550000008f0000000000000900"), 0x00ef01), - (hex!("01222222223333333344444444550000008f0000000000005a30"), 0x00efa1), - (hex!("0122222222333333334444444455000000900000000000000910"), 0x00f041), - (hex!("0122222222333333334444444455000000900000000000006130"), 0x00f0e1), - (hex!("0122222222333333334444444455000000900000000000006550"), 0x00f181), - (hex!("0122222222333333334444444455000000910000000000000920"), 0x00f221), - (hex!("01222222223333333344444444550000009100000000000079f0"), 0x00f2c1), - (hex!("0122222222333333334444444455000000920000000000000930"), 0x00f361), - (hex!("0122222222333333334444444455000000920000000000005620"), 0x00f401), - (hex!("0122222222333333334444444455000000920000000000005e90"), 0x00f4a1), - (hex!("01222222223333333344444444550000009200000000000063d0"), 0x00f541), - (hex!("01222222223333333344444444550000009200000000000076c0"), 0x00f5e1), - (hex!("0122222222333333334444444455000000930000000000000940"), 0x00f681), - (hex!("01222222223333333344444444550000009300000000000044e0"), 0x00f721), - (hex!("0122222222333333334444444455000000940000000000000950"), 0x00f7c1), - (hex!("0122222222333333334444444455000000940000000000007a30"), 0x00f861), - (hex!("0122222222333333334444444455000000950000000000000960"), 0x00f901), - (hex!("0122222222333333334444444455000000950000000000007a70"), 0x00f9a1), - (hex!("0122222222333333334444444455000000960000000000000970"), 0x00fa41), - (hex!("0122222222333333334444444455000000970000000000000980"), 0x00fae1), - (hex!("0122222222333333334444444455000000970000000000007330"), 0x00fb81), - (hex!("0122222222333333334444444455000000980000000000000990"), 0x00fc21), - (hex!("0122222222333333334444444455000000980000000000005af0"), 0x00fcc1), - (hex!("0122222222333333334444444455000000980000000000007ae0"), 0x00fd61), - (hex!("01222222223333333344444444550000009900000000000009a0"), 0x00fe01), - (hex!("0122222222333333334444444455000000990000000000005160"), 0x00fea1), - (hex!("0122222222333333334444444455000000990000000000006850"), 0x00ff41), - (hex!("01222222223333333344444444550000009a00000000000009b0"), 0x00ffe1), - (hex!("01222222223333333344444444550000009b00000000000009c0"), 0x010081), - (hex!("01222222223333333344444444550000009b0000000000005010"), 0x010121), - (hex!("01222222223333333344444444550000009c00000000000009d0"), 0x0101c1), - (hex!("01222222223333333344444444550000009c00000000000042e0"), 0x010261), - (hex!("01222222223333333344444444550000009d00000000000009e0"), 0x010301), - (hex!("01222222223333333344444444550000009d00000000000057f0"), 0x0103a1), - (hex!("01222222223333333344444444550000009e00000000000009f0"), 0x010441), - (hex!("01222222223333333344444444550000009e0000000000004ef0"), 0x0104e1), - (hex!("01222222223333333344444444550000009f0000000000000a00"), 0x010581), - (hex!("01222222223333333344444444550000009f0000000000006110"), 0x010621), - (hex!("0122222222333333334444444455000000a00000000000000a10"), 0x0106c1), - (hex!("0122222222333333334444444455000000a10000000000000a20"), 0x010761), - (hex!("0122222222333333334444444455000000a100000000000040d0"), 0x010801), - (hex!("0122222222333333334444444455000000a10000000000007670"), 0x0108a1), - (hex!("0122222222333333334444444455000000a20000000000000a30"), 0x010941), - (hex!("0122222222333333334444444455000000a200000000000074d0"), 0x0109e1), - (hex!("0122222222333333334444444455000000a30000000000000a40"), 0x010a81), - (hex!("0122222222333333334444444455000000a30000000000004c90"), 0x010b21), - (hex!("0122222222333333334444444455000000a40000000000000a50"), 0x010bc1), - (hex!("0122222222333333334444444455000000a50000000000000a60"), 0x010c61), - (hex!("0122222222333333334444444455000000a60000000000000a70"), 0x010d01), - (hex!("0122222222333333334444444455000000a60000000000006d80"), 0x010da1), - (hex!("0122222222333333334444444455000000a60000000000007830"), 0x010e41), - (hex!("0122222222333333334444444455000000a70000000000000a80"), 0x010ee1), - (hex!("0122222222333333334444444455000000a700000000000064f0"), 0x010f81), - (hex!("0122222222333333334444444455000000a80000000000000a90"), 0x011021), - (hex!("0122222222333333334444444455000000a90000000000000aa0"), 0x0110c1), - (hex!("0122222222333333334444444455000000a90000000000005e30"), 0x011161), - (hex!("0122222222333333334444444455000000aa0000000000000ab0"), 0x011201), - (hex!("0122222222333333334444444455000000ab0000000000000ac0"), 0x0112a1), - (hex!("0122222222333333334444444455000000ac0000000000000ad0"), 0x011341), - (hex!("0122222222333333334444444455000000ac0000000000006d20"), 0x0113e1), - (hex!("0122222222333333334444444455000000ac0000000000007000"), 0x011481), - (hex!("0122222222333333334444444455000000ad0000000000000ae0"), 0x011521), - (hex!("0122222222333333334444444455000000ae0000000000000af0"), 0x0115c1), - (hex!("0122222222333333334444444455000000ae0000000000004a10"), 0x011661), - (hex!("0122222222333333334444444455000000af0000000000000b00"), 0x011701), - (hex!("0122222222333333334444444455000000af0000000000004e10"), 0x0117a1), - (hex!("0122222222333333334444444455000000b00000000000000b10"), 0x011841), - (hex!("0122222222333333334444444455000000b00000000000004280"), 0x0118e1), - (hex!("0122222222333333334444444455000000b000000000000077e0"), 0x011981), - (hex!("0122222222333333334444444455000000b10000000000000b20"), 0x011a21), - (hex!("0122222222333333334444444455000000b20000000000000b30"), 0x011ac1), - (hex!("0122222222333333334444444455000000b30000000000000b40"), 0x011b61), - (hex!("0122222222333333334444444455000000b30000000000004bc0"), 0x011c01), - (hex!("0122222222333333334444444455000000b40000000000000b50"), 0x011ca1), - (hex!("0122222222333333334444444455000000b50000000000000b60"), 0x011d41), - (hex!("0122222222333333334444444455000000b50000000000004fa0"), 0x011de1), - (hex!("0122222222333333334444444455000000b50000000000006a60"), 0x011e81), - (hex!("0122222222333333334444444455000000b60000000000000b70"), 0x011f21), - (hex!("0122222222333333334444444455000000b60000000000005630"), 0x011fc1), - (hex!("0122222222333333334444444455000000b70000000000000b80"), 0x012061), - (hex!("0122222222333333334444444455000000b80000000000000b90"), 0x012101), - (hex!("0122222222333333334444444455000000b80000000000006f80"), 0x0121a1), - (hex!("0122222222333333334444444455000000b90000000000000ba0"), 0x012241), - (hex!("0122222222333333334444444455000000ba0000000000000bb0"), 0x0122e1), - (hex!("0122222222333333334444444455000000bb0000000000000bc0"), 0x012381), - (hex!("0122222222333333334444444455000000bb00000000000047c0"), 0x012421), - (hex!("0122222222333333334444444455000000bb0000000000006060"), 0x0124c1), - (hex!("0122222222333333334444444455000000bc0000000000000bd0"), 0x012561), - (hex!("0122222222333333334444444455000000bd0000000000000be0"), 0x012601), - (hex!("0122222222333333334444444455000000bd0000000000004e80"), 0x0126a1), - (hex!("0122222222333333334444444455000000be0000000000000bf0"), 0x012741), - (hex!("0122222222333333334444444455000000bf0000000000000c00"), 0x0127e1), - (hex!("0122222222333333334444444455000000bf00000000000047a0"), 0x012881), - (hex!("0122222222333333334444444455000000bf0000000000006da0"), 0x012921), - (hex!("0122222222333333334444444455000000c00000000000000c10"), 0x0129c1), - (hex!("0122222222333333334444444455000000c10000000000000c20"), 0x012a61), - (hex!("0122222222333333334444444455000000c20000000000000c30"), 0x012b01), - (hex!("0122222222333333334444444455000000c20000000000004bd0"), 0x012ba1), - (hex!("0122222222333333334444444455000000c20000000000006ac0"), 0x012c41), - (hex!("0122222222333333334444444455000000c30000000000000c40"), 0x012ce1), - (hex!("0122222222333333334444444455000000c30000000000004660"), 0x012d81), - (hex!("0122222222333333334444444455000000c40000000000000c50"), 0x012e21), - (hex!("0122222222333333334444444455000000c50000000000000c60"), 0x012ec1), - (hex!("0122222222333333334444444455000000c60000000000000c70"), 0x012f61), - (hex!("0122222222333333334444444455000000c60000000000005880"), 0x013001), - (hex!("0122222222333333334444444455000000c60000000000006b70"), 0x0130a1), - (hex!("0122222222333333334444444455000000c70000000000000c80"), 0x013141), - (hex!("0122222222333333334444444455000000c80000000000000c90"), 0x0131e1), - (hex!("0122222222333333334444444455000000c80000000000005310"), 0x013281), - (hex!("0122222222333333334444444455000000c80000000000005db0"), 0x013321), - (hex!("0122222222333333334444444455000000c80000000000007040"), 0x0133c1), - (hex!("0122222222333333334444444455000000c80000000000007290"), 0x013461), - (hex!("0122222222333333334444444455000000c90000000000000ca0"), 0x013501), - (hex!("0122222222333333334444444455000000c90000000000004fe0"), 0x0135a1), - (hex!("0122222222333333334444444455000000ca0000000000000cb0"), 0x013641), - (hex!("0122222222333333334444444455000000ca0000000000006140"), 0x0136e1), - (hex!("0122222222333333334444444455000000ca0000000000007700"), 0x013781), - (hex!("0122222222333333334444444455000000cb0000000000000cc0"), 0x013821), - (hex!("0122222222333333334444444455000000cc0000000000000cd0"), 0x0138c1), - (hex!("0122222222333333334444444455000000cd0000000000000ce0"), 0x013961), - (hex!("0122222222333333334444444455000000cd0000000000003f20"), 0x013a01), - (hex!("0122222222333333334444444455000000cd00000000000040f0"), 0x013aa1), - (hex!("0122222222333333334444444455000000cd0000000000004ec0"), 0x013b41), - (hex!("0122222222333333334444444455000000ce0000000000000cf0"), 0x013be1), - (hex!("0122222222333333334444444455000000ce0000000000007200"), 0x013c81), - (hex!("0122222222333333334444444455000000cf0000000000000d00"), 0x013d21), - (hex!("0122222222333333334444444455000000cf00000000000046a0"), 0x013dc1), - (hex!("0122222222333333334444444455000000cf0000000000005960"), 0x013e61), - (hex!("0122222222333333334444444455000000d00000000000000d10"), 0x013f01), - (hex!("0122222222333333334444444455000000d00000000000005f30"), 0x013fa1), - (hex!("0122222222333333334444444455000000d10000000000000d20"), 0x014041), - (hex!("0122222222333333334444444455000000d10000000000007a00"), 0x0140e1), - (hex!("0122222222333333334444444455000000d20000000000000d30"), 0x014181), - (hex!("0122222222333333334444444455000000d30000000000000d40"), 0x014221), - (hex!("0122222222333333334444444455000000d40000000000000d50"), 0x0142c1), - (hex!("0122222222333333334444444455000000d50000000000000d60"), 0x014361), - (hex!("0122222222333333334444444455000000d50000000000004960"), 0x014401), - (hex!("0122222222333333334444444455000000d500000000000055d0"), 0x0144a1), - (hex!("0122222222333333334444444455000000d500000000000067d0"), 0x014541), - (hex!("0122222222333333334444444455000000d60000000000000d70"), 0x0145e1), - (hex!("0122222222333333334444444455000000d70000000000000d80"), 0x014681), - (hex!("0122222222333333334444444455000000d80000000000000d90"), 0x014721), - (hex!("0122222222333333334444444455000000d800000000000065f0"), 0x0147c1), - (hex!("0122222222333333334444444455000000d90000000000000da0"), 0x014861), - (hex!("0122222222333333334444444455000000d90000000000004980"), 0x014901), - (hex!("0122222222333333334444444455000000da0000000000000db0"), 0x0149a1), - (hex!("0122222222333333334444444455000000da00000000000048c0"), 0x014a41), - (hex!("0122222222333333334444444455000000da00000000000072c0"), 0x014ae1), - (hex!("0122222222333333334444444455000000da00000000000076b0"), 0x014b81), - (hex!("0122222222333333334444444455000000db0000000000000dc0"), 0x014c21), - (hex!("0122222222333333334444444455000000dc0000000000000dd0"), 0x014cc1), - (hex!("0122222222333333334444444455000000dc00000000000040a0"), 0x014d61), - (hex!("0122222222333333334444444455000000dc00000000000074c0"), 0x014e01), - (hex!("0122222222333333334444444455000000dd0000000000000de0"), 0x014ea1), - (hex!("0122222222333333334444444455000000dd0000000000004e50"), 0x014f41), - (hex!("0122222222333333334444444455000000dd0000000000007270"), 0x014fe1), - (hex!("0122222222333333334444444455000000de0000000000000df0"), 0x015081), - (hex!("0122222222333333334444444455000000de00000000000078d0"), 0x015121), - (hex!("0122222222333333334444444455000000df0000000000000e00"), 0x0151c1), - (hex!("0122222222333333334444444455000000df0000000000004d30"), 0x015261), - (hex!("0122222222333333334444444455000000df0000000000006c30"), 0x015301), - (hex!("0122222222333333334444444455000000e00000000000000e10"), 0x0153a1), - (hex!("0122222222333333334444444455000000e00000000000005d30"), 0x015441), - (hex!("0122222222333333334444444455000000e10000000000000e20"), 0x0154e1), - (hex!("0122222222333333334444444455000000e10000000000004610"), 0x015581), - (hex!("0122222222333333334444444455000000e100000000000051d0"), 0x015621), - (hex!("0122222222333333334444444455000000e10000000000005f10"), 0x0156c1), - (hex!("0122222222333333334444444455000000e20000000000000e30"), 0x015761), - (hex!("0122222222333333334444444455000000e20000000000007a90"), 0x015801), - (hex!("0122222222333333334444444455000000e30000000000000e40"), 0x0158a1), - (hex!("0122222222333333334444444455000000e30000000000005ae0"), 0x015941), - (hex!("0122222222333333334444444455000000e40000000000000e50"), 0x0159e1), - (hex!("0122222222333333334444444455000000e50000000000000e60"), 0x015a81), - (hex!("0122222222333333334444444455000000e50000000000004700"), 0x015b21), - (hex!("0122222222333333334444444455000000e500000000000065d0"), 0x015bc1), - (hex!("0122222222333333334444444455000000e60000000000000e70"), 0x015c61), - (hex!("0122222222333333334444444455000000e60000000000004fd0"), 0x015d01), - (hex!("0122222222333333334444444455000000e70000000000000e80"), 0x015da1), - (hex!("0122222222333333334444444455000000e70000000000005150"), 0x015e41), - (hex!("0122222222333333334444444455000000e70000000000005920"), 0x015ee1), - (hex!("0122222222333333334444444455000000e80000000000000e90"), 0x015f81), - (hex!("0122222222333333334444444455000000e80000000000004320"), 0x016021), - (hex!("0122222222333333334444444455000000e80000000000005ec0"), 0x0160c1), - (hex!("0122222222333333334444444455000000e90000000000000ea0"), 0x016161), - (hex!("0122222222333333334444444455000000e900000000000043b0"), 0x016201), - (hex!("0122222222333333334444444455000000ea0000000000000eb0"), 0x0162a1), - (hex!("0122222222333333334444444455000000ea0000000000003ea0"), 0x016341), - (hex!("0122222222333333334444444455000000ea0000000000004f50"), 0x0163e1), - (hex!("0122222222333333334444444455000000ea0000000000007520"), 0x016481), - (hex!("0122222222333333334444444455000000eb0000000000000ec0"), 0x016521), - (hex!("0122222222333333334444444455000000ec0000000000000ed0"), 0x0165c1), - (hex!("0122222222333333334444444455000000ec0000000000006670"), 0x016661), - (hex!("0122222222333333334444444455000000ed0000000000000ee0"), 0x016701), - (hex!("0122222222333333334444444455000000ee0000000000000ef0"), 0x0167a1), - (hex!("0122222222333333334444444455000000ee0000000000004d10"), 0x016841), - (hex!("0122222222333333334444444455000000ef0000000000000f00"), 0x0168e1), - (hex!("0122222222333333334444444455000000f00000000000000f10"), 0x016981), - (hex!("0122222222333333334444444455000000f00000000000007220"), 0x016a21), - (hex!("0122222222333333334444444455000000f00000000000007540"), 0x016ac1), - (hex!("0122222222333333334444444455000000f10000000000000f20"), 0x016b61), - (hex!("0122222222333333334444444455000000f100000000000066f0"), 0x016c01), - (hex!("0122222222333333334444444455000000f20000000000000f30"), 0x016ca1), - (hex!("0122222222333333334444444455000000f20000000000007810"), 0x016d41), - (hex!("0122222222333333334444444455000000f30000000000000f40"), 0x016de1), - (hex!("0122222222333333334444444455000000f30000000000007b70"), 0x016e81), - (hex!("0122222222333333334444444455000000f40000000000000f50"), 0x016f21), - (hex!("0122222222333333334444444455000000f400000000000059c0"), 0x016fc1), - (hex!("0122222222333333334444444455000000f50000000000000f60"), 0x017061), - (hex!("0122222222333333334444444455000000f50000000000003fb0"), 0x017101), - (hex!("0122222222333333334444444455000000f50000000000005740"), 0x0171a1), - (hex!("0122222222333333334444444455000000f500000000000064d0"), 0x017241), - (hex!("0122222222333333334444444455000000f50000000000006960"), 0x0172e1), - (hex!("0122222222333333334444444455000000f60000000000000f70"), 0x017381), - (hex!("0122222222333333334444444455000000f60000000000006d00"), 0x017421), - (hex!("0122222222333333334444444455000000f70000000000000f80"), 0x0174c1), - (hex!("0122222222333333334444444455000000f80000000000000f90"), 0x017561), - (hex!("0122222222333333334444444455000000f90000000000000fa0"), 0x017601), - (hex!("0122222222333333334444444455000000fa0000000000000fb0"), 0x0176a1), - (hex!("0122222222333333334444444455000000fa00000000000067b0"), 0x017741), - (hex!("0122222222333333334444444455000000fb0000000000000fc0"), 0x0177e1), - (hex!("0122222222333333334444444455000000fb0000000000004eb0"), 0x017881), - (hex!("0122222222333333334444444455000000fb0000000000006ef0"), 0x017921), - (hex!("0122222222333333334444444455000000fc0000000000000fd0"), 0x0179c1), - (hex!("0122222222333333334444444455000000fc0000000000004470"), 0x017a61), - (hex!("0122222222333333334444444455000000fc0000000000005940"), 0x017b01), - (hex!("0122222222333333334444444455000000fd0000000000000fe0"), 0x017ba1), - (hex!("0122222222333333334444444455000000fe0000000000000ff0"), 0x017c41), - (hex!("0122222222333333334444444455000000ff0000000000001000"), 0x017ce1), - (hex!("0122222222333333334444444455000000ff0000000000005690"), 0x017d81), - (hex!("0122222222333333334444444455000001000000000000001010"), 0x017e21), - (hex!("0122222222333333334444444455000001000000000000005210"), 0x017ec1), - (hex!("01222222223333333344444444550000010000000000000070a0"), 0x017f61), - (hex!("0122222222333333334444444455000001010000000000001020"), 0x018001), - (hex!("0122222222333333334444444455000001010000000000006b80"), 0x0180a1), - (hex!("0122222222333333334444444455000001020000000000001030"), 0x018141), - (hex!("0122222222333333334444444455000001030000000000001040"), 0x0181e1), - (hex!("0122222222333333334444444455000001030000000000004c80"), 0x018281), - (hex!("0122222222333333334444444455000001040000000000001050"), 0x018321), - (hex!("0122222222333333334444444455000001040000000000004850"), 0x0183c1), - (hex!("01222222223333333344444444550000010400000000000057b0"), 0x018461), - (hex!("0122222222333333334444444455000001050000000000001060"), 0x018501), - (hex!("01222222223333333344444444550000010500000000000048d0"), 0x0185a1), - (hex!("0122222222333333334444444455000001050000000000007870"), 0x018641), - (hex!("0122222222333333334444444455000001060000000000001070"), 0x0186e1), - (hex!("0122222222333333334444444455000001060000000000004f90"), 0x018781), - (hex!("0122222222333333334444444455000001060000000000006270"), 0x018821), - (hex!("0122222222333333334444444455000001070000000000001080"), 0x0188c1), - (hex!("01222222223333333344444444550000010700000000000063b0"), 0x018961), - (hex!("0122222222333333334444444455000001080000000000001090"), 0x018a01), - (hex!("01222222223333333344444444550000010900000000000010a0"), 0x018aa1), - (hex!("0122222222333333334444444455000001090000000000006f40"), 0x018b41), - (hex!("01222222223333333344444444550000010a00000000000010b0"), 0x018be1), - (hex!("01222222223333333344444444550000010a0000000000006640"), 0x018c81), - (hex!("01222222223333333344444444550000010b00000000000010c0"), 0x018d21), - (hex!("01222222223333333344444444550000010c00000000000010d0"), 0x018dc1), - (hex!("01222222223333333344444444550000010d00000000000010e0"), 0x018e61), - (hex!("01222222223333333344444444550000010e00000000000010f0"), 0x018f01), - (hex!("01222222223333333344444444550000010e0000000000005c40"), 0x018fa1), - (hex!("01222222223333333344444444550000010e0000000000007ba0"), 0x019041), - (hex!("01222222223333333344444444550000010f0000000000001100"), 0x0190e1), - (hex!("01222222223333333344444444550000010f0000000000005c30"), 0x019181), - (hex!("0122222222333333334444444455000001100000000000001110"), 0x019221), - (hex!("0122222222333333334444444455000001100000000000007640"), 0x0192c1), - (hex!("0122222222333333334444444455000001110000000000001120"), 0x019361), - (hex!("01222222223333333344444444550000011100000000000052c0"), 0x019401), - (hex!("0122222222333333334444444455000001110000000000005710"), 0x0194a1), - (hex!("0122222222333333334444444455000001110000000000006a00"), 0x019541), - (hex!("0122222222333333334444444455000001120000000000001130"), 0x0195e1), - (hex!("0122222222333333334444444455000001130000000000001140"), 0x019681), - (hex!("0122222222333333334444444455000001140000000000001150"), 0x019721), - (hex!("0122222222333333334444444455000001140000000000003fa0"), 0x0197c1), - (hex!("01222222223333333344444444550000011400000000000054b0"), 0x019861), - (hex!("0122222222333333334444444455000001140000000000006070"), 0x019901), - (hex!("0122222222333333334444444455000001150000000000001160"), 0x0199a1), - (hex!("0122222222333333334444444455000001150000000000005320"), 0x019a41), - (hex!("0122222222333333334444444455000001150000000000006600"), 0x019ae1), - (hex!("0122222222333333334444444455000001150000000000006df0"), 0x019b81), - (hex!("01222222223333333344444444550000011500000000000079c0"), 0x019c21), - (hex!("0122222222333333334444444455000001160000000000001170"), 0x019cc1), - (hex!("0122222222333333334444444455000001170000000000001180"), 0x019d61), - (hex!("0122222222333333334444444455000001170000000000004a60"), 0x019e01), - (hex!("01222222223333333344444444550000011700000000000063c0"), 0x019ea1), - (hex!("0122222222333333334444444455000001180000000000001190"), 0x019f41), - (hex!("0122222222333333334444444455000001180000000000004530"), 0x019fe1), - (hex!("01222222223333333344444444550000011800000000000077c0"), 0x01a081), - (hex!("01222222223333333344444444550000011900000000000011a0"), 0x01a121), - (hex!("01222222223333333344444444550000011a00000000000011b0"), 0x01a1c1), - (hex!("01222222223333333344444444550000011a00000000000041c0"), 0x01a261), - (hex!("01222222223333333344444444550000011a00000000000061e0"), 0x01a301), - (hex!("01222222223333333344444444550000011b00000000000011c0"), 0x01a3a1), - (hex!("01222222223333333344444444550000011c00000000000011d0"), 0x01a441), - (hex!("01222222223333333344444444550000011c0000000000005f90"), 0x01a4e1), - (hex!("01222222223333333344444444550000011d00000000000011e0"), 0x01a581), - (hex!("01222222223333333344444444550000011d0000000000004160"), 0x01a621), - (hex!("01222222223333333344444444550000011e00000000000011f0"), 0x01a6c1), - (hex!("01222222223333333344444444550000011e00000000000056d0"), 0x01a761), - (hex!("01222222223333333344444444550000011f0000000000001200"), 0x01a801), - (hex!("01222222223333333344444444550000011f0000000000004510"), 0x01a8a1), - (hex!("0122222222333333334444444455000001200000000000001210"), 0x01a941), - (hex!("0122222222333333334444444455000001210000000000001220"), 0x01a9e1), - (hex!("0122222222333333334444444455000001210000000000005140"), 0x01aa81), - (hex!("0122222222333333334444444455000001210000000000006710"), 0x01ab21), - (hex!("0122222222333333334444444455000001210000000000006f50"), 0x01abc1), - (hex!("0122222222333333334444444455000001220000000000001230"), 0x01ac61), - (hex!("0122222222333333334444444455000001220000000000005570"), 0x01ad01), - (hex!("0122222222333333334444444455000001220000000000007ac0"), 0x01ada1), - (hex!("0122222222333333334444444455000001230000000000001240"), 0x01ae41), - (hex!("0122222222333333334444444455000001240000000000001250"), 0x01aee1), - (hex!("0122222222333333334444444455000001240000000000006cd0"), 0x01af81), - (hex!("0122222222333333334444444455000001250000000000001260"), 0x01b021), - (hex!("01222222223333333344444444550000012500000000000046b0"), 0x01b0c1), - (hex!("0122222222333333334444444455000001250000000000005eb0"), 0x01b161), - (hex!("0122222222333333334444444455000001260000000000001270"), 0x01b201), - (hex!("0122222222333333334444444455000001260000000000004630"), 0x01b2a1), - (hex!("0122222222333333334444444455000001270000000000001280"), 0x01b341), - (hex!("0122222222333333334444444455000001270000000000004ff0"), 0x01b3e1), - (hex!("0122222222333333334444444455000001270000000000006ec0"), 0x01b481), - (hex!("0122222222333333334444444455000001280000000000001290"), 0x01b521), - (hex!("01222222223333333344444444550000012900000000000012a0"), 0x01b5c1), - (hex!("0122222222333333334444444455000001290000000000005f60"), 0x01b661), - (hex!("01222222223333333344444444550000012a00000000000012b0"), 0x01b701), - (hex!("01222222223333333344444444550000012a0000000000005480"), 0x01b7a1), - (hex!("01222222223333333344444444550000012b00000000000012c0"), 0x01b841), - (hex!("01222222223333333344444444550000012b00000000000065a0"), 0x01b8e1), - (hex!("01222222223333333344444444550000012b00000000000066c0"), 0x01b981), - (hex!("01222222223333333344444444550000012c00000000000012d0"), 0x01ba21), - (hex!("01222222223333333344444444550000012c00000000000064b0"), 0x01bac1), - (hex!("01222222223333333344444444550000012d00000000000012e0"), 0x01bb61), - (hex!("01222222223333333344444444550000012d00000000000049c0"), 0x01bc01), - (hex!("01222222223333333344444444550000012d0000000000004bf0"), 0x01bca1), - (hex!("01222222223333333344444444550000012e00000000000012f0"), 0x01bd41), - (hex!("01222222223333333344444444550000012e0000000000005ed0"), 0x01bde1), - (hex!("01222222223333333344444444550000012f0000000000001300"), 0x01be81), - (hex!("01222222223333333344444444550000012f00000000000049a0"), 0x01bf21), - (hex!("0122222222333333334444444455000001300000000000001310"), 0x01bfc1), - (hex!("0122222222333333334444444455000001300000000000007840"), 0x01c061), - (hex!("0122222222333333334444444455000001310000000000001320"), 0x01c101), - (hex!("0122222222333333334444444455000001310000000000005f70"), 0x01c1a1), - (hex!("0122222222333333334444444455000001320000000000001330"), 0x01c241), - (hex!("0122222222333333334444444455000001320000000000005a00"), 0x01c2e1), - (hex!("0122222222333333334444444455000001330000000000001340"), 0x01c381), - (hex!("0122222222333333334444444455000001330000000000006c70"), 0x01c421), - (hex!("0122222222333333334444444455000001340000000000001350"), 0x01c4c1), - (hex!("0122222222333333334444444455000001340000000000005c60"), 0x01c561), - (hex!("0122222222333333334444444455000001350000000000001360"), 0x01c601), - (hex!("0122222222333333334444444455000001350000000000004f10"), 0x01c6a1), - (hex!("0122222222333333334444444455000001360000000000001370"), 0x01c741), - (hex!("0122222222333333334444444455000001360000000000004c60"), 0x01c7e1), - (hex!("0122222222333333334444444455000001370000000000001380"), 0x01c881), - (hex!("0122222222333333334444444455000001380000000000001390"), 0x01c921), - (hex!("01222222223333333344444444550000013900000000000013a0"), 0x01c9c1), - (hex!("0122222222333333334444444455000001390000000000004ea0"), 0x01ca61), - (hex!("01222222223333333344444444550000013a00000000000013b0"), 0x01cb01), - (hex!("01222222223333333344444444550000013a0000000000007350"), 0x01cba1), - (hex!("01222222223333333344444444550000013b00000000000013c0"), 0x01cc41), - (hex!("01222222223333333344444444550000013c00000000000013d0"), 0x01cce1), - (hex!("01222222223333333344444444550000013c0000000000007050"), 0x01cd81), - (hex!("01222222223333333344444444550000013d00000000000013e0"), 0x01ce21), - (hex!("01222222223333333344444444550000013d0000000000006bd0"), 0x01cec1), - (hex!("01222222223333333344444444550000013e00000000000013f0"), 0x01cf61), - (hex!("01222222223333333344444444550000013e00000000000058e0"), 0x01d001), - (hex!("01222222223333333344444444550000013f0000000000001400"), 0x01d0a1), - (hex!("01222222223333333344444444550000013f0000000000004740"), 0x01d141), - (hex!("0122222222333333334444444455000001400000000000001410"), 0x01d1e1), - (hex!("0122222222333333334444444455000001400000000000003f10"), 0x01d281), - (hex!("0122222222333333334444444455000001400000000000006d40"), 0x01d321), - (hex!("01222222223333333344444444550000014000000000000072d0"), 0x01d3c1), - (hex!("0122222222333333334444444455000001410000000000001420"), 0x01d461), - (hex!("0122222222333333334444444455000001420000000000001430"), 0x01d501), - (hex!("0122222222333333334444444455000001430000000000001440"), 0x01d5a1), - (hex!("0122222222333333334444444455000001440000000000001450"), 0x01d641), - (hex!("0122222222333333334444444455000001450000000000001460"), 0x01d6e1), - (hex!("0122222222333333334444444455000001460000000000001470"), 0x01d781), - (hex!("01222222223333333344444444550000014600000000000055c0"), 0x01d821), - (hex!("0122222222333333334444444455000001470000000000001480"), 0x01d8c1), - (hex!("0122222222333333334444444455000001470000000000004570"), 0x01d961), - (hex!("0122222222333333334444444455000001470000000000004be0"), 0x01da01), - (hex!("0122222222333333334444444455000001480000000000001490"), 0x01daa1), - (hex!("0122222222333333334444444455000001480000000000005360"), 0x01db41), - (hex!("01222222223333333344444444550000014900000000000014a0"), 0x01dbe1), - (hex!("01222222223333333344444444550000014a00000000000014b0"), 0x01dc81), - (hex!("01222222223333333344444444550000014a00000000000053d0"), 0x01dd21), - (hex!("01222222223333333344444444550000014b00000000000014c0"), 0x01ddc1), - (hex!("01222222223333333344444444550000014b0000000000005950"), 0x01de61), - (hex!("01222222223333333344444444550000014c00000000000014d0"), 0x01df01), - (hex!("01222222223333333344444444550000014c0000000000004f60"), 0x01dfa1), - (hex!("01222222223333333344444444550000014d00000000000014e0"), 0x01e041), - (hex!("01222222223333333344444444550000014d0000000000004520"), 0x01e0e1), - (hex!("01222222223333333344444444550000014d0000000000005200"), 0x01e181), - (hex!("01222222223333333344444444550000014e00000000000014f0"), 0x01e221), - (hex!("01222222223333333344444444550000014e0000000000005bd0"), 0x01e2c1), - (hex!("01222222223333333344444444550000014f0000000000001500"), 0x01e361), - (hex!("01222222223333333344444444550000014f00000000000060d0"), 0x01e401), - (hex!("0122222222333333334444444455000001500000000000001510"), 0x01e4a1), - (hex!("01222222223333333344444444550000015000000000000075e0"), 0x01e541), - (hex!("0122222222333333334444444455000001510000000000001520"), 0x01e5e1), - (hex!("0122222222333333334444444455000001510000000000005c00"), 0x01e681), - (hex!("0122222222333333334444444455000001510000000000006af0"), 0x01e721), - (hex!("0122222222333333334444444455000001510000000000007b80"), 0x01e7c1), - (hex!("0122222222333333334444444455000001520000000000001530"), 0x01e861), - (hex!("0122222222333333334444444455000001520000000000004c70"), 0x01e901), - (hex!("0122222222333333334444444455000001530000000000001540"), 0x01e9a1), - (hex!("0122222222333333334444444455000001540000000000001550"), 0x01ea41), - (hex!("0122222222333333334444444455000001540000000000007cd0"), 0x01eae1), - (hex!("0122222222333333334444444455000001550000000000001560"), 0x01eb81), - (hex!("0122222222333333334444444455000001550000000000004ae0"), 0x01ec21), - (hex!("01222222223333333344444444550000015500000000000068c0"), 0x01ecc1), - (hex!("0122222222333333334444444455000001560000000000001570"), 0x01ed61), - (hex!("01222222223333333344444444550000015600000000000064a0"), 0x01ee01), - (hex!("0122222222333333334444444455000001570000000000001580"), 0x01eea1), - (hex!("0122222222333333334444444455000001580000000000001590"), 0x01ef41), - (hex!("0122222222333333334444444455000001580000000000006d30"), 0x01efe1), - (hex!("01222222223333333344444444550000015800000000000074f0"), 0x01f081), - (hex!("01222222223333333344444444550000015900000000000015a0"), 0x01f121), - (hex!("01222222223333333344444444550000015900000000000053a0"), 0x01f1c1), - (hex!("01222222223333333344444444550000015900000000000055e0"), 0x01f261), - (hex!("0122222222333333334444444455000001590000000000006210"), 0x01f301), - (hex!("01222222223333333344444444550000015900000000000067c0"), 0x01f3a1), - (hex!("01222222223333333344444444550000015a00000000000015b0"), 0x01f441), - (hex!("01222222223333333344444444550000015b00000000000015c0"), 0x01f4e1), - (hex!("01222222223333333344444444550000015c00000000000015d0"), 0x01f581), - (hex!("01222222223333333344444444550000015c0000000000004d80"), 0x01f621), - (hex!("01222222223333333344444444550000015c00000000000073f0"), 0x01f6c1), - (hex!("01222222223333333344444444550000015d00000000000015e0"), 0x01f761), - (hex!("01222222223333333344444444550000015e00000000000015f0"), 0x01f801), - (hex!("01222222223333333344444444550000015e0000000000004120"), 0x01f8a1), - (hex!("01222222223333333344444444550000015e0000000000004350"), 0x01f941), - (hex!("01222222223333333344444444550000015e0000000000007c50"), 0x01f9e1), - (hex!("01222222223333333344444444550000015f0000000000001600"), 0x01fa81), - (hex!("0122222222333333334444444455000001600000000000001610"), 0x01fb21), - (hex!("0122222222333333334444444455000001600000000000004840"), 0x01fbc1), - (hex!("0122222222333333334444444455000001600000000000004b10"), 0x01fc61), - (hex!("0122222222333333334444444455000001600000000000007060"), 0x01fd01), - (hex!("0122222222333333334444444455000001610000000000001620"), 0x01fda1), - (hex!("0122222222333333334444444455000001610000000000005300"), 0x01fe41), - (hex!("0122222222333333334444444455000001620000000000001630"), 0x01fee1), - (hex!("0122222222333333334444444455000001620000000000006530"), 0x01ff81), - (hex!("0122222222333333334444444455000001630000000000001640"), 0x020021), - (hex!("0122222222333333334444444455000001640000000000001650"), 0x0200c1), - (hex!("0122222222333333334444444455000001650000000000001660"), 0x020161), - (hex!("0122222222333333334444444455000001660000000000001670"), 0x020201), - (hex!("0122222222333333334444444455000001670000000000001680"), 0x0202a1), - (hex!("0122222222333333334444444455000001670000000000007310"), 0x020341), - (hex!("0122222222333333334444444455000001680000000000001690"), 0x0203e1), - (hex!("0122222222333333334444444455000001680000000000007b50"), 0x020481), - (hex!("01222222223333333344444444550000016900000000000016a0"), 0x020521), - (hex!("01222222223333333344444444550000016900000000000049d0"), 0x0205c1), - (hex!("01222222223333333344444444550000016a00000000000016b0"), 0x020661), - (hex!("01222222223333333344444444550000016a00000000000078b0"), 0x020701), - (hex!("01222222223333333344444444550000016b00000000000016c0"), 0x0207a1), - (hex!("01222222223333333344444444550000016b0000000000004100"), 0x020841), - (hex!("01222222223333333344444444550000016c00000000000016d0"), 0x0208e1), - (hex!("01222222223333333344444444550000016c0000000000006e00"), 0x020981), - (hex!("01222222223333333344444444550000016d00000000000016e0"), 0x020a21), - (hex!("01222222223333333344444444550000016e00000000000016f0"), 0x020ac1), - (hex!("01222222223333333344444444550000016e0000000000004ac0"), 0x020b61), - (hex!("01222222223333333344444444550000016e0000000000007820"), 0x020c01), - (hex!("01222222223333333344444444550000016f0000000000001700"), 0x020ca1), - (hex!("0122222222333333334444444455000001700000000000001710"), 0x020d41), - (hex!("0122222222333333334444444455000001700000000000005830"), 0x020de1), - (hex!("0122222222333333334444444455000001710000000000001720"), 0x020e81), - (hex!("01222222223333333344444444550000017100000000000072f0"), 0x020f21), - (hex!("0122222222333333334444444455000001720000000000001730"), 0x020fc1), - (hex!("0122222222333333334444444455000001720000000000004870"), 0x021061), - (hex!("01222222223333333344444444550000017200000000000070b0"), 0x021101), - (hex!("0122222222333333334444444455000001730000000000001740"), 0x0211a1), - (hex!("0122222222333333334444444455000001740000000000001750"), 0x021241), - (hex!("0122222222333333334444444455000001750000000000001760"), 0x0212e1), - (hex!("0122222222333333334444444455000001750000000000005670"), 0x021381), - (hex!("0122222222333333334444444455000001750000000000005870"), 0x021421), - (hex!("0122222222333333334444444455000001760000000000001770"), 0x0214c1), - (hex!("0122222222333333334444444455000001770000000000001780"), 0x021561), - (hex!("0122222222333333334444444455000001770000000000005000"), 0x021601), - (hex!("0122222222333333334444444455000001770000000000007090"), 0x0216a1), - (hex!("0122222222333333334444444455000001780000000000001790"), 0x021741), - (hex!("01222222223333333344444444550000017800000000000048a0"), 0x0217e1), - (hex!("0122222222333333334444444455000001780000000000006bf0"), 0x021881), - (hex!("01222222223333333344444444550000017900000000000017a0"), 0x021921), - (hex!("01222222223333333344444444550000017900000000000057d0"), 0x0219c1), - (hex!("0122222222333333334444444455000001790000000000006660"), 0x021a61), - (hex!("01222222223333333344444444550000017a00000000000017b0"), 0x021b01), - (hex!("01222222223333333344444444550000017a0000000000004970"), 0x021ba1), - (hex!("01222222223333333344444444550000017a0000000000005dc0"), 0x021c41), - (hex!("01222222223333333344444444550000017b00000000000017c0"), 0x021ce1), - (hex!("01222222223333333344444444550000017b0000000000004ee0"), 0x021d81), - (hex!("01222222223333333344444444550000017b00000000000054c0"), 0x021e21), - (hex!("01222222223333333344444444550000017c00000000000017d0"), 0x021ec1), - (hex!("01222222223333333344444444550000017c0000000000003fc0"), 0x021f61), - (hex!("01222222223333333344444444550000017c00000000000063e0"), 0x022001), - (hex!("01222222223333333344444444550000017c0000000000006520"), 0x0220a1), - (hex!("01222222223333333344444444550000017d00000000000017e0"), 0x022141), - (hex!("01222222223333333344444444550000017d0000000000006220"), 0x0221e1), - (hex!("01222222223333333344444444550000017d0000000000007120"), 0x022281), - (hex!("01222222223333333344444444550000017e00000000000017f0"), 0x022321), - (hex!("01222222223333333344444444550000017f0000000000001800"), 0x0223c1), - (hex!("0122222222333333334444444455000001800000000000001810"), 0x022461), - (hex!("0122222222333333334444444455000001810000000000001820"), 0x022501), - (hex!("01222222223333333344444444550000018100000000000041f0"), 0x0225a1), - (hex!("0122222222333333334444444455000001810000000000007590"), 0x022641), - (hex!("0122222222333333334444444455000001820000000000001830"), 0x0226e1), - (hex!("0122222222333333334444444455000001820000000000004ce0"), 0x022781), - (hex!("0122222222333333334444444455000001830000000000001840"), 0x022821), - (hex!("01222222223333333344444444550000018300000000000042c0"), 0x0228c1), - (hex!("0122222222333333334444444455000001840000000000001850"), 0x022961), - (hex!("0122222222333333334444444455000001840000000000004f70"), 0x022a01), - (hex!("0122222222333333334444444455000001850000000000001860"), 0x022aa1), - (hex!("0122222222333333334444444455000001850000000000006470"), 0x022b41), - (hex!("0122222222333333334444444455000001850000000000007500"), 0x022be1), - (hex!("0122222222333333334444444455000001860000000000001870"), 0x022c81), - (hex!("0122222222333333334444444455000001860000000000004770"), 0x022d21), - (hex!("0122222222333333334444444455000001870000000000001880"), 0x022dc1), - (hex!("0122222222333333334444444455000001870000000000006a30"), 0x022e61), - (hex!("0122222222333333334444444455000001880000000000001890"), 0x022f01), - (hex!("0122222222333333334444444455000001880000000000007410"), 0x022fa1), - (hex!("01222222223333333344444444550000018900000000000018a0"), 0x023041), - (hex!("01222222223333333344444444550000018900000000000044d0"), 0x0230e1), - (hex!("0122222222333333334444444455000001890000000000005ac0"), 0x023181), - (hex!("01222222223333333344444444550000018a00000000000018b0"), 0x023221), - (hex!("01222222223333333344444444550000018a0000000000006260"), 0x0232c1), - (hex!("01222222223333333344444444550000018a0000000000006d70"), 0x023361), - (hex!("01222222223333333344444444550000018b00000000000018c0"), 0x023401), - (hex!("01222222223333333344444444550000018b0000000000004aa0"), 0x0234a1), - (hex!("01222222223333333344444444550000018b0000000000006fd0"), 0x023541), - (hex!("01222222223333333344444444550000018c00000000000018d0"), 0x0235e1), - (hex!("01222222223333333344444444550000018c00000000000051b0"), 0x023681), - (hex!("01222222223333333344444444550000018c0000000000006650"), 0x023721), - (hex!("01222222223333333344444444550000018d00000000000018e0"), 0x0237c1), - (hex!("01222222223333333344444444550000018e00000000000018f0"), 0x023861), - (hex!("01222222223333333344444444550000018e00000000000041d0"), 0x023901), - (hex!("01222222223333333344444444550000018f0000000000001900"), 0x0239a1), - (hex!("01222222223333333344444444550000018f0000000000007600"), 0x023a41), - (hex!("0122222222333333334444444455000001900000000000001910"), 0x023ae1), - (hex!("0122222222333333334444444455000001900000000000005410"), 0x023b81), - (hex!("0122222222333333334444444455000001900000000000006760"), 0x023c21), - (hex!("0122222222333333334444444455000001910000000000001920"), 0x023cc1), - (hex!("0122222222333333334444444455000001920000000000001930"), 0x023d61), - (hex!("0122222222333333334444444455000001920000000000004ca0"), 0x023e01), - (hex!("0122222222333333334444444455000001920000000000005d80"), 0x023ea1), - (hex!("0122222222333333334444444455000001920000000000005fd0"), 0x023f41), - (hex!("01222222223333333344444444550000019200000000000070d0"), 0x023fe1), - (hex!("0122222222333333334444444455000001930000000000001940"), 0x024081), - (hex!("0122222222333333334444444455000001930000000000004010"), 0x024121), - (hex!("0122222222333333334444444455000001930000000000007ca0"), 0x0241c1), - (hex!("0122222222333333334444444455000001940000000000001950"), 0x024261), - (hex!("0122222222333333334444444455000001950000000000001960"), 0x024301), - (hex!("0122222222333333334444444455000001950000000000005380"), 0x0243a1), - (hex!("0122222222333333334444444455000001960000000000001970"), 0x024441), - (hex!("0122222222333333334444444455000001960000000000006de0"), 0x0244e1), - (hex!("0122222222333333334444444455000001970000000000001980"), 0x024581), - (hex!("01222222223333333344444444550000019700000000000048f0"), 0x024621), - (hex!("0122222222333333334444444455000001980000000000001990"), 0x0246c1), - (hex!("0122222222333333334444444455000001980000000000006510"), 0x024761), - (hex!("01222222223333333344444444550000019900000000000019a0"), 0x024801), - (hex!("0122222222333333334444444455000001990000000000007570"), 0x0248a1), - (hex!("0122222222333333334444444455000001990000000000007580"), 0x024941), - (hex!("01222222223333333344444444550000019a00000000000019b0"), 0x0249e1), - (hex!("01222222223333333344444444550000019a0000000000004050"), 0x024a81), - (hex!("01222222223333333344444444550000019a0000000000004ba0"), 0x024b21), - (hex!("01222222223333333344444444550000019a0000000000005540"), 0x024bc1), - (hex!("01222222223333333344444444550000019a00000000000061c0"), 0x024c61), - (hex!("01222222223333333344444444550000019a0000000000007c60"), 0x024d01), - (hex!("01222222223333333344444444550000019b00000000000019c0"), 0x024da1), - (hex!("01222222223333333344444444550000019b0000000000006240"), 0x024e41), - (hex!("01222222223333333344444444550000019c00000000000019d0"), 0x024ee1), - (hex!("01222222223333333344444444550000019d00000000000019e0"), 0x024f81), - (hex!("01222222223333333344444444550000019d0000000000004640"), 0x025021), - (hex!("01222222223333333344444444550000019d00000000000052a0"), 0x0250c1), - (hex!("01222222223333333344444444550000019d00000000000052b0"), 0x025161), - (hex!("01222222223333333344444444550000019e00000000000019f0"), 0x025201), - (hex!("01222222223333333344444444550000019f0000000000001a00"), 0x0252a1), - (hex!("01222222223333333344444444550000019f0000000000006b20"), 0x025341), - (hex!("0122222222333333334444444455000001a00000000000001a10"), 0x0253e1), - (hex!("0122222222333333334444444455000001a10000000000001a20"), 0x025481), - (hex!("0122222222333333334444444455000001a10000000000005460"), 0x025521), - (hex!("0122222222333333334444444455000001a10000000000005d20"), 0x0255c1), - (hex!("0122222222333333334444444455000001a100000000000068f0"), 0x025661), - (hex!("0122222222333333334444444455000001a20000000000001a30"), 0x025701), - (hex!("0122222222333333334444444455000001a20000000000007170"), 0x0257a1), - (hex!("0122222222333333334444444455000001a30000000000001a40"), 0x025841), - (hex!("0122222222333333334444444455000001a40000000000001a50"), 0x0258e1), - (hex!("0122222222333333334444444455000001a50000000000001a60"), 0x025981), - (hex!("0122222222333333334444444455000001a60000000000001a70"), 0x025a21), - (hex!("0122222222333333334444444455000001a70000000000001a80"), 0x025ac1), - (hex!("0122222222333333334444444455000001a70000000000005a90"), 0x025b61), - (hex!("0122222222333333334444444455000001a70000000000006440"), 0x025c01), - (hex!("0122222222333333334444444455000001a80000000000001a90"), 0x025ca1), - (hex!("0122222222333333334444444455000001a80000000000004800"), 0x025d41), - (hex!("0122222222333333334444444455000001a90000000000001aa0"), 0x025de1), - (hex!("0122222222333333334444444455000001aa0000000000001ab0"), 0x025e81), - (hex!("0122222222333333334444444455000001aa0000000000005b60"), 0x025f21), - (hex!("0122222222333333334444444455000001ab0000000000001ac0"), 0x025fc1), - (hex!("0122222222333333334444444455000001ab0000000000006700"), 0x026061), - (hex!("0122222222333333334444444455000001ab00000000000071d0"), 0x026101), - (hex!("0122222222333333334444444455000001ac0000000000001ad0"), 0x0261a1), - (hex!("0122222222333333334444444455000001ac0000000000007380"), 0x026241), - (hex!("0122222222333333334444444455000001ad0000000000001ae0"), 0x0262e1), - (hex!("0122222222333333334444444455000001ad0000000000006350"), 0x026381), - (hex!("0122222222333333334444444455000001ae0000000000001af0"), 0x026421), - (hex!("0122222222333333334444444455000001af0000000000001b00"), 0x0264c1), - (hex!("0122222222333333334444444455000001af0000000000007390"), 0x026561), - (hex!("0122222222333333334444444455000001b00000000000001b10"), 0x026601), - (hex!("0122222222333333334444444455000001b10000000000001b20"), 0x0266a1), - (hex!("0122222222333333334444444455000001b10000000000005cc0"), 0x026741), - (hex!("0122222222333333334444444455000001b20000000000001b30"), 0x0267e1), - (hex!("0122222222333333334444444455000001b20000000000004fb0"), 0x026881), - (hex!("0122222222333333334444444455000001b30000000000001b40"), 0x026921), - (hex!("0122222222333333334444444455000001b40000000000001b50"), 0x0269c1), - (hex!("0122222222333333334444444455000001b50000000000001b60"), 0x026a61), - (hex!("0122222222333333334444444455000001b60000000000001b70"), 0x026b01), - (hex!("0122222222333333334444444455000001b600000000000048e0"), 0x026ba1), - (hex!("0122222222333333334444444455000001b70000000000001b80"), 0x026c41), - (hex!("0122222222333333334444444455000001b70000000000005ca0"), 0x026ce1), - (hex!("0122222222333333334444444455000001b70000000000007900"), 0x026d81), - (hex!("0122222222333333334444444455000001b80000000000001b90"), 0x026e21), - (hex!("0122222222333333334444444455000001b80000000000004d90"), 0x026ec1), - (hex!("0122222222333333334444444455000001b90000000000001ba0"), 0x026f61), - (hex!("0122222222333333334444444455000001b90000000000003f40"), 0x027001), - (hex!("0122222222333333334444444455000001ba0000000000001bb0"), 0x0270a1), - (hex!("0122222222333333334444444455000001ba00000000000042a0"), 0x027141), - (hex!("0122222222333333334444444455000001ba00000000000067f0"), 0x0271e1), - (hex!("0122222222333333334444444455000001ba00000000000073a0"), 0x027281), - (hex!("0122222222333333334444444455000001bb0000000000001bc0"), 0x027321), - (hex!("0122222222333333334444444455000001bb0000000000004a00"), 0x0273c1), - (hex!("0122222222333333334444444455000001bb0000000000005e00"), 0x027461), - (hex!("0122222222333333334444444455000001bc0000000000001bd0"), 0x027501), - (hex!("0122222222333333334444444455000001bc0000000000004230"), 0x0275a1), - (hex!("0122222222333333334444444455000001bc0000000000005860"), 0x027641), - (hex!("0122222222333333334444444455000001bd0000000000001be0"), 0x0276e1), - (hex!("0122222222333333334444444455000001bd0000000000007c70"), 0x027781), - (hex!("0122222222333333334444444455000001be0000000000001bf0"), 0x027821), - (hex!("0122222222333333334444444455000001be0000000000007770"), 0x0278c1), - (hex!("0122222222333333334444444455000001be0000000000007cf0"), 0x027961), - (hex!("0122222222333333334444444455000001bf0000000000001c00"), 0x027a01), - (hex!("0122222222333333334444444455000001bf0000000000006490"), 0x027aa1), - (hex!("0122222222333333334444444455000001c00000000000001c10"), 0x027b41), - (hex!("0122222222333333334444444455000001c10000000000001c20"), 0x027be1), - (hex!("0122222222333333334444444455000001c10000000000004600"), 0x027c81), - (hex!("0122222222333333334444444455000001c20000000000001c30"), 0x027d21), - (hex!("0122222222333333334444444455000001c20000000000006e30"), 0x027dc1), - (hex!("0122222222333333334444444455000001c30000000000001c40"), 0x027e61), - (hex!("0122222222333333334444444455000001c40000000000001c50"), 0x027f01), - (hex!("0122222222333333334444444455000001c50000000000001c60"), 0x027fa1), - (hex!("0122222222333333334444444455000001c60000000000001c70"), 0x028041), - (hex!("0122222222333333334444444455000001c60000000000004240"), 0x0280e1), - (hex!("0122222222333333334444444455000001c60000000000005bb0"), 0x028181), - (hex!("0122222222333333334444444455000001c70000000000001c80"), 0x028221), - (hex!("0122222222333333334444444455000001c80000000000001c90"), 0x0282c1), - (hex!("0122222222333333334444444455000001c90000000000001ca0"), 0x028361), - (hex!("0122222222333333334444444455000001c90000000000006730"), 0x028401), - (hex!("0122222222333333334444444455000001ca0000000000001cb0"), 0x0284a1), - (hex!("0122222222333333334444444455000001ca00000000000070f0"), 0x028541), - (hex!("0122222222333333334444444455000001cb0000000000001cc0"), 0x0285e1), - (hex!("0122222222333333334444444455000001cb00000000000071a0"), 0x028681), - (hex!("0122222222333333334444444455000001cc0000000000001cd0"), 0x028721), - (hex!("0122222222333333334444444455000001cc0000000000005280"), 0x0287c1), - (hex!("0122222222333333334444444455000001cc0000000000005d90"), 0x028861), - (hex!("0122222222333333334444444455000001cd0000000000001ce0"), 0x028901), - (hex!("0122222222333333334444444455000001cd00000000000069b0"), 0x0289a1), - (hex!("0122222222333333334444444455000001ce0000000000001cf0"), 0x028a41), - (hex!("0122222222333333334444444455000001ce0000000000004540"), 0x028ae1), - (hex!("0122222222333333334444444455000001cf0000000000001d00"), 0x028b81), - (hex!("0122222222333333334444444455000001cf00000000000076a0"), 0x028c21), - (hex!("0122222222333333334444444455000001d00000000000001d10"), 0x028cc1), - (hex!("0122222222333333334444444455000001d000000000000060a0"), 0x028d61), - (hex!("0122222222333333334444444455000001d10000000000001d20"), 0x028e01), - (hex!("0122222222333333334444444455000001d20000000000001d30"), 0x028ea1), - (hex!("0122222222333333334444444455000001d30000000000001d40"), 0x028f41), - (hex!("0122222222333333334444444455000001d30000000000004000"), 0x028fe1), - (hex!("0122222222333333334444444455000001d30000000000004140"), 0x029081), - (hex!("0122222222333333334444444455000001d30000000000006790"), 0x029121), - (hex!("0122222222333333334444444455000001d40000000000001d50"), 0x0291c1), - (hex!("0122222222333333334444444455000001d50000000000001d60"), 0x029261), - (hex!("0122222222333333334444444455000001d60000000000001d70"), 0x029301), - (hex!("0122222222333333334444444455000001d60000000000004b50"), 0x0293a1), - (hex!("0122222222333333334444444455000001d60000000000007430"), 0x029441), - (hex!("0122222222333333334444444455000001d70000000000001d80"), 0x0294e1), - (hex!("0122222222333333334444444455000001d70000000000006920"), 0x029581), - (hex!("0122222222333333334444444455000001d80000000000001d90"), 0x029621), - (hex!("0122222222333333334444444455000001d80000000000005b30"), 0x0296c1), - (hex!("0122222222333333334444444455000001d90000000000001da0"), 0x029761), - (hex!("0122222222333333334444444455000001da0000000000001db0"), 0x029801), - (hex!("0122222222333333334444444455000001da0000000000004af0"), 0x0298a1), - (hex!("0122222222333333334444444455000001da0000000000007240"), 0x029941), - (hex!("0122222222333333334444444455000001da0000000000007470"), 0x0299e1), - (hex!("0122222222333333334444444455000001db0000000000001dc0"), 0x029a81), - (hex!("0122222222333333334444444455000001db00000000000045d0"), 0x029b21), - (hex!("0122222222333333334444444455000001dc0000000000001dd0"), 0x029bc1), - (hex!("0122222222333333334444444455000001dd0000000000001de0"), 0x029c61), - (hex!("0122222222333333334444444455000001dd0000000000004bb0"), 0x029d01), - (hex!("0122222222333333334444444455000001dd0000000000004cd0"), 0x029da1), - (hex!("0122222222333333334444444455000001dd0000000000006100"), 0x029e41), - (hex!("0122222222333333334444444455000001dd0000000000007bb0"), 0x029ee1), - (hex!("0122222222333333334444444455000001de0000000000001df0"), 0x029f81), - (hex!("0122222222333333334444444455000001de0000000000004260"), 0x02a021), - (hex!("0122222222333333334444444455000001de0000000000006040"), 0x02a0c1), - (hex!("0122222222333333334444444455000001df0000000000001e00"), 0x02a161), - (hex!("0122222222333333334444444455000001df0000000000005fa0"), 0x02a201), - (hex!("0122222222333333334444444455000001df0000000000006a70"), 0x02a2a1), - (hex!("0122222222333333334444444455000001df0000000000006dc0"), 0x02a341), - (hex!("0122222222333333334444444455000001e00000000000001e10"), 0x02a3e1), - (hex!("0122222222333333334444444455000001e00000000000007010"), 0x02a481), - (hex!("0122222222333333334444444455000001e10000000000001e20"), 0x02a521), - (hex!("0122222222333333334444444455000001e10000000000005720"), 0x02a5c1), - (hex!("0122222222333333334444444455000001e10000000000006830"), 0x02a661), - (hex!("0122222222333333334444444455000001e20000000000001e30"), 0x02a701), - (hex!("0122222222333333334444444455000001e20000000000005100"), 0x02a7a1), - (hex!("0122222222333333334444444455000001e30000000000001e40"), 0x02a841), - (hex!("0122222222333333334444444455000001e40000000000001e50"), 0x02a8e1), - (hex!("0122222222333333334444444455000001e40000000000003f30"), 0x02a981), - (hex!("0122222222333333334444444455000001e40000000000005220"), 0x02aa21), - (hex!("0122222222333333334444444455000001e50000000000001e60"), 0x02aac1), - (hex!("0122222222333333334444444455000001e50000000000006f60"), 0x02ab61), - (hex!("0122222222333333334444444455000001e60000000000001e70"), 0x02ac01), - (hex!("0122222222333333334444444455000001e60000000000006c80"), 0x02aca1), - (hex!("0122222222333333334444444455000001e70000000000001e80"), 0x02ad41), - (hex!("0122222222333333334444444455000001e80000000000001e90"), 0x02ade1), - (hex!("0122222222333333334444444455000001e80000000000004e30"), 0x02ae81), - (hex!("0122222222333333334444444455000001e90000000000001ea0"), 0x02af21), - (hex!("0122222222333333334444444455000001e90000000000005470"), 0x02afc1), - (hex!("0122222222333333334444444455000001ea0000000000001eb0"), 0x02b061), - (hex!("0122222222333333334444444455000001ea0000000000007980"), 0x02b101), - (hex!("0122222222333333334444444455000001eb0000000000001ec0"), 0x02b1a1), - (hex!("0122222222333333334444444455000001eb0000000000004390"), 0x02b241), - (hex!("0122222222333333334444444455000001eb0000000000005970"), 0x02b2e1), - (hex!("0122222222333333334444444455000001ec0000000000001ed0"), 0x02b381), - (hex!("0122222222333333334444444455000001ec0000000000005d50"), 0x02b421), - (hex!("0122222222333333334444444455000001ec00000000000076e0"), 0x02b4c1), - (hex!("0122222222333333334444444455000001ed0000000000001ee0"), 0x02b561), - (hex!("0122222222333333334444444455000001ed0000000000006190"), 0x02b601), - (hex!("0122222222333333334444444455000001ee0000000000001ef0"), 0x02b6a1), - (hex!("0122222222333333334444444455000001ee0000000000004900"), 0x02b741), - (hex!("0122222222333333334444444455000001ef0000000000001f00"), 0x02b7e1), - (hex!("0122222222333333334444444455000001ef0000000000006c60"), 0x02b881), - (hex!("0122222222333333334444444455000001f00000000000001f10"), 0x02b921), - (hex!("0122222222333333334444444455000001f00000000000006950"), 0x02b9c1), - (hex!("0122222222333333334444444455000001f10000000000001f20"), 0x02ba61), - (hex!("0122222222333333334444444455000001f10000000000006400"), 0x02bb01), - (hex!("0122222222333333334444444455000001f20000000000001f30"), 0x02bba1), - (hex!("0122222222333333334444444455000001f20000000000006f00"), 0x02bc41), - (hex!("0122222222333333334444444455000001f20000000000007b10"), 0x02bce1), - (hex!("0122222222333333334444444455000001f30000000000001f40"), 0x02bd81), - (hex!("0122222222333333334444444455000001f40000000000001f50"), 0x02be21), - (hex!("0122222222333333334444444455000001f50000000000001f60"), 0x02bec1), - (hex!("0122222222333333334444444455000001f500000000000044f0"), 0x02bf61), - (hex!("0122222222333333334444444455000001f60000000000001f70"), 0x02c001), - (hex!("0122222222333333334444444455000001f70000000000001f80"), 0x02c0a1), - (hex!("0122222222333333334444444455000001f70000000000004ad0"), 0x02c141), - (hex!("0122222222333333334444444455000001f80000000000001f90"), 0x02c1e1), - (hex!("0122222222333333334444444455000001f90000000000001fa0"), 0x02c281), - (hex!("0122222222333333334444444455000001f90000000000003f60"), 0x02c321), - (hex!("0122222222333333334444444455000001f90000000000004a80"), 0x02c3c1), - (hex!("0122222222333333334444444455000001fa0000000000001fb0"), 0x02c461), - (hex!("0122222222333333334444444455000001fa0000000000006f90"), 0x02c501), - (hex!("0122222222333333334444444455000001fb0000000000001fc0"), 0x02c5a1), - (hex!("0122222222333333334444444455000001fc0000000000001fd0"), 0x02c641), - (hex!("0122222222333333334444444455000001fc0000000000004a90"), 0x02c6e1), - (hex!("0122222222333333334444444455000001fd0000000000001fe0"), 0x02c781), - (hex!("0122222222333333334444444455000001fd0000000000005f50"), 0x02c821), - (hex!("0122222222333333334444444455000001fe0000000000001ff0"), 0x02c8c1), - (hex!("0122222222333333334444444455000001ff0000000000002000"), 0x02c961), - (hex!("0122222222333333334444444455000002000000000000002010"), 0x02ca01), - (hex!("0122222222333333334444444455000002000000000000005f00"), 0x02caa1), - (hex!("0122222222333333334444444455000002000000000000006840"), 0x02cb41), - (hex!("0122222222333333334444444455000002010000000000002020"), 0x02cbe1), - (hex!("0122222222333333334444444455000002020000000000002030"), 0x02cc81), - (hex!("0122222222333333334444444455000002030000000000002040"), 0x02cd21), - (hex!("0122222222333333334444444455000002040000000000002050"), 0x02cdc1), - (hex!("01222222223333333344444444550000020400000000000051f0"), 0x02ce61), - (hex!("0122222222333333334444444455000002050000000000002060"), 0x02cf01), - (hex!("0122222222333333334444444455000002060000000000002070"), 0x02cfa1), - (hex!("0122222222333333334444444455000002060000000000005c80"), 0x02d041), - (hex!("01222222223333333344444444550000020600000000000061d0"), 0x02d0e1), - (hex!("01222222223333333344444444550000020600000000000078c0"), 0x02d181), - (hex!("0122222222333333334444444455000002070000000000002080"), 0x02d221), - (hex!("0122222222333333334444444455000002070000000000006ba0"), 0x02d2c1), - (hex!("0122222222333333334444444455000002080000000000002090"), 0x02d361), - (hex!("01222222223333333344444444550000020900000000000020a0"), 0x02d401), - (hex!("01222222223333333344444444550000020900000000000067a0"), 0x02d4a1), - (hex!("01222222223333333344444444550000020a00000000000020b0"), 0x02d541), - (hex!("01222222223333333344444444550000020a0000000000004950"), 0x02d5e1), - (hex!("01222222223333333344444444550000020a0000000000004de0"), 0x02d681), - (hex!("01222222223333333344444444550000020b00000000000020c0"), 0x02d721), - (hex!("01222222223333333344444444550000020b0000000000004b00"), 0x02d7c1), - (hex!("01222222223333333344444444550000020c00000000000020d0"), 0x02d861), - (hex!("01222222223333333344444444550000020d00000000000020e0"), 0x02d901), - (hex!("01222222223333333344444444550000020e00000000000020f0"), 0x02d9a1), - (hex!("01222222223333333344444444550000020f0000000000002100"), 0x02da41), - (hex!("0122222222333333334444444455000002100000000000002110"), 0x02dae1), - (hex!("0122222222333333334444444455000002110000000000002120"), 0x02db81), - (hex!("0122222222333333334444444455000002110000000000004490"), 0x02dc21), - (hex!("0122222222333333334444444455000002120000000000002130"), 0x02dcc1), - (hex!("0122222222333333334444444455000002130000000000002140"), 0x02dd61), - (hex!("01222222223333333344444444550000021300000000000046d0"), 0x02de01), - (hex!("01222222223333333344444444550000021300000000000046e0"), 0x02dea1), - (hex!("0122222222333333334444444455000002130000000000004b70"), 0x02df41), - (hex!("0122222222333333334444444455000002140000000000002150"), 0x02dfe1), - (hex!("0122222222333333334444444455000002140000000000006c50"), 0x02e081), - (hex!("0122222222333333334444444455000002150000000000002160"), 0x02e121), - (hex!("01222222223333333344444444550000021500000000000043c0"), 0x02e1c1), - (hex!("0122222222333333334444444455000002160000000000002170"), 0x02e261), - (hex!("01222222223333333344444444550000021600000000000055b0"), 0x02e301), - (hex!("0122222222333333334444444455000002160000000000006150"), 0x02e3a1), - (hex!("0122222222333333334444444455000002170000000000002180"), 0x02e441), - (hex!("01222222223333333344444444550000021700000000000053b0"), 0x02e4e1), - (hex!("0122222222333333334444444455000002170000000000007460"), 0x02e581), - (hex!("0122222222333333334444444455000002180000000000002190"), 0x02e621), - (hex!("01222222223333333344444444550000021900000000000021a0"), 0x02e6c1), - (hex!("01222222223333333344444444550000021a00000000000021b0"), 0x02e761), - (hex!("01222222223333333344444444550000021a0000000000007650"), 0x02e801), - (hex!("01222222223333333344444444550000021b00000000000021c0"), 0x02e8a1), - (hex!("01222222223333333344444444550000021b0000000000004b20"), 0x02e941), - (hex!("01222222223333333344444444550000021c00000000000021d0"), 0x02e9e1), - (hex!("01222222223333333344444444550000021c0000000000007610"), 0x02ea81), - (hex!("01222222223333333344444444550000021d00000000000021e0"), 0x02eb21), - (hex!("01222222223333333344444444550000021d0000000000005f40"), 0x02ebc1), - (hex!("01222222223333333344444444550000021e00000000000021f0"), 0x02ec61), - (hex!("01222222223333333344444444550000021e0000000000005a50"), 0x02ed01), - (hex!("01222222223333333344444444550000021e0000000000005ff0"), 0x02eda1), - (hex!("01222222223333333344444444550000021f0000000000002200"), 0x02ee41), - (hex!("01222222223333333344444444550000021f00000000000043a0"), 0x02eee1), - (hex!("01222222223333333344444444550000021f0000000000004cb0"), 0x02ef81), - (hex!("01222222223333333344444444550000021f0000000000004e00"), 0x02f021), - (hex!("0122222222333333334444444455000002200000000000002210"), 0x02f0c1), - (hex!("0122222222333333334444444455000002210000000000002220"), 0x02f161), - (hex!("0122222222333333334444444455000002210000000000006290"), 0x02f201), - (hex!("0122222222333333334444444455000002210000000000007230"), 0x02f2a1), - (hex!("0122222222333333334444444455000002220000000000002230"), 0x02f341), - (hex!("0122222222333333334444444455000002220000000000006ea0"), 0x02f3e1), - (hex!("0122222222333333334444444455000002230000000000002240"), 0x02f481), - (hex!("0122222222333333334444444455000002230000000000004710"), 0x02f521), - (hex!("0122222222333333334444444455000002240000000000002250"), 0x02f5c1), - (hex!("0122222222333333334444444455000002250000000000002260"), 0x02f661), - (hex!("0122222222333333334444444455000002260000000000002270"), 0x02f701), - (hex!("0122222222333333334444444455000002260000000000005b40"), 0x02f7a1), - (hex!("0122222222333333334444444455000002260000000000006300"), 0x02f841), - (hex!("0122222222333333334444444455000002270000000000002280"), 0x02f8e1), - (hex!("0122222222333333334444444455000002270000000000005b80"), 0x02f981), - (hex!("0122222222333333334444444455000002280000000000002290"), 0x02fa21), - (hex!("0122222222333333334444444455000002280000000000003ed0"), 0x02fac1), - (hex!("0122222222333333334444444455000002280000000000004550"), 0x02fb61), - (hex!("01222222223333333344444444550000022800000000000077d0"), 0x02fc01), - (hex!("01222222223333333344444444550000022900000000000022a0"), 0x02fca1), - (hex!("0122222222333333334444444455000002290000000000006480"), 0x02fd41), - (hex!("01222222223333333344444444550000022a00000000000022b0"), 0x02fde1), - (hex!("01222222223333333344444444550000022a0000000000005450"), 0x02fe81), - (hex!("01222222223333333344444444550000022b00000000000022c0"), 0x02ff21), - (hex!("01222222223333333344444444550000022b0000000000006dd0"), 0x02ffc1), - (hex!("01222222223333333344444444550000022c00000000000022d0"), 0x030061), - (hex!("01222222223333333344444444550000022c0000000000006890"), 0x030101), - (hex!("01222222223333333344444444550000022d00000000000022e0"), 0x0301a1), - (hex!("01222222223333333344444444550000022e00000000000022f0"), 0x030241), - (hex!("01222222223333333344444444550000022e0000000000004f20"), 0x0302e1), - (hex!("01222222223333333344444444550000022f0000000000002300"), 0x030381), - (hex!("01222222223333333344444444550000022f0000000000005260"), 0x030421), - (hex!("01222222223333333344444444550000022f00000000000053f0"), 0x0304c1), - (hex!("0122222222333333334444444455000002300000000000002310"), 0x030561), - (hex!("01222222223333333344444444550000023000000000000050e0"), 0x030601), - (hex!("0122222222333333334444444455000002310000000000002320"), 0x0306a1), - (hex!("0122222222333333334444444455000002310000000000007800"), 0x030741), - (hex!("0122222222333333334444444455000002320000000000002330"), 0x0307e1), - (hex!("0122222222333333334444444455000002330000000000002340"), 0x030881), - (hex!("0122222222333333334444444455000002330000000000004d70"), 0x030921), - (hex!("0122222222333333334444444455000002330000000000005cf0"), 0x0309c1), - (hex!("0122222222333333334444444455000002340000000000002350"), 0x030a61), - (hex!("0122222222333333334444444455000002350000000000002360"), 0x030b01), - (hex!("0122222222333333334444444455000002350000000000006970"), 0x030ba1), - (hex!("0122222222333333334444444455000002360000000000002370"), 0x030c41), - (hex!("0122222222333333334444444455000002360000000000005270"), 0x030ce1), - (hex!("0122222222333333334444444455000002370000000000002380"), 0x030d81), - (hex!("0122222222333333334444444455000002370000000000005d70"), 0x030e21), - (hex!("0122222222333333334444444455000002380000000000002390"), 0x030ec1), - (hex!("01222222223333333344444444550000023800000000000069a0"), 0x030f61), - (hex!("01222222223333333344444444550000023900000000000023a0"), 0x031001), - (hex!("01222222223333333344444444550000023900000000000052e0"), 0x0310a1), - (hex!("0122222222333333334444444455000002390000000000005a10"), 0x031141), - (hex!("0122222222333333334444444455000002390000000000007440"), 0x0311e1), - (hex!("01222222223333333344444444550000023a00000000000023b0"), 0x031281), - (hex!("01222222223333333344444444550000023a0000000000003f00"), 0x031321), - (hex!("01222222223333333344444444550000023a0000000000004430"), 0x0313c1), - (hex!("01222222223333333344444444550000023a0000000000007070"), 0x031461), - (hex!("01222222223333333344444444550000023a00000000000074a0"), 0x031501), - (hex!("01222222223333333344444444550000023b00000000000023c0"), 0x0315a1), - (hex!("01222222223333333344444444550000023b0000000000004730"), 0x031641), - (hex!("01222222223333333344444444550000023b00000000000068b0"), 0x0316e1), - (hex!("01222222223333333344444444550000023c00000000000023d0"), 0x031781), - (hex!("01222222223333333344444444550000023c0000000000004680"), 0x031821), - (hex!("01222222223333333344444444550000023d00000000000023e0"), 0x0318c1), - (hex!("01222222223333333344444444550000023d00000000000059a0"), 0x031961), - (hex!("01222222223333333344444444550000023e00000000000023f0"), 0x031a01), - (hex!("01222222223333333344444444550000023f0000000000002400"), 0x031aa1), - (hex!("0122222222333333334444444455000002400000000000002410"), 0x031b41), - (hex!("0122222222333333334444444455000002400000000000004920"), 0x031be1), - (hex!("01222222223333333344444444550000024000000000000066e0"), 0x031c81), - (hex!("01222222223333333344444444550000024000000000000076f0"), 0x031d21), - (hex!("01222222223333333344444444550000024000000000000078e0"), 0x031dc1), - (hex!("0122222222333333334444444455000002410000000000002420"), 0x031e61), - (hex!("0122222222333333334444444455000002420000000000002430"), 0x031f01), - (hex!("0122222222333333334444444455000002420000000000006590"), 0x031fa1), - (hex!("0122222222333333334444444455000002430000000000002440"), 0x032041), - (hex!("0122222222333333334444444455000002430000000000004d00"), 0x0320e1), - (hex!("0122222222333333334444444455000002440000000000002450"), 0x032181), - (hex!("0122222222333333334444444455000002440000000000005f80"), 0x032221), - (hex!("0122222222333333334444444455000002450000000000002460"), 0x0322c1), - (hex!("0122222222333333334444444455000002450000000000004940"), 0x032361), - (hex!("0122222222333333334444444455000002460000000000002470"), 0x032401), - (hex!("0122222222333333334444444455000002470000000000002480"), 0x0324a1), - (hex!("0122222222333333334444444455000002470000000000004dd0"), 0x032541), - (hex!("0122222222333333334444444455000002470000000000005930"), 0x0325e1), - (hex!("01222222223333333344444444550000024700000000000061b0"), 0x032681), - (hex!("0122222222333333334444444455000002470000000000007740"), 0x032721), - (hex!("0122222222333333334444444455000002480000000000002490"), 0x0327c1), - (hex!("0122222222333333334444444455000002480000000000004890"), 0x032861), - (hex!("01222222223333333344444444550000024900000000000024a0"), 0x032901), - (hex!("01222222223333333344444444550000024a00000000000024b0"), 0x0329a1), - (hex!("01222222223333333344444444550000024b00000000000024c0"), 0x032a41), - (hex!("01222222223333333344444444550000024c00000000000024d0"), 0x032ae1), - (hex!("01222222223333333344444444550000024d00000000000024e0"), 0x032b81), - (hex!("01222222223333333344444444550000024d0000000000004070"), 0x032c21), - (hex!("01222222223333333344444444550000024e00000000000024f0"), 0x032cc1), - (hex!("01222222223333333344444444550000024e00000000000066a0"), 0x032d61), - (hex!("01222222223333333344444444550000024e0000000000006ab0"), 0x032e01), - (hex!("01222222223333333344444444550000024f0000000000002500"), 0x032ea1), - (hex!("0122222222333333334444444455000002500000000000002510"), 0x032f41), - (hex!("0122222222333333334444444455000002510000000000002520"), 0x032fe1), - (hex!("0122222222333333334444444455000002510000000000007320"), 0x033081), - (hex!("0122222222333333334444444455000002520000000000002530"), 0x033121), - (hex!("0122222222333333334444444455000002520000000000006410"), 0x0331c1), - (hex!("0122222222333333334444444455000002530000000000002540"), 0x033261), - (hex!("0122222222333333334444444455000002530000000000005110"), 0x033301), - (hex!("0122222222333333334444444455000002540000000000002550"), 0x0333a1), - (hex!("01222222223333333344444444550000025400000000000040c0"), 0x033441), - (hex!("0122222222333333334444444455000002540000000000006a40"), 0x0334e1), - (hex!("0122222222333333334444444455000002550000000000002560"), 0x033581), - (hex!("0122222222333333334444444455000002550000000000005190"), 0x033621), - (hex!("0122222222333333334444444455000002560000000000002570"), 0x0336c1), - (hex!("01222222223333333344444444550000025600000000000061f0"), 0x033761), - (hex!("0122222222333333334444444455000002570000000000002580"), 0x033801), - (hex!("0122222222333333334444444455000002580000000000002590"), 0x0338a1), - (hex!("01222222223333333344444444550000025800000000000043d0"), 0x033941), - (hex!("01222222223333333344444444550000025900000000000025a0"), 0x0339e1), - (hex!("0122222222333333334444444455000002590000000000006bb0"), 0x033a81), - (hex!("01222222223333333344444444550000025a00000000000025b0"), 0x033b21), - (hex!("01222222223333333344444444550000025a0000000000005fb0"), 0x033bc1), - (hex!("01222222223333333344444444550000025a00000000000064c0"), 0x033c61), - (hex!("01222222223333333344444444550000025b00000000000025c0"), 0x033d01), - (hex!("01222222223333333344444444550000025b0000000000005c10"), 0x033da1), - (hex!("01222222223333333344444444550000025c00000000000025d0"), 0x033e41), - (hex!("01222222223333333344444444550000025c0000000000007d00"), 0x033ee1), - (hex!("01222222223333333344444444550000025d00000000000025e0"), 0x033f81), - (hex!("01222222223333333344444444550000025e00000000000025f0"), 0x034021), - (hex!("01222222223333333344444444550000025e00000000000045e0"), 0x0340c1), - (hex!("01222222223333333344444444550000025e0000000000006ee0"), 0x034161), - (hex!("01222222223333333344444444550000025f0000000000002600"), 0x034201), - (hex!("01222222223333333344444444550000025f00000000000050b0"), 0x0342a1), - (hex!("01222222223333333344444444550000025f0000000000007690"), 0x034341), - (hex!("0122222222333333334444444455000002600000000000002610"), 0x0343e1), - (hex!("0122222222333333334444444455000002600000000000007b60"), 0x034481), - (hex!("0122222222333333334444444455000002610000000000002620"), 0x034521), - (hex!("0122222222333333334444444455000002620000000000002630"), 0x0345c1), - (hex!("0122222222333333334444444455000002630000000000002640"), 0x034661), - (hex!("0122222222333333334444444455000002640000000000002650"), 0x034701), - (hex!("0122222222333333334444444455000002650000000000002660"), 0x0347a1), - (hex!("0122222222333333334444444455000002650000000000006180"), 0x034841), - (hex!("0122222222333333334444444455000002660000000000002670"), 0x0348e1), - (hex!("0122222222333333334444444455000002660000000000005430"), 0x034981), - (hex!("0122222222333333334444444455000002660000000000007a60"), 0x034a21), - (hex!("0122222222333333334444444455000002670000000000002680"), 0x034ac1), - (hex!("01222222223333333344444444550000026700000000000077f0"), 0x034b61), - (hex!("0122222222333333334444444455000002680000000000002690"), 0x034c01), - (hex!("01222222223333333344444444550000026900000000000026a0"), 0x034ca1), - (hex!("01222222223333333344444444550000026a00000000000026b0"), 0x034d41), - (hex!("01222222223333333344444444550000026a0000000000007530"), 0x034de1), - (hex!("01222222223333333344444444550000026b00000000000026c0"), 0x034e81), - (hex!("01222222223333333344444444550000026b00000000000058b0"), 0x034f21), - (hex!("01222222223333333344444444550000026b00000000000066b0"), 0x034fc1), - (hex!("01222222223333333344444444550000026b0000000000006b10"), 0x035061), - (hex!("01222222223333333344444444550000026c00000000000026d0"), 0x035101), - (hex!("01222222223333333344444444550000026d00000000000026e0"), 0x0351a1), - (hex!("01222222223333333344444444550000026d0000000000004210"), 0x035241), - (hex!("01222222223333333344444444550000026d0000000000005490"), 0x0352e1), - (hex!("01222222223333333344444444550000026d0000000000005e60"), 0x035381), - (hex!("01222222223333333344444444550000026d00000000000068e0"), 0x035421), - (hex!("01222222223333333344444444550000026d0000000000007020"), 0x0354c1), - (hex!("01222222223333333344444444550000026d0000000000007300"), 0x035561), - (hex!("01222222223333333344444444550000026e00000000000026f0"), 0x035601), - (hex!("01222222223333333344444444550000026f0000000000002700"), 0x0356a1), - (hex!("01222222223333333344444444550000026f0000000000004910"), 0x035741), - (hex!("0122222222333333334444444455000002700000000000002710"), 0x0357e1), - (hex!("0122222222333333334444444455000002710000000000002720"), 0x035881), - (hex!("01222222223333333344444444550000027100000000000050c0"), 0x035921), - (hex!("0122222222333333334444444455000002720000000000002730"), 0x0359c1), - (hex!("0122222222333333334444444455000002730000000000002740"), 0x035a61), - (hex!("0122222222333333334444444455000002740000000000002750"), 0x035b01), - (hex!("0122222222333333334444444455000002740000000000007490"), 0x035ba1), - (hex!("0122222222333333334444444455000002750000000000002760"), 0x035c41), - (hex!("0122222222333333334444444455000002760000000000002770"), 0x035ce1), - (hex!("0122222222333333334444444455000002760000000000004790"), 0x035d81), - (hex!("0122222222333333334444444455000002770000000000002780"), 0x035e21), - (hex!("01222222223333333344444444550000027700000000000050a0"), 0x035ec1), - (hex!("0122222222333333334444444455000002780000000000002790"), 0x035f61), - (hex!("0122222222333333334444444455000002780000000000004330"), 0x036001), - (hex!("0122222222333333334444444455000002780000000000006b00"), 0x0360a1), - (hex!("01222222223333333344444444550000027900000000000027a0"), 0x036141), - (hex!("01222222223333333344444444550000027a00000000000027b0"), 0x0361e1), - (hex!("01222222223333333344444444550000027b00000000000027c0"), 0x036281), - (hex!("01222222223333333344444444550000027b0000000000004930"), 0x036321), - (hex!("01222222223333333344444444550000027b0000000000006250"), 0x0363c1), - (hex!("01222222223333333344444444550000027c00000000000027d0"), 0x036461), - (hex!("01222222223333333344444444550000027d00000000000027e0"), 0x036501), - (hex!("01222222223333333344444444550000027d0000000000005ce0"), 0x0365a1), - (hex!("01222222223333333344444444550000027d0000000000005fe0"), 0x036641), - (hex!("01222222223333333344444444550000027e00000000000027f0"), 0x0366e1), - (hex!("01222222223333333344444444550000027f0000000000002800"), 0x036781), - (hex!("01222222223333333344444444550000027f0000000000003e90"), 0x036821), - (hex!("01222222223333333344444444550000027f0000000000007910"), 0x0368c1), - (hex!("0122222222333333334444444455000002800000000000002810"), 0x036961), - (hex!("0122222222333333334444444455000002800000000000004990"), 0x036a01), - (hex!("0122222222333333334444444455000002800000000000006160"), 0x036aa1), - (hex!("0122222222333333334444444455000002800000000000006740"), 0x036b41), - (hex!("0122222222333333334444444455000002810000000000002820"), 0x036be1), - (hex!("0122222222333333334444444455000002820000000000002830"), 0x036c81), - (hex!("0122222222333333334444444455000002820000000000005170"), 0x036d21), - (hex!("0122222222333333334444444455000002830000000000002840"), 0x036dc1), - (hex!("0122222222333333334444444455000002840000000000002850"), 0x036e61), - (hex!("0122222222333333334444444455000002840000000000004810"), 0x036f01), - (hex!("0122222222333333334444444455000002840000000000006aa0"), 0x036fa1), - (hex!("0122222222333333334444444455000002850000000000002860"), 0x037041), - (hex!("0122222222333333334444444455000002860000000000002870"), 0x0370e1), - (hex!("0122222222333333334444444455000002860000000000005080"), 0x037181), - (hex!("0122222222333333334444444455000002870000000000002880"), 0x037221), - (hex!("0122222222333333334444444455000002870000000000004e60"), 0x0372c1), - (hex!("0122222222333333334444444455000002880000000000002890"), 0x037361), - (hex!("0122222222333333334444444455000002880000000000005060"), 0x037401), - (hex!("0122222222333333334444444455000002880000000000006f20"), 0x0374a1), - (hex!("01222222223333333344444444550000028900000000000028a0"), 0x037541), - (hex!("01222222223333333344444444550000028900000000000047e0"), 0x0375e1), - (hex!("01222222223333333344444444550000028a00000000000028b0"), 0x037681), - (hex!("01222222223333333344444444550000028a0000000000005ab0"), 0x037721), - (hex!("01222222223333333344444444550000028a0000000000007130"), 0x0377c1), - (hex!("01222222223333333344444444550000028a0000000000007660"), 0x037861), - (hex!("01222222223333333344444444550000028b00000000000028c0"), 0x037901), - (hex!("01222222223333333344444444550000028b00000000000054e0"), 0x0379a1), - (hex!("01222222223333333344444444550000028c00000000000028d0"), 0x037a41), - (hex!("01222222223333333344444444550000028c00000000000046f0"), 0x037ae1), - (hex!("01222222223333333344444444550000028c00000000000061a0"), 0x037b81), - (hex!("01222222223333333344444444550000028d00000000000028e0"), 0x037c21), - (hex!("01222222223333333344444444550000028e00000000000028f0"), 0x037cc1), - (hex!("01222222223333333344444444550000028e0000000000004130"), 0x037d61), - (hex!("01222222223333333344444444550000028f0000000000002900"), 0x037e01), - (hex!("01222222223333333344444444550000028f0000000000007510"), 0x037ea1), - (hex!("0122222222333333334444444455000002900000000000002910"), 0x037f41), - (hex!("0122222222333333334444444455000002900000000000004a40"), 0x037fe1), - (hex!("0122222222333333334444444455000002910000000000002920"), 0x038081), - (hex!("0122222222333333334444444455000002920000000000002930"), 0x038121), - (hex!("0122222222333333334444444455000002920000000000004e90"), 0x0381c1), - (hex!("0122222222333333334444444455000002930000000000002940"), 0x038261), - (hex!("0122222222333333334444444455000002930000000000006880"), 0x038301), - (hex!("0122222222333333334444444455000002940000000000002950"), 0x0383a1), - (hex!("0122222222333333334444444455000002940000000000007bc0"), 0x038441), - (hex!("0122222222333333334444444455000002950000000000002960"), 0x0384e1), - (hex!("0122222222333333334444444455000002960000000000002970"), 0x038581), - (hex!("01222222223333333344444444550000029600000000000059d0"), 0x038621), - (hex!("0122222222333333334444444455000002970000000000002980"), 0x0386c1), - (hex!("0122222222333333334444444455000002970000000000004a50"), 0x038761), - (hex!("0122222222333333334444444455000002970000000000005f20"), 0x038801), - (hex!("01222222223333333344444444550000029700000000000068d0"), 0x0388a1), - (hex!("0122222222333333334444444455000002980000000000002990"), 0x038941), - (hex!("0122222222333333334444444455000002980000000000004370"), 0x0389e1), - (hex!("0122222222333333334444444455000002980000000000004420"), 0x038a81), - (hex!("01222222223333333344444444550000029900000000000029a0"), 0x038b21), - (hex!("01222222223333333344444444550000029a00000000000029b0"), 0x038bc1), - (hex!("01222222223333333344444444550000029a0000000000006010"), 0x038c61), - (hex!("01222222223333333344444444550000029a0000000000006980"), 0x038d01), - (hex!("01222222223333333344444444550000029b00000000000029c0"), 0x038da1), - (hex!("01222222223333333344444444550000029c00000000000029d0"), 0x038e41), - (hex!("01222222223333333344444444550000029c0000000000007480"), 0x038ee1), - (hex!("01222222223333333344444444550000029d00000000000029e0"), 0x038f81), - (hex!("01222222223333333344444444550000029d0000000000005030"), 0x039021), - (hex!("01222222223333333344444444550000029d0000000000007780"), 0x0390c1), - (hex!("01222222223333333344444444550000029d0000000000007a50"), 0x039161), - (hex!("01222222223333333344444444550000029e00000000000029f0"), 0x039201), - (hex!("01222222223333333344444444550000029e00000000000074b0"), 0x0392a1), - (hex!("01222222223333333344444444550000029f0000000000002a00"), 0x039341), - (hex!("0122222222333333334444444455000002a00000000000002a10"), 0x0393e1), - (hex!("0122222222333333334444444455000002a10000000000002a20"), 0x039481), - (hex!("0122222222333333334444444455000002a20000000000002a30"), 0x039521), - (hex!("0122222222333333334444444455000002a20000000000004c50"), 0x0395c1), - (hex!("0122222222333333334444444455000002a20000000000006f10"), 0x039661), - (hex!("0122222222333333334444444455000002a30000000000002a40"), 0x039701), - (hex!("0122222222333333334444444455000002a40000000000002a50"), 0x0397a1), - (hex!("0122222222333333334444444455000002a40000000000005d60"), 0x039841), - (hex!("0122222222333333334444444455000002a50000000000002a60"), 0x0398e1), - (hex!("0122222222333333334444444455000002a50000000000005440"), 0x039981), - (hex!("0122222222333333334444444455000002a50000000000005890"), 0x039a21), - (hex!("0122222222333333334444444455000002a60000000000002a70"), 0x039ac1), - (hex!("0122222222333333334444444455000002a70000000000002a80"), 0x039b61), - (hex!("0122222222333333334444444455000002a700000000000054a0"), 0x039c01), - (hex!("0122222222333333334444444455000002a70000000000007280"), 0x039ca1), - (hex!("0122222222333333334444444455000002a80000000000002a90"), 0x039d41), - (hex!("0122222222333333334444444455000002a90000000000002aa0"), 0x039de1), - (hex!("0122222222333333334444444455000002aa0000000000002ab0"), 0x039e81), - (hex!("0122222222333333334444444455000002ab0000000000002ac0"), 0x039f21), - (hex!("0122222222333333334444444455000002ab0000000000006c90"), 0x039fc1), - (hex!("0122222222333333334444444455000002ac0000000000002ad0"), 0x03a061), - (hex!("0122222222333333334444444455000002ac0000000000006db0"), 0x03a101), - (hex!("0122222222333333334444444455000002ad0000000000002ae0"), 0x03a1a1), - (hex!("0122222222333333334444444455000002ad00000000000065e0"), 0x03a241), - (hex!("0122222222333333334444444455000002ad0000000000007b40"), 0x03a2e1), - (hex!("0122222222333333334444444455000002ae0000000000002af0"), 0x03a381), - (hex!("0122222222333333334444444455000002ae0000000000004d20"), 0x03a421), - (hex!("0122222222333333334444444455000002ae0000000000006f30"), 0x03a4c1), - (hex!("0122222222333333334444444455000002af0000000000002b00"), 0x03a561), - (hex!("0122222222333333334444444455000002b00000000000002b10"), 0x03a601), - (hex!("0122222222333333334444444455000002b00000000000004560"), 0x03a6a1), - (hex!("0122222222333333334444444455000002b00000000000005800"), 0x03a741), - (hex!("0122222222333333334444444455000002b00000000000005a60"), 0x03a7e1), - (hex!("0122222222333333334444444455000002b10000000000002b20"), 0x03a881), - (hex!("0122222222333333334444444455000002b10000000000007b30"), 0x03a921), - (hex!("0122222222333333334444444455000002b20000000000002b30"), 0x03a9c1), - (hex!("0122222222333333334444444455000002b20000000000004440"), 0x03aa61), - (hex!("0122222222333333334444444455000002b20000000000004f80"), 0x03ab01), - (hex!("0122222222333333334444444455000002b20000000000005020"), 0x03aba1), - (hex!("0122222222333333334444444455000002b30000000000002b40"), 0x03ac41), - (hex!("0122222222333333334444444455000002b40000000000002b50"), 0x03ace1), - (hex!("0122222222333333334444444455000002b50000000000002b60"), 0x03ad81), - (hex!("0122222222333333334444444455000002b500000000000059e0"), 0x03ae21), - (hex!("0122222222333333334444444455000002b60000000000002b70"), 0x03aec1), - (hex!("0122222222333333334444444455000002b70000000000002b80"), 0x03af61), - (hex!("0122222222333333334444444455000002b80000000000002b90"), 0x03b001), - (hex!("0122222222333333334444444455000002b80000000000004590"), 0x03b0a1), - (hex!("0122222222333333334444444455000002b800000000000047d0"), 0x03b141), - (hex!("0122222222333333334444444455000002b80000000000006030"), 0x03b1e1), - (hex!("0122222222333333334444444455000002b80000000000006a20"), 0x03b281), - (hex!("0122222222333333334444444455000002b80000000000006a90"), 0x03b321), - (hex!("0122222222333333334444444455000002b90000000000002ba0"), 0x03b3c1), - (hex!("0122222222333333334444444455000002ba0000000000002bb0"), 0x03b461), - (hex!("0122222222333333334444444455000002ba0000000000006e80"), 0x03b501), - (hex!("0122222222333333334444444455000002bb0000000000002bc0"), 0x03b5a1), - (hex!("0122222222333333334444444455000002bc0000000000002bd0"), 0x03b641), - (hex!("0122222222333333334444444455000002bc0000000000004b30"), 0x03b6e1), - (hex!("0122222222333333334444444455000002bd0000000000002be0"), 0x03b781), - (hex!("0122222222333333334444444455000002bd0000000000005e10"), 0x03b821), - (hex!("0122222222333333334444444455000002be0000000000002bf0"), 0x03b8c1), - (hex!("0122222222333333334444444455000002bf0000000000002c00"), 0x03b961), - (hex!("0122222222333333334444444455000002c00000000000002c10"), 0x03ba01), - (hex!("0122222222333333334444444455000002c10000000000002c20"), 0x03baa1), - (hex!("0122222222333333334444444455000002c10000000000003ef0"), 0x03bb41), - (hex!("0122222222333333334444444455000002c20000000000002c30"), 0x03bbe1), - (hex!("0122222222333333334444444455000002c200000000000056e0"), 0x03bc81), - (hex!("0122222222333333334444444455000002c30000000000002c40"), 0x03bd21), - (hex!("0122222222333333334444444455000002c30000000000004b60"), 0x03bdc1), - (hex!("0122222222333333334444444455000002c40000000000002c50"), 0x03be61), - (hex!("0122222222333333334444444455000002c400000000000045f0"), 0x03bf01), - (hex!("0122222222333333334444444455000002c40000000000005290"), 0x03bfa1), - (hex!("0122222222333333334444444455000002c50000000000002c60"), 0x03c041), - (hex!("0122222222333333334444444455000002c60000000000002c70"), 0x03c0e1), - (hex!("0122222222333333334444444455000002c60000000000006ae0"), 0x03c181), - (hex!("0122222222333333334444444455000002c70000000000002c80"), 0x03c221), - (hex!("0122222222333333334444444455000002c70000000000005680"), 0x03c2c1), - (hex!("0122222222333333334444444455000002c70000000000006e10"), 0x03c361), - (hex!("0122222222333333334444444455000002c80000000000002c90"), 0x03c401), - (hex!("0122222222333333334444444455000002c90000000000002ca0"), 0x03c4a1), - (hex!("0122222222333333334444444455000002ca0000000000002cb0"), 0x03c541), - (hex!("0122222222333333334444444455000002cb0000000000002cc0"), 0x03c5e1), - (hex!("0122222222333333334444444455000002cc0000000000002cd0"), 0x03c681), - (hex!("0122222222333333334444444455000002cc0000000000005b50"), 0x03c721), - (hex!("0122222222333333334444444455000002cd0000000000002ce0"), 0x03c7c1), - (hex!("0122222222333333334444444455000002ce0000000000002cf0"), 0x03c861), - (hex!("0122222222333333334444444455000002ce00000000000043f0"), 0x03c901), - (hex!("0122222222333333334444444455000002ce0000000000006420"), 0x03c9a1), - (hex!("0122222222333333334444444455000002cf0000000000002d00"), 0x03ca41), - (hex!("0122222222333333334444444455000002d00000000000002d10"), 0x03cae1), - (hex!("0122222222333333334444444455000002d10000000000002d20"), 0x03cb81), - (hex!("0122222222333333334444444455000002d10000000000005370"), 0x03cc21), - (hex!("0122222222333333334444444455000002d20000000000002d30"), 0x03ccc1), - (hex!("0122222222333333334444444455000002d20000000000005ef0"), 0x03cd61), - (hex!("0122222222333333334444444455000002d20000000000006570"), 0x03ce01), - (hex!("0122222222333333334444444455000002d30000000000002d40"), 0x03cea1), - (hex!("0122222222333333334444444455000002d30000000000007360"), 0x03cf41), - (hex!("0122222222333333334444444455000002d40000000000002d50"), 0x03cfe1), - (hex!("0122222222333333334444444455000002d400000000000079a0"), 0x03d081), - (hex!("0122222222333333334444444455000002d50000000000002d60"), 0x03d121), - (hex!("0122222222333333334444444455000002d50000000000004250"), 0x03d1c1), - (hex!("0122222222333333334444444455000002d50000000000006050"), 0x03d261), - (hex!("0122222222333333334444444455000002d60000000000002d70"), 0x03d301), - (hex!("0122222222333333334444444455000002d60000000000007080"), 0x03d3a1), - (hex!("0122222222333333334444444455000002d70000000000002d80"), 0x03d441), - (hex!("0122222222333333334444444455000002d80000000000002d90"), 0x03d4e1), - (hex!("0122222222333333334444444455000002d80000000000007110"), 0x03d581), - (hex!("0122222222333333334444444455000002d800000000000073c0"), 0x03d621), - (hex!("0122222222333333334444444455000002d800000000000075a0"), 0x03d6c1), - (hex!("0122222222333333334444444455000002d90000000000002da0"), 0x03d761), - (hex!("0122222222333333334444444455000002d90000000000004860"), 0x03d801), - (hex!("0122222222333333334444444455000002d90000000000006b60"), 0x03d8a1), - (hex!("0122222222333333334444444455000002da0000000000002db0"), 0x03d941), - (hex!("0122222222333333334444444455000002da0000000000006630"), 0x03d9e1), - (hex!("0122222222333333334444444455000002db0000000000002dc0"), 0x03da81), - (hex!("0122222222333333334444444455000002dc0000000000002dd0"), 0x03db21), - (hex!("0122222222333333334444444455000002dc0000000000004830"), 0x03dbc1), - (hex!("0122222222333333334444444455000002dd0000000000002de0"), 0x03dc61), - (hex!("0122222222333333334444444455000002de0000000000002df0"), 0x03dd01), - (hex!("0122222222333333334444444455000002de0000000000004f00"), 0x03dda1), - (hex!("0122222222333333334444444455000002df0000000000002e00"), 0x03de41), - (hex!("0122222222333333334444444455000002e00000000000002e10"), 0x03dee1), - (hex!("0122222222333333334444444455000002e10000000000002e20"), 0x03df81), - (hex!("0122222222333333334444444455000002e10000000000006e90"), 0x03e021), - (hex!("0122222222333333334444444455000002e20000000000002e30"), 0x03e0c1), - (hex!("0122222222333333334444444455000002e200000000000053e0"), 0x03e161), - (hex!("0122222222333333334444444455000002e30000000000002e40"), 0x03e201), - (hex!("0122222222333333334444444455000002e30000000000006020"), 0x03e2a1), - (hex!("0122222222333333334444444455000002e30000000000006540"), 0x03e341), - (hex!("0122222222333333334444444455000002e40000000000002e50"), 0x03e3e1), - (hex!("0122222222333333334444444455000002e50000000000002e60"), 0x03e481), - (hex!("0122222222333333334444444455000002e50000000000005180"), 0x03e521), - (hex!("0122222222333333334444444455000002e50000000000007bf0"), 0x03e5c1), - (hex!("0122222222333333334444444455000002e60000000000002e70"), 0x03e661), - (hex!("0122222222333333334444444455000002e60000000000005350"), 0x03e701), - (hex!("0122222222333333334444444455000002e60000000000007960"), 0x03e7a1), - (hex!("0122222222333333334444444455000002e70000000000002e80"), 0x03e841), - (hex!("0122222222333333334444444455000002e80000000000002e90"), 0x03e8e1), - (hex!("0122222222333333334444444455000002e90000000000002ea0"), 0x03e981), - (hex!("0122222222333333334444444455000002ea0000000000002eb0"), 0x03ea21), - (hex!("0122222222333333334444444455000002eb0000000000002ec0"), 0x03eac1), - (hex!("0122222222333333334444444455000002ec0000000000002ed0"), 0x03eb61), - (hex!("0122222222333333334444444455000002ec0000000000006c10"), 0x03ec01), - (hex!("0122222222333333334444444455000002ed0000000000002ee0"), 0x03eca1), - (hex!("0122222222333333334444444455000002ed0000000000005590"), 0x03ed41), - (hex!("0122222222333333334444444455000002ed0000000000005cd0"), 0x03ede1), - (hex!("0122222222333333334444444455000002ed0000000000006910"), 0x03ee81), - (hex!("0122222222333333334444444455000002ee0000000000002ef0"), 0x03ef21), - (hex!("0122222222333333334444444455000002ef0000000000002f00"), 0x03efc1), - (hex!("0122222222333333334444444455000002ef0000000000004ed0"), 0x03f061), - (hex!("0122222222333333334444444455000002f00000000000002f10"), 0x03f101), - (hex!("0122222222333333334444444455000002f00000000000004cf0"), 0x03f1a1), - (hex!("0122222222333333334444444455000002f00000000000005d10"), 0x03f241), - (hex!("0122222222333333334444444455000002f00000000000006860"), 0x03f2e1), - (hex!("0122222222333333334444444455000002f00000000000006b50"), 0x03f381), - (hex!("0122222222333333334444444455000002f00000000000007100"), 0x03f421), - (hex!("0122222222333333334444444455000002f00000000000007aa0"), 0x03f4c1), - (hex!("0122222222333333334444444455000002f10000000000002f20"), 0x03f561), - (hex!("0122222222333333334444444455000002f20000000000002f30"), 0x03f601), - (hex!("0122222222333333334444444455000002f200000000000044b0"), 0x03f6a1), - (hex!("0122222222333333334444444455000002f30000000000002f40"), 0x03f741), - (hex!("0122222222333333334444444455000002f300000000000075b0"), 0x03f7e1), - (hex!("0122222222333333334444444455000002f40000000000002f50"), 0x03f881), - (hex!("0122222222333333334444444455000002f400000000000060f0"), 0x03f921), - (hex!("0122222222333333334444444455000002f50000000000002f60"), 0x03f9c1), - (hex!("0122222222333333334444444455000002f50000000000007210"), 0x03fa61), - (hex!("0122222222333333334444444455000002f60000000000002f70"), 0x03fb01), - (hex!("0122222222333333334444444455000002f60000000000006610"), 0x03fba1), - (hex!("0122222222333333334444444455000002f70000000000002f80"), 0x03fc41), - (hex!("0122222222333333334444444455000002f70000000000007560"), 0x03fce1), - (hex!("0122222222333333334444444455000002f80000000000002f90"), 0x03fd81), - (hex!("0122222222333333334444444455000002f80000000000006320"), 0x03fe21), - (hex!("0122222222333333334444444455000002f90000000000002fa0"), 0x03fec1), - (hex!("0122222222333333334444444455000002f90000000000006e50"), 0x03ff61), - (hex!("0122222222333333334444444455000002fa0000000000002fb0"), 0x040001), - (hex!("0122222222333333334444444455000002fb0000000000002fc0"), 0x0400a1), - (hex!("0122222222333333334444444455000002fb0000000000004780"), 0x040141), - (hex!("0122222222333333334444444455000002fc0000000000002fd0"), 0x0401e1), - (hex!("0122222222333333334444444455000002fd0000000000002fe0"), 0x040281), - (hex!("0122222222333333334444444455000002fd0000000000005600"), 0x040321), - (hex!("0122222222333333334444444455000002fd0000000000006c00"), 0x0403c1), - (hex!("0122222222333333334444444455000002fe0000000000002ff0"), 0x040461), - (hex!("0122222222333333334444444455000002ff0000000000003000"), 0x040501), - (hex!("0122222222333333334444444455000003000000000000003010"), 0x0405a1), - (hex!("0122222222333333334444444455000003000000000000004080"), 0x040641), - (hex!("0122222222333333334444444455000003010000000000003020"), 0x0406e1), - (hex!("0122222222333333334444444455000003010000000000006340"), 0x040781), - (hex!("0122222222333333334444444455000003020000000000003030"), 0x040821), - (hex!("0122222222333333334444444455000003020000000000005b00"), 0x0408c1), - (hex!("0122222222333333334444444455000003020000000000007b20"), 0x040961), - (hex!("0122222222333333334444444455000003030000000000003040"), 0x040a01), - (hex!("01222222223333333344444444550000030300000000000056b0"), 0x040aa1), - (hex!("0122222222333333334444444455000003030000000000006280"), 0x040b41), - (hex!("0122222222333333334444444455000003030000000000007ad0"), 0x040be1), - (hex!("0122222222333333334444444455000003040000000000003050"), 0x040c81), - (hex!("0122222222333333334444444455000003040000000000005c50"), 0x040d21), - (hex!("0122222222333333334444444455000003050000000000003060"), 0x040dc1), - (hex!("01222222223333333344444444550000030500000000000072e0"), 0x040e61), - (hex!("0122222222333333334444444455000003060000000000003070"), 0x040f01), - (hex!("0122222222333333334444444455000003060000000000004360"), 0x040fa1), - (hex!("0122222222333333334444444455000003060000000000004380"), 0x041041), - (hex!("0122222222333333334444444455000003060000000000004820"), 0x0410e1), - (hex!("0122222222333333334444444455000003060000000000006d10"), 0x041181), - (hex!("0122222222333333334444444455000003070000000000003080"), 0x041221), - (hex!("0122222222333333334444444455000003070000000000004450"), 0x0412c1), - (hex!("0122222222333333334444444455000003080000000000003090"), 0x041361), - (hex!("0122222222333333334444444455000003080000000000005ad0"), 0x041401), - (hex!("01222222223333333344444444550000030900000000000030a0"), 0x0414a1), - (hex!("01222222223333333344444444550000030a00000000000030b0"), 0x041541), - (hex!("01222222223333333344444444550000030a0000000000007760"), 0x0415e1), - (hex!("01222222223333333344444444550000030b00000000000030c0"), 0x041681), - (hex!("01222222223333333344444444550000030b0000000000007a80"), 0x041721), - (hex!("01222222223333333344444444550000030c00000000000030d0"), 0x0417c1), - (hex!("01222222223333333344444444550000030d00000000000030e0"), 0x041861), - (hex!("01222222223333333344444444550000030d0000000000003eb0"), 0x041901), - (hex!("01222222223333333344444444550000030e00000000000030f0"), 0x0419a1), - (hex!("01222222223333333344444444550000030f0000000000003100"), 0x041a41), - (hex!("01222222223333333344444444550000030f0000000000004690"), 0x041ae1), - (hex!("01222222223333333344444444550000030f0000000000006900"), 0x041b81), - (hex!("0122222222333333334444444455000003100000000000003110"), 0x041c21), - (hex!("01222222223333333344444444550000031000000000000058a0"), 0x041cc1), - (hex!("0122222222333333334444444455000003110000000000003120"), 0x041d61), - (hex!("0122222222333333334444444455000003110000000000004200"), 0x041e01), - (hex!("0122222222333333334444444455000003120000000000003130"), 0x041ea1), - (hex!("0122222222333333334444444455000003130000000000003140"), 0x041f41), - (hex!("0122222222333333334444444455000003130000000000004d50"), 0x041fe1), - (hex!("0122222222333333334444444455000003130000000000005400"), 0x042081), - (hex!("0122222222333333334444444455000003130000000000005520"), 0x042121), - (hex!("0122222222333333334444444455000003140000000000003150"), 0x0421c1), - (hex!("0122222222333333334444444455000003140000000000006450"), 0x042261), - (hex!("0122222222333333334444444455000003150000000000003160"), 0x042301), - (hex!("01222222223333333344444444550000031500000000000062d0"), 0x0423a1), - (hex!("0122222222333333334444444455000003160000000000003170"), 0x042441), - (hex!("0122222222333333334444444455000003160000000000004c40"), 0x0424e1), - (hex!("0122222222333333334444444455000003160000000000007c80"), 0x042581), - (hex!("0122222222333333334444444455000003170000000000003180"), 0x042621), - (hex!("0122222222333333334444444455000003170000000000004400"), 0x0426c1), - (hex!("0122222222333333334444444455000003170000000000005090"), 0x042761), - (hex!("0122222222333333334444444455000003170000000000006cb0"), 0x042801), - (hex!("0122222222333333334444444455000003180000000000003190"), 0x0428a1), - (hex!("0122222222333333334444444455000003180000000000006560"), 0x042941), - (hex!("01222222223333333344444444550000031900000000000031a0"), 0x0429e1), - (hex!("01222222223333333344444444550000031900000000000052d0"), 0x042a81), - (hex!("01222222223333333344444444550000031900000000000057e0"), 0x042b21), - (hex!("01222222223333333344444444550000031a00000000000031b0"), 0x042bc1), - (hex!("01222222223333333344444444550000031a00000000000071e0"), 0x042c61), - (hex!("01222222223333333344444444550000031b00000000000031c0"), 0x042d01), - (hex!("01222222223333333344444444550000031c00000000000031d0"), 0x042da1), - (hex!("01222222223333333344444444550000031c0000000000004480"), 0x042e41), - (hex!("01222222223333333344444444550000031c0000000000005790"), 0x042ee1), - (hex!("01222222223333333344444444550000031c0000000000007be0"), 0x042f81), - (hex!("01222222223333333344444444550000031d00000000000031e0"), 0x043021), - (hex!("01222222223333333344444444550000031d0000000000005560"), 0x0430c1), - (hex!("01222222223333333344444444550000031e00000000000031f0"), 0x043161), - (hex!("01222222223333333344444444550000031f0000000000003200"), 0x043201), - (hex!("01222222223333333344444444550000031f0000000000004190"), 0x0432a1), - (hex!("0122222222333333334444444455000003200000000000003210"), 0x043341), - (hex!("0122222222333333334444444455000003210000000000003220"), 0x0433e1), - (hex!("0122222222333333334444444455000003220000000000003230"), 0x043481), - (hex!("0122222222333333334444444455000003230000000000003240"), 0x043521), - (hex!("01222222223333333344444444550000032300000000000069d0"), 0x0435c1), - (hex!("0122222222333333334444444455000003240000000000003250"), 0x043661), - (hex!("0122222222333333334444444455000003250000000000003260"), 0x043701), - (hex!("01222222223333333344444444550000032500000000000042b0"), 0x0437a1), - (hex!("01222222223333333344444444550000032500000000000064e0"), 0x043841), - (hex!("0122222222333333334444444455000003260000000000003270"), 0x0438e1), - (hex!("0122222222333333334444444455000003270000000000003280"), 0x043981), - (hex!("0122222222333333334444444455000003270000000000005b20"), 0x043a21), - (hex!("0122222222333333334444444455000003270000000000006330"), 0x043ac1), - (hex!("0122222222333333334444444455000003270000000000006810"), 0x043b61), - (hex!("0122222222333333334444444455000003280000000000003290"), 0x043c01), - (hex!("01222222223333333344444444550000032900000000000032a0"), 0x043ca1), - (hex!("01222222223333333344444444550000032900000000000056f0"), 0x043d41), - (hex!("0122222222333333334444444455000003290000000000005e20"), 0x043de1), - (hex!("0122222222333333334444444455000003290000000000005e70"), 0x043e81), - (hex!("01222222223333333344444444550000032a00000000000032b0"), 0x043f21), - (hex!("01222222223333333344444444550000032b00000000000032c0"), 0x043fc1), - (hex!("01222222223333333344444444550000032b0000000000005500"), 0x044061), - (hex!("01222222223333333344444444550000032b0000000000005a20"), 0x044101), - (hex!("01222222223333333344444444550000032c00000000000032d0"), 0x0441a1), - (hex!("01222222223333333344444444550000032c0000000000004060"), 0x044241), - (hex!("01222222223333333344444444550000032c0000000000004760"), 0x0442e1), - (hex!("01222222223333333344444444550000032d00000000000032e0"), 0x044381), - (hex!("01222222223333333344444444550000032d00000000000068a0"), 0x044421), - (hex!("01222222223333333344444444550000032e00000000000032f0"), 0x0444c1), - (hex!("01222222223333333344444444550000032f0000000000003300"), 0x044561), - (hex!("0122222222333333334444444455000003300000000000003310"), 0x044601), - (hex!("0122222222333333334444444455000003300000000000006e40"), 0x0446a1), - (hex!("0122222222333333334444444455000003310000000000003320"), 0x044741), - (hex!("0122222222333333334444444455000003310000000000004620"), 0x0447e1), - (hex!("0122222222333333334444444455000003320000000000003330"), 0x044881), - (hex!("0122222222333333334444444455000003330000000000003340"), 0x044921), - (hex!("0122222222333333334444444455000003330000000000004b80"), 0x0449c1), - (hex!("0122222222333333334444444455000003340000000000003350"), 0x044a61), - (hex!("0122222222333333334444444455000003350000000000003360"), 0x044b01), - (hex!("0122222222333333334444444455000003360000000000003370"), 0x044ba1), - (hex!("0122222222333333334444444455000003370000000000003380"), 0x044c41), - (hex!("0122222222333333334444444455000003380000000000003390"), 0x044ce1), - (hex!("01222222223333333344444444550000033900000000000033a0"), 0x044d81), - (hex!("0122222222333333334444444455000003390000000000006b90"), 0x044e21), - (hex!("01222222223333333344444444550000033a00000000000033b0"), 0x044ec1), - (hex!("01222222223333333344444444550000033a0000000000007420"), 0x044f61), - (hex!("01222222223333333344444444550000033b00000000000033c0"), 0x045001), - (hex!("01222222223333333344444444550000033b0000000000007620"), 0x0450a1), - (hex!("01222222223333333344444444550000033c00000000000033d0"), 0x045141), - (hex!("01222222223333333344444444550000033c0000000000006b30"), 0x0451e1), - (hex!("01222222223333333344444444550000033d00000000000033e0"), 0x045281), - (hex!("01222222223333333344444444550000033e00000000000033f0"), 0x045321), - (hex!("01222222223333333344444444550000033e00000000000048b0"), 0x0453c1), - (hex!("01222222223333333344444444550000033e0000000000004e70"), 0x045461), - (hex!("01222222223333333344444444550000033f0000000000003400"), 0x045501), - (hex!("01222222223333333344444444550000033f0000000000006380"), 0x0455a1), - (hex!("0122222222333333334444444455000003400000000000003410"), 0x045641), - (hex!("0122222222333333334444444455000003410000000000003420"), 0x0456e1), - (hex!("0122222222333333334444444455000003410000000000006090"), 0x045781), - (hex!("0122222222333333334444444455000003420000000000003430"), 0x045821), - (hex!("01222222223333333344444444550000034200000000000073d0"), 0x0458c1), - (hex!("0122222222333333334444444455000003430000000000003440"), 0x045961), - (hex!("0122222222333333334444444455000003430000000000006370"), 0x045a01), - (hex!("01222222223333333344444444550000034300000000000075c0"), 0x045aa1), - (hex!("0122222222333333334444444455000003440000000000003450"), 0x045b41), - (hex!("0122222222333333334444444455000003450000000000003460"), 0x045be1), - (hex!("0122222222333333334444444455000003460000000000003470"), 0x045c81), - (hex!("01222222223333333344444444550000034600000000000055f0"), 0x045d21), - (hex!("0122222222333333334444444455000003470000000000003480"), 0x045dc1), - (hex!("0122222222333333334444444455000003470000000000003fe0"), 0x045e61), - (hex!("0122222222333333334444444455000003480000000000003490"), 0x045f01), - (hex!("0122222222333333334444444455000003480000000000007990"), 0x045fa1), - (hex!("01222222223333333344444444550000034900000000000034a0"), 0x046041), - (hex!("0122222222333333334444444455000003490000000000004410"), 0x0460e1), - (hex!("01222222223333333344444444550000034a00000000000034b0"), 0x046181), - (hex!("01222222223333333344444444550000034a00000000000062a0"), 0x046221), - (hex!("01222222223333333344444444550000034a0000000000007260"), 0x0462c1), - (hex!("01222222223333333344444444550000034b00000000000034c0"), 0x046361), - (hex!("01222222223333333344444444550000034b0000000000005760"), 0x046401), - (hex!("01222222223333333344444444550000034b0000000000006200"), 0x0464a1), - (hex!("01222222223333333344444444550000034c00000000000034d0"), 0x046541), - (hex!("01222222223333333344444444550000034d00000000000034e0"), 0x0465e1), - (hex!("01222222223333333344444444550000034e00000000000034f0"), 0x046681), - (hex!("01222222223333333344444444550000034e0000000000007790"), 0x046721), - (hex!("01222222223333333344444444550000034f0000000000003500"), 0x0467c1), - (hex!("0122222222333333334444444455000003500000000000003510"), 0x046861), - (hex!("0122222222333333334444444455000003510000000000003520"), 0x046901), - (hex!("0122222222333333334444444455000003520000000000003530"), 0x0469a1), - (hex!("01222222223333333344444444550000035200000000000056a0"), 0x046a41), - (hex!("0122222222333333334444444455000003530000000000003540"), 0x046ae1), - (hex!("0122222222333333334444444455000003540000000000003550"), 0x046b81), - (hex!("01222222223333333344444444550000035400000000000047b0"), 0x046c21), - (hex!("0122222222333333334444444455000003550000000000003560"), 0x046cc1), - (hex!("0122222222333333334444444455000003550000000000004500"), 0x046d61), - (hex!("0122222222333333334444444455000003560000000000003570"), 0x046e01), - (hex!("0122222222333333334444444455000003560000000000004fc0"), 0x046ea1), - (hex!("0122222222333333334444444455000003560000000000007160"), 0x046f41), - (hex!("0122222222333333334444444455000003560000000000007400"), 0x046fe1), - (hex!("0122222222333333334444444455000003570000000000003580"), 0x047081), - (hex!("0122222222333333334444444455000003580000000000003590"), 0x047121), - (hex!("0122222222333333334444444455000003580000000000005a80"), 0x0471c1), - (hex!("01222222223333333344444444550000035900000000000035a0"), 0x047261), - (hex!("01222222223333333344444444550000035900000000000073b0"), 0x047301), - (hex!("01222222223333333344444444550000035a00000000000035b0"), 0x0473a1), - (hex!("01222222223333333344444444550000035a0000000000004c20"), 0x047441), - (hex!("01222222223333333344444444550000035b00000000000035c0"), 0x0474e1), - (hex!("01222222223333333344444444550000035b0000000000005120"), 0x047581), - (hex!("01222222223333333344444444550000035c00000000000035d0"), 0x047621), - (hex!("01222222223333333344444444550000035c0000000000004300"), 0x0476c1), - (hex!("01222222223333333344444444550000035c0000000000005a40"), 0x047761), - (hex!("01222222223333333344444444550000035c0000000000006620"), 0x047801), - (hex!("01222222223333333344444444550000035c0000000000006ed0"), 0x0478a1), - (hex!("01222222223333333344444444550000035d00000000000035e0"), 0x047941), - (hex!("01222222223333333344444444550000035d0000000000005df0"), 0x0479e1), - (hex!("01222222223333333344444444550000035e00000000000035f0"), 0x047a81), - (hex!("01222222223333333344444444550000035f0000000000003600"), 0x047b21), - (hex!("01222222223333333344444444550000035f00000000000058d0"), 0x047bc1), - (hex!("0122222222333333334444444455000003600000000000003610"), 0x047c61), - (hex!("0122222222333333334444444455000003600000000000007b90"), 0x047d01), - (hex!("0122222222333333334444444455000003610000000000003620"), 0x047da1), - (hex!("0122222222333333334444444455000003610000000000006ad0"), 0x047e41), - (hex!("0122222222333333334444444455000003620000000000003630"), 0x047ee1), - (hex!("01222222223333333344444444550000036200000000000063a0"), 0x047f81), - (hex!("0122222222333333334444444455000003630000000000003640"), 0x048021), - (hex!("0122222222333333334444444455000003630000000000007250"), 0x0480c1), - (hex!("0122222222333333334444444455000003640000000000003650"), 0x048161), - (hex!("0122222222333333334444444455000003640000000000005510"), 0x048201), - (hex!("0122222222333333334444444455000003640000000000007850"), 0x0482a1), - (hex!("0122222222333333334444444455000003650000000000003660"), 0x048341), - (hex!("0122222222333333334444444455000003660000000000003670"), 0x0483e1), - (hex!("0122222222333333334444444455000003660000000000004650"), 0x048481), - (hex!("01222222223333333344444444550000036600000000000050d0"), 0x048521), - (hex!("0122222222333333334444444455000003660000000000006eb0"), 0x0485c1), - (hex!("0122222222333333334444444455000003670000000000003680"), 0x048661), - (hex!("01222222223333333344444444550000036700000000000071f0"), 0x048701), - (hex!("0122222222333333334444444455000003680000000000003690"), 0x0487a1), - (hex!("01222222223333333344444444550000036900000000000036a0"), 0x048841), - (hex!("0122222222333333334444444455000003690000000000005c70"), 0x0488e1), - (hex!("01222222223333333344444444550000036a00000000000036b0"), 0x048981), - (hex!("01222222223333333344444444550000036a00000000000071b0"), 0x048a21), - (hex!("01222222223333333344444444550000036b00000000000036c0"), 0x048ac1), - (hex!("01222222223333333344444444550000036b0000000000004670"), 0x048b61), - (hex!("01222222223333333344444444550000036c00000000000036d0"), 0x048c01), - (hex!("01222222223333333344444444550000036c0000000000004750"), 0x048ca1), - (hex!("01222222223333333344444444550000036c0000000000006fa0"), 0x048d41), - (hex!("01222222223333333344444444550000036d00000000000036e0"), 0x048de1), - (hex!("01222222223333333344444444550000036d0000000000003f70"), 0x048e81), - (hex!("01222222223333333344444444550000036d0000000000004b90"), 0x048f21), - (hex!("01222222223333333344444444550000036d00000000000057a0"), 0x048fc1), - (hex!("01222222223333333344444444550000036e00000000000036f0"), 0x049061), - (hex!("01222222223333333344444444550000036e00000000000075d0"), 0x049101), - (hex!("01222222223333333344444444550000036f0000000000003700"), 0x0491a1), - (hex!("0122222222333333334444444455000003700000000000003710"), 0x049241), - (hex!("0122222222333333334444444455000003700000000000005aa0"), 0x0492e1), - (hex!("0122222222333333334444444455000003710000000000003720"), 0x049381), - (hex!("0122222222333333334444444455000003710000000000005130"), 0x049421), - (hex!("0122222222333333334444444455000003710000000000006fc0"), 0x0494c1), - (hex!("0122222222333333334444444455000003710000000000007b00"), 0x049561), - (hex!("0122222222333333334444444455000003720000000000003730"), 0x049601), - (hex!("01222222223333333344444444550000037200000000000054d0"), 0x0496a1), - (hex!("0122222222333333334444444455000003730000000000003740"), 0x049741), - (hex!("0122222222333333334444444455000003730000000000004220"), 0x0497e1), - (hex!("0122222222333333334444444455000003740000000000003750"), 0x049881), - (hex!("0122222222333333334444444455000003740000000000004720"), 0x049921), - (hex!("0122222222333333334444444455000003750000000000003760"), 0x0499c1), - (hex!("0122222222333333334444444455000003750000000000004110"), 0x049a61), - (hex!("0122222222333333334444444455000003760000000000003770"), 0x049b01), - (hex!("0122222222333333334444444455000003770000000000003780"), 0x049ba1), - (hex!("0122222222333333334444444455000003780000000000003790"), 0x049c41), - (hex!("0122222222333333334444444455000003780000000000004b40"), 0x049ce1), - (hex!("0122222222333333334444444455000003780000000000005660"), 0x049d81), - (hex!("0122222222333333334444444455000003780000000000005ea0"), 0x049e21), - (hex!("01222222223333333344444444550000037900000000000037a0"), 0x049ec1), - (hex!("01222222223333333344444444550000037a00000000000037b0"), 0x049f61), - (hex!("01222222223333333344444444550000037b00000000000037c0"), 0x04a001), - (hex!("01222222223333333344444444550000037c00000000000037d0"), 0x04a0a1), - (hex!("01222222223333333344444444550000037c0000000000004340"), 0x04a141), - (hex!("01222222223333333344444444550000037c0000000000005230"), 0x04a1e1), - (hex!("01222222223333333344444444550000037d00000000000037e0"), 0x04a281), - (hex!("01222222223333333344444444550000037d00000000000051e0"), 0x04a321), - (hex!("01222222223333333344444444550000037e00000000000037f0"), 0x04a3c1), - (hex!("01222222223333333344444444550000037e0000000000004090"), 0x04a461), - (hex!("01222222223333333344444444550000037e0000000000005c20"), 0x04a501), - (hex!("01222222223333333344444444550000037f0000000000003800"), 0x04a5a1), - (hex!("0122222222333333334444444455000003800000000000003810"), 0x04a641), - (hex!("0122222222333333334444444455000003800000000000007630"), 0x04a6e1), - (hex!("0122222222333333334444444455000003810000000000003820"), 0x04a781), - (hex!("0122222222333333334444444455000003820000000000003830"), 0x04a821), - (hex!("0122222222333333334444444455000003820000000000004170"), 0x04a8c1), - (hex!("0122222222333333334444444455000003830000000000003840"), 0x04a961), - (hex!("0122222222333333334444444455000003840000000000003850"), 0x04aa01), - (hex!("0122222222333333334444444455000003850000000000003860"), 0x04aaa1), - (hex!("0122222222333333334444444455000003850000000000004180"), 0x04ab41), - (hex!("0122222222333333334444444455000003850000000000005c90"), 0x04abe1), - (hex!("0122222222333333334444444455000003850000000000005da0"), 0x04ac81), - (hex!("0122222222333333334444444455000003850000000000006ff0"), 0x04ad21), - (hex!("0122222222333333334444444455000003860000000000003870"), 0x04adc1), - (hex!("01222222223333333344444444550000038600000000000065c0"), 0x04ae61), - (hex!("0122222222333333334444444455000003870000000000003880"), 0x04af01), - (hex!("0122222222333333334444444455000003870000000000007cc0"), 0x04afa1), - (hex!("0122222222333333334444444455000003880000000000003890"), 0x04b041), - (hex!("01222222223333333344444444550000038900000000000038a0"), 0x04b0e1), - (hex!("01222222223333333344444444550000038a00000000000038b0"), 0x04b181), - (hex!("01222222223333333344444444550000038a00000000000073e0"), 0x04b221), - (hex!("01222222223333333344444444550000038b00000000000038c0"), 0x04b2c1), - (hex!("01222222223333333344444444550000038c00000000000038d0"), 0x04b361), - (hex!("01222222223333333344444444550000038d00000000000038e0"), 0x04b401), - (hex!("01222222223333333344444444550000038d00000000000069f0"), 0x04b4a1), - (hex!("01222222223333333344444444550000038d0000000000007680"), 0x04b541), - (hex!("01222222223333333344444444550000038e00000000000038f0"), 0x04b5e1), - (hex!("01222222223333333344444444550000038f0000000000003900"), 0x04b681), - (hex!("01222222223333333344444444550000038f00000000000045b0"), 0x04b721), - (hex!("01222222223333333344444444550000038f0000000000007180"), 0x04b7c1), - (hex!("0122222222333333334444444455000003900000000000003910"), 0x04b861), - (hex!("0122222222333333334444444455000003910000000000003920"), 0x04b901), - (hex!("0122222222333333334444444455000003910000000000004a20"), 0x04b9a1), - (hex!("0122222222333333334444444455000003920000000000003930"), 0x04ba41), - (hex!("01222222223333333344444444550000039200000000000059b0"), 0x04bae1), - (hex!("0122222222333333334444444455000003930000000000003940"), 0x04bb81), - (hex!("0122222222333333334444444455000003930000000000006cc0"), 0x04bc21), - (hex!("0122222222333333334444444455000003940000000000003950"), 0x04bcc1), - (hex!("01222222223333333344444444550000039400000000000056c0"), 0x04bd61), - (hex!("0122222222333333334444444455000003950000000000003960"), 0x04be01), - (hex!("0122222222333333334444444455000003950000000000004cc0"), 0x04bea1), - (hex!("0122222222333333334444444455000003950000000000007720"), 0x04bf41), - (hex!("0122222222333333334444444455000003960000000000003970"), 0x04bfe1), - (hex!("0122222222333333334444444455000003960000000000004da0"), 0x04c081), - (hex!("0122222222333333334444444455000003960000000000004df0"), 0x04c121), - (hex!("0122222222333333334444444455000003960000000000004f30"), 0x04c1c1), - (hex!("01222222223333333344444444550000039600000000000050f0"), 0x04c261), - (hex!("0122222222333333334444444455000003960000000000007940"), 0x04c301), - (hex!("0122222222333333334444444455000003970000000000003980"), 0x04c3a1), - (hex!("0122222222333333334444444455000003970000000000005850"), 0x04c441), - (hex!("0122222222333333334444444455000003970000000000007bd0"), 0x04c4e1), - (hex!("0122222222333333334444444455000003980000000000003990"), 0x04c581), - (hex!("0122222222333333334444444455000003980000000000004c00"), 0x04c621), - (hex!("0122222222333333334444444455000003980000000000005580"), 0x04c6c1), - (hex!("01222222223333333344444444550000039900000000000039a0"), 0x04c761), - (hex!("0122222222333333334444444455000003990000000000005820"), 0x04c801), - (hex!("01222222223333333344444444550000039a00000000000039b0"), 0x04c8a1), - (hex!("01222222223333333344444444550000039b00000000000039c0"), 0x04c941), - (hex!("01222222223333333344444444550000039b0000000000004c10"), 0x04c9e1), - (hex!("01222222223333333344444444550000039b0000000000006460"), 0x04ca81), - (hex!("01222222223333333344444444550000039c00000000000039d0"), 0x04cb21), - (hex!("01222222223333333344444444550000039d00000000000039e0"), 0x04cbc1), - (hex!("01222222223333333344444444550000039d00000000000044c0"), 0x04cc61), - (hex!("01222222223333333344444444550000039d00000000000049e0"), 0x04cd01), - (hex!("01222222223333333344444444550000039e00000000000039f0"), 0x04cda1), - (hex!("01222222223333333344444444550000039f0000000000003a00"), 0x04ce41), - (hex!("0122222222333333334444444455000003a00000000000003a10"), 0x04cee1), - (hex!("0122222222333333334444444455000003a10000000000003a20"), 0x04cf81), - (hex!("0122222222333333334444444455000003a10000000000006a80"), 0x04d021), - (hex!("0122222222333333334444444455000003a20000000000003a30"), 0x04d0c1), - (hex!("0122222222333333334444444455000003a200000000000062b0"), 0x04d161), - (hex!("0122222222333333334444444455000003a30000000000003a40"), 0x04d201), - (hex!("0122222222333333334444444455000003a30000000000006ce0"), 0x04d2a1), - (hex!("0122222222333333334444444455000003a40000000000003a50"), 0x04d341), - (hex!("0122222222333333334444444455000003a50000000000003a60"), 0x04d3e1), - (hex!("0122222222333333334444444455000003a60000000000003a70"), 0x04d481), - (hex!("0122222222333333334444444455000003a60000000000007750"), 0x04d521), - (hex!("0122222222333333334444444455000003a70000000000003a80"), 0x04d5c1), - (hex!("0122222222333333334444444455000003a70000000000005b10"), 0x04d661), - (hex!("0122222222333333334444444455000003a80000000000003a90"), 0x04d701), - (hex!("0122222222333333334444444455000003a80000000000006c20"), 0x04d7a1), - (hex!("0122222222333333334444444455000003a90000000000003aa0"), 0x04d841), - (hex!("0122222222333333334444444455000003a90000000000005b70"), 0x04d8e1), - (hex!("0122222222333333334444444455000003a900000000000070e0"), 0x04d981), - (hex!("0122222222333333334444444455000003aa0000000000003ab0"), 0x04da21), - (hex!("0122222222333333334444444455000003aa00000000000049f0"), 0x04dac1), - (hex!("0122222222333333334444444455000003aa0000000000004d60"), 0x04db61), - (hex!("0122222222333333334444444455000003ab0000000000003ac0"), 0x04dc01), - (hex!("0122222222333333334444444455000003ac0000000000003ad0"), 0x04dca1), - (hex!("0122222222333333334444444455000003ac0000000000004580"), 0x04dd41), - (hex!("0122222222333333334444444455000003ad0000000000003ae0"), 0x04dde1), - (hex!("0122222222333333334444444455000003ae0000000000003af0"), 0x04de81), - (hex!("0122222222333333334444444455000003af0000000000003b00"), 0x04df21), - (hex!("0122222222333333334444444455000003b00000000000003b10"), 0x04dfc1), - (hex!("0122222222333333334444444455000003b10000000000003b20"), 0x04e061), - (hex!("0122222222333333334444444455000003b10000000000003fd0"), 0x04e101), - (hex!("0122222222333333334444444455000003b20000000000003b30"), 0x04e1a1), - (hex!("0122222222333333334444444455000003b30000000000003b40"), 0x04e241), - (hex!("0122222222333333334444444455000003b40000000000003b50"), 0x04e2e1), - (hex!("0122222222333333334444444455000003b40000000000007450"), 0x04e381), - (hex!("0122222222333333334444444455000003b50000000000003b60"), 0x04e421), - (hex!("0122222222333333334444444455000003b60000000000003b70"), 0x04e4c1), - (hex!("0122222222333333334444444455000003b70000000000003b80"), 0x04e561), - (hex!("0122222222333333334444444455000003b70000000000006d50"), 0x04e601), - (hex!("0122222222333333334444444455000003b80000000000003b90"), 0x04e6a1), - (hex!("0122222222333333334444444455000003b800000000000057c0"), 0x04e741), - (hex!("0122222222333333334444444455000003b800000000000078a0"), 0x04e7e1), - (hex!("0122222222333333334444444455000003b90000000000003ba0"), 0x04e881), - (hex!("0122222222333333334444444455000003b90000000000006750"), 0x04e921), - (hex!("0122222222333333334444444455000003ba0000000000003bb0"), 0x04e9c1), - (hex!("0122222222333333334444444455000003ba0000000000007a10"), 0x04ea61), - (hex!("0122222222333333334444444455000003ba0000000000007a20"), 0x04eb01), - (hex!("0122222222333333334444444455000003bb0000000000003bc0"), 0x04eba1), - (hex!("0122222222333333334444444455000003bb0000000000005bc0"), 0x04ec41), - (hex!("0122222222333333334444444455000003bc0000000000003bd0"), 0x04ece1), - (hex!("0122222222333333334444444455000003bc0000000000005e80"), 0x04ed81), - (hex!("0122222222333333334444444455000003bc0000000000007ab0"), 0x04ee21), - (hex!("0122222222333333334444444455000003bd0000000000003be0"), 0x04eec1), - (hex!("0122222222333333334444444455000003bd00000000000049b0"), 0x04ef61), - (hex!("0122222222333333334444444455000003be0000000000003bf0"), 0x04f001), - (hex!("0122222222333333334444444455000003be0000000000005780"), 0x04f0a1), - (hex!("0122222222333333334444444455000003be0000000000007930"), 0x04f141), - (hex!("0122222222333333334444444455000003bf0000000000003c00"), 0x04f1e1), - (hex!("0122222222333333334444444455000003bf0000000000005de0"), 0x04f281), - (hex!("0122222222333333334444444455000003bf00000000000060b0"), 0x04f321), - (hex!("0122222222333333334444444455000003bf00000000000060c0"), 0x04f3c1), - (hex!("0122222222333333334444444455000003bf0000000000006a50"), 0x04f461), - (hex!("0122222222333333334444444455000003c00000000000003c10"), 0x04f501), - (hex!("0122222222333333334444444455000003c00000000000004030"), 0x04f5a1), - (hex!("0122222222333333334444444455000003c10000000000003c20"), 0x04f641), - (hex!("0122222222333333334444444455000003c20000000000003c30"), 0x04f6e1), - (hex!("0122222222333333334444444455000003c200000000000040b0"), 0x04f781), - (hex!("0122222222333333334444444455000003c30000000000003c40"), 0x04f821), - (hex!("0122222222333333334444444455000003c40000000000003c50"), 0x04f8c1), - (hex!("0122222222333333334444444455000003c40000000000005ba0"), 0x04f961), - (hex!("0122222222333333334444444455000003c50000000000003c60"), 0x04fa01), - (hex!("0122222222333333334444444455000003c60000000000003c70"), 0x04faa1), - (hex!("0122222222333333334444444455000003c70000000000003c80"), 0x04fb41), - (hex!("0122222222333333334444444455000003c70000000000004270"), 0x04fbe1), - (hex!("0122222222333333334444444455000003c80000000000003c90"), 0x04fc81), - (hex!("0122222222333333334444444455000003c80000000000006e70"), 0x04fd21), - (hex!("0122222222333333334444444455000003c90000000000003ca0"), 0x04fdc1), - (hex!("0122222222333333334444444455000003ca0000000000003cb0"), 0x04fe61), - (hex!("0122222222333333334444444455000003ca0000000000006e20"), 0x04ff01), - (hex!("0122222222333333334444444455000003ca0000000000007c20"), 0x04ffa1), - (hex!("0122222222333333334444444455000003cb0000000000003cc0"), 0x050041), - (hex!("0122222222333333334444444455000003cc0000000000003cd0"), 0x0500e1), - (hex!("0122222222333333334444444455000003cc0000000000006120"), 0x050181), - (hex!("0122222222333333334444444455000003cc0000000000007950"), 0x050221), - (hex!("0122222222333333334444444455000003cd0000000000003ce0"), 0x0502c1), - (hex!("0122222222333333334444444455000003ce0000000000003cf0"), 0x050361), - (hex!("0122222222333333334444444455000003cf0000000000003d00"), 0x050401), - (hex!("0122222222333333334444444455000003d00000000000003d10"), 0x0504a1), - (hex!("0122222222333333334444444455000003d10000000000003d20"), 0x050541), - (hex!("0122222222333333334444444455000003d10000000000005e50"), 0x0505e1), - (hex!("0122222222333333334444444455000003d10000000000007880"), 0x050681), - (hex!("0122222222333333334444444455000003d20000000000003d30"), 0x050721), - (hex!("0122222222333333334444444455000003d20000000000005d00"), 0x0507c1), - (hex!("0122222222333333334444444455000003d30000000000003d40"), 0x050861), - (hex!("0122222222333333334444444455000003d30000000000005d40"), 0x050901), - (hex!("0122222222333333334444444455000003d300000000000063f0"), 0x0509a1), - (hex!("0122222222333333334444444455000003d40000000000003d50"), 0x050a41), - (hex!("0122222222333333334444444455000003d40000000000005700"), 0x050ae1), - (hex!("0122222222333333334444444455000003d400000000000078f0"), 0x050b81), - (hex!("0122222222333333334444444455000003d50000000000003d60"), 0x050c21), - (hex!("0122222222333333334444444455000003d60000000000003d70"), 0x050cc1), - (hex!("0122222222333333334444444455000003d70000000000003d80"), 0x050d61), - (hex!("0122222222333333334444444455000003d80000000000003d90"), 0x050e01), - (hex!("0122222222333333334444444455000003d80000000000006690"), 0x050ea1), - (hex!("0122222222333333334444444455000003d90000000000003da0"), 0x050f41), - (hex!("0122222222333333334444444455000003d900000000000076d0"), 0x050fe1), - (hex!("0122222222333333334444444455000003da0000000000003db0"), 0x051081), - (hex!("0122222222333333334444444455000003db0000000000003dc0"), 0x051121), - (hex!("0122222222333333334444444455000003db0000000000004a30"), 0x0511c1), - (hex!("0122222222333333334444444455000003db0000000000005390"), 0x051261), - (hex!("0122222222333333334444444455000003dc0000000000003dd0"), 0x051301), - (hex!("0122222222333333334444444455000003dc0000000000006d60"), 0x0513a1), - (hex!("0122222222333333334444444455000003dd0000000000003de0"), 0x051441), - (hex!("0122222222333333334444444455000003de0000000000003df0"), 0x0514e1), - (hex!("0122222222333333334444444455000003df0000000000003e00"), 0x051581), - (hex!("0122222222333333334444444455000003df0000000000005240"), 0x051621), - (hex!("0122222222333333334444444455000003df0000000000005610"), 0x0516c1), - (hex!("0122222222333333334444444455000003e00000000000003e10"), 0x051761), - (hex!("0122222222333333334444444455000003e00000000000006500"), 0x051801), - (hex!("0122222222333333334444444455000003e10000000000003e20"), 0x0518a1), - (hex!("0122222222333333334444444455000003e10000000000006a10"), 0x051941), - (hex!("0122222222333333334444444455000003e10000000000007c10"), 0x0519e1), - (hex!("0122222222333333334444444455000003e20000000000003e30"), 0x051a81), - (hex!("0122222222333333334444444455000003e20000000000006310"), 0x051b21), - (hex!("0122222222333333334444444455000003e30000000000003e40"), 0x051bc1), - (hex!("0122222222333333334444444455000003e40000000000003e50"), 0x051c61), - (hex!("0122222222333333334444444455000003e40000000000006780"), 0x051d01), - (hex!("0122222222333333334444444455000003e40000000000007ce0"), 0x051da1), - (hex!("0122222222333333334444444455000003e50000000000003e60"), 0x051e41), - (hex!("0122222222333333334444444455000003e60000000000003e70"), 0x051ee1), - (hex!("0122222222333333334444444455000003e60000000000005040"), 0x051f81), - (hex!("0122222222333333334444444455000003e60000000000005bf0"), 0x052021), - (hex!("0122222222333333334444444455000003e70000000000003e80"), 0x0520c1), - (hex!("0122222222333333334444444455000003e70000000000003f50"), 0x052161), + (hex!("0100000000333333334444444455000000000000000000000010"), 0x004001), + (hex!("0100000000333333334444444455000000000000000000007cb0"), 0x0040a1), + (hex!("0100000000333333334444444455000000010000000000000020"), 0x004141), + (hex!("0100000000333333334444444455000000020000000000000030"), 0x0041e1), + (hex!("01000000003333333344444444550000000200000000000051a0"), 0x004281), + (hex!("0100000000333333334444444455000000030000000000000040"), 0x004321), + (hex!("0100000000333333334444444455000000030000000000006cf0"), 0x0043c1), + (hex!("0100000000333333334444444455000000030000000000007140"), 0x004461), + (hex!("0100000000333333334444444455000000040000000000000050"), 0x004501), + (hex!("01000000003333333344444444550000000400000000000047f0"), 0x0045a1), + (hex!("01000000003333333344444444550000000400000000000072b0"), 0x004641), + (hex!("0100000000333333334444444455000000050000000000000060"), 0x0046e1), + (hex!("0100000000333333334444444455000000050000000000005550"), 0x004781), + (hex!("0100000000333333334444444455000000060000000000000070"), 0x004821), + (hex!("01000000003333333344444444550000000600000000000044a0"), 0x0048c1), + (hex!("0100000000333333334444444455000000060000000000006870"), 0x004961), + (hex!("0100000000333333334444444455000000070000000000000080"), 0x004a01), + (hex!("0100000000333333334444444455000000080000000000000090"), 0x004aa1), + (hex!("0100000000333333334444444455000000080000000000004150"), 0x004b41), + (hex!("01000000003333333344444444550000000900000000000000a0"), 0x004be1), + (hex!("01000000003333333344444444550000000a00000000000000b0"), 0x004c81), + (hex!("01000000003333333344444444550000000a0000000000006680"), 0x004d21), + (hex!("01000000003333333344444444550000000b00000000000000c0"), 0x004dc1), + (hex!("01000000003333333344444444550000000b0000000000006230"), 0x004e61), + (hex!("01000000003333333344444444550000000c00000000000000d0"), 0x004f01), + (hex!("01000000003333333344444444550000000d00000000000000e0"), 0x004fa1), + (hex!("01000000003333333344444444550000000e00000000000000f0"), 0x005041), + (hex!("01000000003333333344444444550000000e0000000000006000"), 0x0050e1), + (hex!("01000000003333333344444444550000000f0000000000000100"), 0x005181), + (hex!("01000000003333333344444444550000000f00000000000053c0"), 0x005221), + (hex!("01000000003333333344444444550000000f0000000000006580"), 0x0052c1), + (hex!("0100000000333333334444444455000000100000000000000110"), 0x005361), + (hex!("01000000003333333344444444550000001000000000000046c0"), 0x005401), + (hex!("0100000000333333334444444455000000100000000000004e40"), 0x0054a1), + (hex!("0100000000333333334444444455000000110000000000000120"), 0x005541), + (hex!("0100000000333333334444444455000000120000000000000130"), 0x0055e1), + (hex!("01000000003333333344444444550000001200000000000066d0"), 0x005681), + (hex!("0100000000333333334444444455000000130000000000000140"), 0x005721), + (hex!("0100000000333333334444444455000000130000000000007710"), 0x0057c1), + (hex!("0100000000333333334444444455000000140000000000000150"), 0x005861), + (hex!("0100000000333333334444444455000000140000000000006c40"), 0x005901), + (hex!("0100000000333333334444444455000000150000000000000160"), 0x0059a1), + (hex!("0100000000333333334444444455000000150000000000005990"), 0x005a41), + (hex!("0100000000333333334444444455000000160000000000000170"), 0x005ae1), + (hex!("0100000000333333334444444455000000160000000000005530"), 0x005b81), + (hex!("0100000000333333334444444455000000170000000000000180"), 0x005c21), + (hex!("0100000000333333334444444455000000170000000000004290"), 0x005cc1), + (hex!("0100000000333333334444444455000000180000000000000190"), 0x005d61), + (hex!("01000000003333333344444444550000001800000000000051c0"), 0x005e01), + (hex!("01000000003333333344444444550000001900000000000001a0"), 0x005ea1), + (hex!("0100000000333333334444444455000000190000000000005420"), 0x005f41), + (hex!("0100000000333333334444444455000000190000000000005770"), 0x005fe1), + (hex!("01000000003333333344444444550000001900000000000079d0"), 0x006081), + (hex!("01000000003333333344444444550000001a00000000000001b0"), 0x006121), + (hex!("01000000003333333344444444550000001a0000000000006f70"), 0x0061c1), + (hex!("01000000003333333344444444550000001a0000000000007150"), 0x006261), + (hex!("01000000003333333344444444550000001b00000000000001c0"), 0x006301), + (hex!("01000000003333333344444444550000001b0000000000005070"), 0x0063a1), + (hex!("01000000003333333344444444550000001c00000000000001d0"), 0x006441), + (hex!("01000000003333333344444444550000001d00000000000001e0"), 0x0064e1), + (hex!("01000000003333333344444444550000001e00000000000001f0"), 0x006581), + (hex!("01000000003333333344444444550000001e0000000000005650"), 0x006621), + (hex!("01000000003333333344444444550000001f0000000000000200"), 0x0066c1), + (hex!("01000000003333333344444444550000001f0000000000006ca0"), 0x006761), + (hex!("0100000000333333334444444455000000200000000000000210"), 0x006801), + (hex!("0100000000333333334444444455000000200000000000005fc0"), 0x0068a1), + (hex!("0100000000333333334444444455000000210000000000000220"), 0x006941), + (hex!("0100000000333333334444444455000000210000000000006430"), 0x0069e1), + (hex!("0100000000333333334444444455000000220000000000000230"), 0x006a81), + (hex!("01000000003333333344444444550000002200000000000040e0"), 0x006b21), + (hex!("0100000000333333334444444455000000230000000000000240"), 0x006bc1), + (hex!("01000000003333333344444444550000002300000000000042d0"), 0x006c61), + (hex!("0100000000333333334444444455000000240000000000000250"), 0x006d01), + (hex!("0100000000333333334444444455000000250000000000000260"), 0x006da1), + (hex!("01000000003333333344444444550000002500000000000058c0"), 0x006e41), + (hex!("0100000000333333334444444455000000260000000000000270"), 0x006ee1), + (hex!("0100000000333333334444444455000000260000000000004020"), 0x006f81), + (hex!("0100000000333333334444444455000000270000000000000280"), 0x007021), + (hex!("0100000000333333334444444455000000280000000000000290"), 0x0070c1), + (hex!("0100000000333333334444444455000000280000000000007c00"), 0x007161), + (hex!("01000000003333333344444444550000002900000000000002a0"), 0x007201), + (hex!("01000000003333333344444444550000002a00000000000002b0"), 0x0072a1), + (hex!("01000000003333333344444444550000002b00000000000002c0"), 0x007341), + (hex!("01000000003333333344444444550000002c00000000000002d0"), 0x0073e1), + (hex!("01000000003333333344444444550000002c00000000000041b0"), 0x007481), + (hex!("01000000003333333344444444550000002c0000000000004c30"), 0x007521), + (hex!("01000000003333333344444444550000002d00000000000002e0"), 0x0075c1), + (hex!("01000000003333333344444444550000002d0000000000005e40"), 0x007661), + (hex!("01000000003333333344444444550000002d0000000000006990"), 0x007701), + (hex!("01000000003333333344444444550000002e00000000000002f0"), 0x0077a1), + (hex!("01000000003333333344444444550000002f0000000000000300"), 0x007841), + (hex!("01000000003333333344444444550000002f0000000000004a70"), 0x0078e1), + (hex!("01000000003333333344444444550000002f0000000000006b40"), 0x007981), + (hex!("0100000000333333334444444455000000300000000000000310"), 0x007a21), + (hex!("0100000000333333334444444455000000310000000000000320"), 0x007ac1), + (hex!("0100000000333333334444444455000000320000000000000330"), 0x007b61), + (hex!("01000000003333333344444444550000003200000000000041a0"), 0x007c01), + (hex!("0100000000333333334444444455000000320000000000007340"), 0x007ca1), + (hex!("0100000000333333334444444455000000320000000000007730"), 0x007d41), + (hex!("0100000000333333334444444455000000330000000000000340"), 0x007de1), + (hex!("01000000003333333344444444550000003300000000000055a0"), 0x007e81), + (hex!("0100000000333333334444444455000000340000000000000350"), 0x007f21), + (hex!("0100000000333333334444444455000000350000000000000360"), 0x007fc1), + (hex!("01000000003333333344444444550000003500000000000077a0"), 0x008061), + (hex!("0100000000333333334444444455000000360000000000000370"), 0x008101), + (hex!("0100000000333333334444444455000000370000000000000380"), 0x0081a1), + (hex!("0100000000333333334444444455000000380000000000000390"), 0x008241), + (hex!("01000000003333333344444444550000003900000000000003a0"), 0x0082e1), + (hex!("01000000003333333344444444550000003a00000000000003b0"), 0x008381), + (hex!("01000000003333333344444444550000003a00000000000071c0"), 0x008421), + (hex!("01000000003333333344444444550000003b00000000000003c0"), 0x0084c1), + (hex!("01000000003333333344444444550000003c00000000000003d0"), 0x008561), + (hex!("01000000003333333344444444550000003d00000000000003e0"), 0x008601), + (hex!("01000000003333333344444444550000003e00000000000003f0"), 0x0086a1), + (hex!("01000000003333333344444444550000003e00000000000062e0"), 0x008741), + (hex!("01000000003333333344444444550000003f0000000000000400"), 0x0087e1), + (hex!("0100000000333333334444444455000000400000000000000410"), 0x008881), + (hex!("0100000000333333334444444455000000400000000000004460"), 0x008921), + (hex!("0100000000333333334444444455000000400000000000005b90"), 0x0089c1), + (hex!("01000000003333333344444444550000004000000000000079b0"), 0x008a61), + (hex!("0100000000333333334444444455000000410000000000000420"), 0x008b01), + (hex!("0100000000333333334444444455000000420000000000000430"), 0x008ba1), + (hex!("0100000000333333334444444455000000420000000000005640"), 0x008c41), + (hex!("0100000000333333334444444455000000430000000000000440"), 0x008ce1), + (hex!("01000000003333333344444444550000004300000000000072a0"), 0x008d81), + (hex!("0100000000333333334444444455000000440000000000000450"), 0x008e21), + (hex!("0100000000333333334444444455000000450000000000000460"), 0x008ec1), + (hex!("0100000000333333334444444455000000450000000000005750"), 0x008f61), + (hex!("01000000003333333344444444550000004500000000000077b0"), 0x009001), + (hex!("0100000000333333334444444455000000460000000000000470"), 0x0090a1), + (hex!("0100000000333333334444444455000000470000000000000480"), 0x009141), + (hex!("0100000000333333334444444455000000480000000000000490"), 0x0091e1), + (hex!("01000000003333333344444444550000004800000000000069e0"), 0x009281), + (hex!("01000000003333333344444444550000004900000000000004a0"), 0x009321), + (hex!("0100000000333333334444444455000000490000000000007370"), 0x0093c1), + (hex!("01000000003333333344444444550000004a00000000000004b0"), 0x009461), + (hex!("01000000003333333344444444550000004a0000000000005cb0"), 0x009501), + (hex!("01000000003333333344444444550000004b00000000000004c0"), 0x0095a1), + (hex!("01000000003333333344444444550000004c00000000000004d0"), 0x009641), + (hex!("01000000003333333344444444550000004c0000000000004880"), 0x0096e1), + (hex!("01000000003333333344444444550000004c0000000000007a40"), 0x009781), + (hex!("01000000003333333344444444550000004d00000000000004e0"), 0x009821), + (hex!("01000000003333333344444444550000004d0000000000006390"), 0x0098c1), + (hex!("01000000003333333344444444550000004e00000000000004f0"), 0x009961), + (hex!("01000000003333333344444444550000004e0000000000004db0"), 0x009a01), + (hex!("01000000003333333344444444550000004f0000000000000500"), 0x009aa1), + (hex!("0100000000333333334444444455000000500000000000000510"), 0x009b41), + (hex!("0100000000333333334444444455000000510000000000000520"), 0x009be1), + (hex!("01000000003333333344444444550000005100000000000069c0"), 0x009c81), + (hex!("0100000000333333334444444455000000520000000000000530"), 0x009d21), + (hex!("0100000000333333334444444455000000520000000000006e60"), 0x009dc1), + (hex!("01000000003333333344444444550000005200000000000070c0"), 0x009e61), + (hex!("0100000000333333334444444455000000530000000000000540"), 0x009f01), + (hex!("0100000000333333334444444455000000530000000000005840"), 0x009fa1), + (hex!("0100000000333333334444444455000000540000000000000550"), 0x00a041), + (hex!("01000000003333333344444444550000005400000000000043e0"), 0x00a0e1), + (hex!("01000000003333333344444444550000005400000000000074e0"), 0x00a181), + (hex!("0100000000333333334444444455000000550000000000000560"), 0x00a221), + (hex!("0100000000333333334444444455000000550000000000003ee0"), 0x00a2c1), + (hex!("0100000000333333334444444455000000560000000000000570"), 0x00a361), + (hex!("0100000000333333334444444455000000570000000000000580"), 0x00a401), + (hex!("0100000000333333334444444455000000570000000000007030"), 0x00a4a1), + (hex!("0100000000333333334444444455000000580000000000000590"), 0x00a541), + (hex!("0100000000333333334444444455000000580000000000005340"), 0x00a5e1), + (hex!("01000000003333333344444444550000005800000000000059f0"), 0x00a681), + (hex!("0100000000333333334444444455000000580000000000006930"), 0x00a721), + (hex!("01000000003333333344444444550000005900000000000005a0"), 0x00a7c1), + (hex!("0100000000333333334444444455000000590000000000003f90"), 0x00a861), + (hex!("01000000003333333344444444550000005a00000000000005b0"), 0x00a901), + (hex!("01000000003333333344444444550000005b00000000000005c0"), 0x00a9a1), + (hex!("01000000003333333344444444550000005b00000000000062c0"), 0x00aa41), + (hex!("01000000003333333344444444550000005c00000000000005d0"), 0x00aae1), + (hex!("01000000003333333344444444550000005c0000000000005a70"), 0x00ab81), + (hex!("01000000003333333344444444550000005c0000000000005dd0"), 0x00ac21), + (hex!("01000000003333333344444444550000005d00000000000005e0"), 0x00acc1), + (hex!("01000000003333333344444444550000005d0000000000005730"), 0x00ad61), + (hex!("01000000003333333344444444550000005e00000000000005f0"), 0x00ae01), + (hex!("01000000003333333344444444550000005e0000000000004f40"), 0x00aea1), + (hex!("01000000003333333344444444550000005f0000000000000600"), 0x00af41), + (hex!("0100000000333333334444444455000000600000000000000610"), 0x00afe1), + (hex!("0100000000333333334444444455000000600000000000007c40"), 0x00b081), + (hex!("0100000000333333334444444455000000610000000000000620"), 0x00b121), + (hex!("0100000000333333334444444455000000610000000000007860"), 0x00b1c1), + (hex!("0100000000333333334444444455000000620000000000000630"), 0x00b261), + (hex!("0100000000333333334444444455000000620000000000005050"), 0x00b301), + (hex!("0100000000333333334444444455000000630000000000000640"), 0x00b3a1), + (hex!("0100000000333333334444444455000000640000000000000650"), 0x00b441), + (hex!("0100000000333333334444444455000000650000000000000660"), 0x00b4e1), + (hex!("0100000000333333334444444455000000650000000000005330"), 0x00b581), + (hex!("0100000000333333334444444455000000660000000000000670"), 0x00b621), + (hex!("0100000000333333334444444455000000660000000000004e20"), 0x00b6c1), + (hex!("0100000000333333334444444455000000660000000000005ee0"), 0x00b761), + (hex!("0100000000333333334444444455000000660000000000006360"), 0x00b801), + (hex!("0100000000333333334444444455000000670000000000000680"), 0x00b8a1), + (hex!("0100000000333333334444444455000000670000000000004040"), 0x00b941), + (hex!("0100000000333333334444444455000000680000000000000690"), 0x00b9e1), + (hex!("0100000000333333334444444455000000680000000000003f80"), 0x00ba81), + (hex!("01000000003333333344444444550000006800000000000041e0"), 0x00bb21), + (hex!("01000000003333333344444444550000006900000000000006a0"), 0x00bbc1), + (hex!("0100000000333333334444444455000000690000000000006080"), 0x00bc61), + (hex!("01000000003333333344444444550000006a00000000000006b0"), 0x00bd01), + (hex!("01000000003333333344444444550000006a00000000000042f0"), 0x00bda1), + (hex!("01000000003333333344444444550000006b00000000000006c0"), 0x00be41), + (hex!("01000000003333333344444444550000006b00000000000052f0"), 0x00bee1), + (hex!("01000000003333333344444444550000006b0000000000005980"), 0x00bf81), + (hex!("01000000003333333344444444550000006b0000000000006170"), 0x00c021), + (hex!("01000000003333333344444444550000006c00000000000006d0"), 0x00c0c1), + (hex!("01000000003333333344444444550000006d00000000000006e0"), 0x00c161), + (hex!("01000000003333333344444444550000006d0000000000006fb0"), 0x00c201), + (hex!("01000000003333333344444444550000006e00000000000006f0"), 0x00c2a1), + (hex!("01000000003333333344444444550000006e00000000000065b0"), 0x00c341), + (hex!("01000000003333333344444444550000006e0000000000007970"), 0x00c3e1), + (hex!("01000000003333333344444444550000006f0000000000000700"), 0x00c481), + (hex!("01000000003333333344444444550000006f0000000000005900"), 0x00c521), + (hex!("01000000003333333344444444550000006f0000000000006d90"), 0x00c5c1), + (hex!("0100000000333333334444444455000000700000000000000710"), 0x00c661), + (hex!("01000000003333333344444444550000007000000000000045c0"), 0x00c701), + (hex!("0100000000333333334444444455000000700000000000004d40"), 0x00c7a1), + (hex!("0100000000333333334444444455000000710000000000000720"), 0x00c841), + (hex!("0100000000333333334444444455000000710000000000004dc0"), 0x00c8e1), + (hex!("0100000000333333334444444455000000710000000000007550"), 0x00c981), + (hex!("0100000000333333334444444455000000720000000000000730"), 0x00ca21), + (hex!("0100000000333333334444444455000000720000000000003ec0"), 0x00cac1), + (hex!("01000000003333333344444444550000007200000000000045a0"), 0x00cb61), + (hex!("0100000000333333334444444455000000720000000000006770"), 0x00cc01), + (hex!("0100000000333333334444444455000000720000000000006bc0"), 0x00cca1), + (hex!("0100000000333333334444444455000000730000000000000740"), 0x00cd41), + (hex!("0100000000333333334444444455000000730000000000005250"), 0x00cde1), + (hex!("01000000003333333344444444550000007300000000000075f0"), 0x00ce81), + (hex!("0100000000333333334444444455000000740000000000000750"), 0x00cf21), + (hex!("0100000000333333334444444455000000740000000000003ff0"), 0x00cfc1), + (hex!("01000000003333333344444444550000007400000000000079e0"), 0x00d061), + (hex!("0100000000333333334444444455000000750000000000000760"), 0x00d101), + (hex!("0100000000333333334444444455000000750000000000004310"), 0x00d1a1), + (hex!("0100000000333333334444444455000000760000000000000770"), 0x00d241), + (hex!("0100000000333333334444444455000000770000000000000780"), 0x00d2e1), + (hex!("01000000003333333344444444550000007700000000000062f0"), 0x00d381), + (hex!("0100000000333333334444444455000000770000000000006940"), 0x00d421), + (hex!("0100000000333333334444444455000000780000000000000790"), 0x00d4c1), + (hex!("01000000003333333344444444550000007900000000000007a0"), 0x00d561), + (hex!("0100000000333333334444444455000000790000000000007af0"), 0x00d601), + (hex!("01000000003333333344444444550000007a00000000000007b0"), 0x00d6a1), + (hex!("01000000003333333344444444550000007b00000000000007c0"), 0x00d741), + (hex!("01000000003333333344444444550000007b00000000000067e0"), 0x00d7e1), + (hex!("01000000003333333344444444550000007b0000000000007890"), 0x00d881), + (hex!("01000000003333333344444444550000007c00000000000007d0"), 0x00d921), + (hex!("01000000003333333344444444550000007d00000000000007e0"), 0x00d9c1), + (hex!("01000000003333333344444444550000007e00000000000007f0"), 0x00da61), + (hex!("01000000003333333344444444550000007f0000000000000800"), 0x00db01), + (hex!("01000000003333333344444444550000007f0000000000005be0"), 0x00dba1), + (hex!("0100000000333333334444444455000000800000000000000810"), 0x00dc41), + (hex!("0100000000333333334444444455000000810000000000000820"), 0x00dce1), + (hex!("0100000000333333334444444455000000810000000000007190"), 0x00dd81), + (hex!("0100000000333333334444444455000000820000000000000830"), 0x00de21), + (hex!("0100000000333333334444444455000000820000000000004ab0"), 0x00dec1), + (hex!("0100000000333333334444444455000000830000000000000840"), 0x00df61), + (hex!("0100000000333333334444444455000000830000000000006720"), 0x00e001), + (hex!("0100000000333333334444444455000000840000000000000850"), 0x00e0a1), + (hex!("0100000000333333334444444455000000850000000000000860"), 0x00e141), + (hex!("01000000003333333344444444550000008500000000000054f0"), 0x00e1e1), + (hex!("0100000000333333334444444455000000850000000000007920"), 0x00e281), + (hex!("0100000000333333334444444455000000860000000000000870"), 0x00e321), + (hex!("01000000003333333344444444550000008600000000000060e0"), 0x00e3c1), + (hex!("0100000000333333334444444455000000860000000000006be0"), 0x00e461), + (hex!("0100000000333333334444444455000000870000000000000880"), 0x00e501), + (hex!("0100000000333333334444444455000000870000000000006820"), 0x00e5a1), + (hex!("0100000000333333334444444455000000880000000000000890"), 0x00e641), + (hex!("01000000003333333344444444550000008900000000000008a0"), 0x00e6e1), + (hex!("0100000000333333334444444455000000890000000000007c30"), 0x00e781), + (hex!("01000000003333333344444444550000008a00000000000008b0"), 0x00e821), + (hex!("01000000003333333344444444550000008b00000000000008c0"), 0x00e8c1), + (hex!("01000000003333333344444444550000008b0000000000005910"), 0x00e961), + (hex!("01000000003333333344444444550000008b0000000000006fe0"), 0x00ea01), + (hex!("01000000003333333344444444550000008c00000000000008d0"), 0x00eaa1), + (hex!("01000000003333333344444444550000008c0000000000006800"), 0x00eb41), + (hex!("01000000003333333344444444550000008d00000000000008e0"), 0x00ebe1), + (hex!("01000000003333333344444444550000008d0000000000005810"), 0x00ec81), + (hex!("01000000003333333344444444550000008d0000000000007c90"), 0x00ed21), + (hex!("01000000003333333344444444550000008e00000000000008f0"), 0x00edc1), + (hex!("01000000003333333344444444550000008e00000000000058f0"), 0x00ee61), + (hex!("01000000003333333344444444550000008f0000000000000900"), 0x00ef01), + (hex!("01000000003333333344444444550000008f0000000000005a30"), 0x00efa1), + (hex!("0100000000333333334444444455000000900000000000000910"), 0x00f041), + (hex!("0100000000333333334444444455000000900000000000006130"), 0x00f0e1), + (hex!("0100000000333333334444444455000000900000000000006550"), 0x00f181), + (hex!("0100000000333333334444444455000000910000000000000920"), 0x00f221), + (hex!("01000000003333333344444444550000009100000000000079f0"), 0x00f2c1), + (hex!("0100000000333333334444444455000000920000000000000930"), 0x00f361), + (hex!("0100000000333333334444444455000000920000000000005620"), 0x00f401), + (hex!("0100000000333333334444444455000000920000000000005e90"), 0x00f4a1), + (hex!("01000000003333333344444444550000009200000000000063d0"), 0x00f541), + (hex!("01000000003333333344444444550000009200000000000076c0"), 0x00f5e1), + (hex!("0100000000333333334444444455000000930000000000000940"), 0x00f681), + (hex!("01000000003333333344444444550000009300000000000044e0"), 0x00f721), + (hex!("0100000000333333334444444455000000940000000000000950"), 0x00f7c1), + (hex!("0100000000333333334444444455000000940000000000007a30"), 0x00f861), + (hex!("0100000000333333334444444455000000950000000000000960"), 0x00f901), + (hex!("0100000000333333334444444455000000950000000000007a70"), 0x00f9a1), + (hex!("0100000000333333334444444455000000960000000000000970"), 0x00fa41), + (hex!("0100000000333333334444444455000000970000000000000980"), 0x00fae1), + (hex!("0100000000333333334444444455000000970000000000007330"), 0x00fb81), + (hex!("0100000000333333334444444455000000980000000000000990"), 0x00fc21), + (hex!("0100000000333333334444444455000000980000000000005af0"), 0x00fcc1), + (hex!("0100000000333333334444444455000000980000000000007ae0"), 0x00fd61), + (hex!("01000000003333333344444444550000009900000000000009a0"), 0x00fe01), + (hex!("0100000000333333334444444455000000990000000000005160"), 0x00fea1), + (hex!("0100000000333333334444444455000000990000000000006850"), 0x00ff41), + (hex!("01000000003333333344444444550000009a00000000000009b0"), 0x00ffe1), + (hex!("01000000003333333344444444550000009b00000000000009c0"), 0x010081), + (hex!("01000000003333333344444444550000009b0000000000005010"), 0x010121), + (hex!("01000000003333333344444444550000009c00000000000009d0"), 0x0101c1), + (hex!("01000000003333333344444444550000009c00000000000042e0"), 0x010261), + (hex!("01000000003333333344444444550000009d00000000000009e0"), 0x010301), + (hex!("01000000003333333344444444550000009d00000000000057f0"), 0x0103a1), + (hex!("01000000003333333344444444550000009e00000000000009f0"), 0x010441), + (hex!("01000000003333333344444444550000009e0000000000004ef0"), 0x0104e1), + (hex!("01000000003333333344444444550000009f0000000000000a00"), 0x010581), + (hex!("01000000003333333344444444550000009f0000000000006110"), 0x010621), + (hex!("0100000000333333334444444455000000a00000000000000a10"), 0x0106c1), + (hex!("0100000000333333334444444455000000a10000000000000a20"), 0x010761), + (hex!("0100000000333333334444444455000000a100000000000040d0"), 0x010801), + (hex!("0100000000333333334444444455000000a10000000000007670"), 0x0108a1), + (hex!("0100000000333333334444444455000000a20000000000000a30"), 0x010941), + (hex!("0100000000333333334444444455000000a200000000000074d0"), 0x0109e1), + (hex!("0100000000333333334444444455000000a30000000000000a40"), 0x010a81), + (hex!("0100000000333333334444444455000000a30000000000004c90"), 0x010b21), + (hex!("0100000000333333334444444455000000a40000000000000a50"), 0x010bc1), + (hex!("0100000000333333334444444455000000a50000000000000a60"), 0x010c61), + (hex!("0100000000333333334444444455000000a60000000000000a70"), 0x010d01), + (hex!("0100000000333333334444444455000000a60000000000006d80"), 0x010da1), + (hex!("0100000000333333334444444455000000a60000000000007830"), 0x010e41), + (hex!("0100000000333333334444444455000000a70000000000000a80"), 0x010ee1), + (hex!("0100000000333333334444444455000000a700000000000064f0"), 0x010f81), + (hex!("0100000000333333334444444455000000a80000000000000a90"), 0x011021), + (hex!("0100000000333333334444444455000000a90000000000000aa0"), 0x0110c1), + (hex!("0100000000333333334444444455000000a90000000000005e30"), 0x011161), + (hex!("0100000000333333334444444455000000aa0000000000000ab0"), 0x011201), + (hex!("0100000000333333334444444455000000ab0000000000000ac0"), 0x0112a1), + (hex!("0100000000333333334444444455000000ac0000000000000ad0"), 0x011341), + (hex!("0100000000333333334444444455000000ac0000000000006d20"), 0x0113e1), + (hex!("0100000000333333334444444455000000ac0000000000007000"), 0x011481), + (hex!("0100000000333333334444444455000000ad0000000000000ae0"), 0x011521), + (hex!("0100000000333333334444444455000000ae0000000000000af0"), 0x0115c1), + (hex!("0100000000333333334444444455000000ae0000000000004a10"), 0x011661), + (hex!("0100000000333333334444444455000000af0000000000000b00"), 0x011701), + (hex!("0100000000333333334444444455000000af0000000000004e10"), 0x0117a1), + (hex!("0100000000333333334444444455000000b00000000000000b10"), 0x011841), + (hex!("0100000000333333334444444455000000b00000000000004280"), 0x0118e1), + (hex!("0100000000333333334444444455000000b000000000000077e0"), 0x011981), + (hex!("0100000000333333334444444455000000b10000000000000b20"), 0x011a21), + (hex!("0100000000333333334444444455000000b20000000000000b30"), 0x011ac1), + (hex!("0100000000333333334444444455000000b30000000000000b40"), 0x011b61), + (hex!("0100000000333333334444444455000000b30000000000004bc0"), 0x011c01), + (hex!("0100000000333333334444444455000000b40000000000000b50"), 0x011ca1), + (hex!("0100000000333333334444444455000000b50000000000000b60"), 0x011d41), + (hex!("0100000000333333334444444455000000b50000000000004fa0"), 0x011de1), + (hex!("0100000000333333334444444455000000b50000000000006a60"), 0x011e81), + (hex!("0100000000333333334444444455000000b60000000000000b70"), 0x011f21), + (hex!("0100000000333333334444444455000000b60000000000005630"), 0x011fc1), + (hex!("0100000000333333334444444455000000b70000000000000b80"), 0x012061), + (hex!("0100000000333333334444444455000000b80000000000000b90"), 0x012101), + (hex!("0100000000333333334444444455000000b80000000000006f80"), 0x0121a1), + (hex!("0100000000333333334444444455000000b90000000000000ba0"), 0x012241), + (hex!("0100000000333333334444444455000000ba0000000000000bb0"), 0x0122e1), + (hex!("0100000000333333334444444455000000bb0000000000000bc0"), 0x012381), + (hex!("0100000000333333334444444455000000bb00000000000047c0"), 0x012421), + (hex!("0100000000333333334444444455000000bb0000000000006060"), 0x0124c1), + (hex!("0100000000333333334444444455000000bc0000000000000bd0"), 0x012561), + (hex!("0100000000333333334444444455000000bd0000000000000be0"), 0x012601), + (hex!("0100000000333333334444444455000000bd0000000000004e80"), 0x0126a1), + (hex!("0100000000333333334444444455000000be0000000000000bf0"), 0x012741), + (hex!("0100000000333333334444444455000000bf0000000000000c00"), 0x0127e1), + (hex!("0100000000333333334444444455000000bf00000000000047a0"), 0x012881), + (hex!("0100000000333333334444444455000000bf0000000000006da0"), 0x012921), + (hex!("0100000000333333334444444455000000c00000000000000c10"), 0x0129c1), + (hex!("0100000000333333334444444455000000c10000000000000c20"), 0x012a61), + (hex!("0100000000333333334444444455000000c20000000000000c30"), 0x012b01), + (hex!("0100000000333333334444444455000000c20000000000004bd0"), 0x012ba1), + (hex!("0100000000333333334444444455000000c20000000000006ac0"), 0x012c41), + (hex!("0100000000333333334444444455000000c30000000000000c40"), 0x012ce1), + (hex!("0100000000333333334444444455000000c30000000000004660"), 0x012d81), + (hex!("0100000000333333334444444455000000c40000000000000c50"), 0x012e21), + (hex!("0100000000333333334444444455000000c50000000000000c60"), 0x012ec1), + (hex!("0100000000333333334444444455000000c60000000000000c70"), 0x012f61), + (hex!("0100000000333333334444444455000000c60000000000005880"), 0x013001), + (hex!("0100000000333333334444444455000000c60000000000006b70"), 0x0130a1), + (hex!("0100000000333333334444444455000000c70000000000000c80"), 0x013141), + (hex!("0100000000333333334444444455000000c80000000000000c90"), 0x0131e1), + (hex!("0100000000333333334444444455000000c80000000000005310"), 0x013281), + (hex!("0100000000333333334444444455000000c80000000000005db0"), 0x013321), + (hex!("0100000000333333334444444455000000c80000000000007040"), 0x0133c1), + (hex!("0100000000333333334444444455000000c80000000000007290"), 0x013461), + (hex!("0100000000333333334444444455000000c90000000000000ca0"), 0x013501), + (hex!("0100000000333333334444444455000000c90000000000004fe0"), 0x0135a1), + (hex!("0100000000333333334444444455000000ca0000000000000cb0"), 0x013641), + (hex!("0100000000333333334444444455000000ca0000000000006140"), 0x0136e1), + (hex!("0100000000333333334444444455000000ca0000000000007700"), 0x013781), + (hex!("0100000000333333334444444455000000cb0000000000000cc0"), 0x013821), + (hex!("0100000000333333334444444455000000cc0000000000000cd0"), 0x0138c1), + (hex!("0100000000333333334444444455000000cd0000000000000ce0"), 0x013961), + (hex!("0100000000333333334444444455000000cd0000000000003f20"), 0x013a01), + (hex!("0100000000333333334444444455000000cd00000000000040f0"), 0x013aa1), + (hex!("0100000000333333334444444455000000cd0000000000004ec0"), 0x013b41), + (hex!("0100000000333333334444444455000000ce0000000000000cf0"), 0x013be1), + (hex!("0100000000333333334444444455000000ce0000000000007200"), 0x013c81), + (hex!("0100000000333333334444444455000000cf0000000000000d00"), 0x013d21), + (hex!("0100000000333333334444444455000000cf00000000000046a0"), 0x013dc1), + (hex!("0100000000333333334444444455000000cf0000000000005960"), 0x013e61), + (hex!("0100000000333333334444444455000000d00000000000000d10"), 0x013f01), + (hex!("0100000000333333334444444455000000d00000000000005f30"), 0x013fa1), + (hex!("0100000000333333334444444455000000d10000000000000d20"), 0x014041), + (hex!("0100000000333333334444444455000000d10000000000007a00"), 0x0140e1), + (hex!("0100000000333333334444444455000000d20000000000000d30"), 0x014181), + (hex!("0100000000333333334444444455000000d30000000000000d40"), 0x014221), + (hex!("0100000000333333334444444455000000d40000000000000d50"), 0x0142c1), + (hex!("0100000000333333334444444455000000d50000000000000d60"), 0x014361), + (hex!("0100000000333333334444444455000000d50000000000004960"), 0x014401), + (hex!("0100000000333333334444444455000000d500000000000055d0"), 0x0144a1), + (hex!("0100000000333333334444444455000000d500000000000067d0"), 0x014541), + (hex!("0100000000333333334444444455000000d60000000000000d70"), 0x0145e1), + (hex!("0100000000333333334444444455000000d70000000000000d80"), 0x014681), + (hex!("0100000000333333334444444455000000d80000000000000d90"), 0x014721), + (hex!("0100000000333333334444444455000000d800000000000065f0"), 0x0147c1), + (hex!("0100000000333333334444444455000000d90000000000000da0"), 0x014861), + (hex!("0100000000333333334444444455000000d90000000000004980"), 0x014901), + (hex!("0100000000333333334444444455000000da0000000000000db0"), 0x0149a1), + (hex!("0100000000333333334444444455000000da00000000000048c0"), 0x014a41), + (hex!("0100000000333333334444444455000000da00000000000072c0"), 0x014ae1), + (hex!("0100000000333333334444444455000000da00000000000076b0"), 0x014b81), + (hex!("0100000000333333334444444455000000db0000000000000dc0"), 0x014c21), + (hex!("0100000000333333334444444455000000dc0000000000000dd0"), 0x014cc1), + (hex!("0100000000333333334444444455000000dc00000000000040a0"), 0x014d61), + (hex!("0100000000333333334444444455000000dc00000000000074c0"), 0x014e01), + (hex!("0100000000333333334444444455000000dd0000000000000de0"), 0x014ea1), + (hex!("0100000000333333334444444455000000dd0000000000004e50"), 0x014f41), + (hex!("0100000000333333334444444455000000dd0000000000007270"), 0x014fe1), + (hex!("0100000000333333334444444455000000de0000000000000df0"), 0x015081), + (hex!("0100000000333333334444444455000000de00000000000078d0"), 0x015121), + (hex!("0100000000333333334444444455000000df0000000000000e00"), 0x0151c1), + (hex!("0100000000333333334444444455000000df0000000000004d30"), 0x015261), + (hex!("0100000000333333334444444455000000df0000000000006c30"), 0x015301), + (hex!("0100000000333333334444444455000000e00000000000000e10"), 0x0153a1), + (hex!("0100000000333333334444444455000000e00000000000005d30"), 0x015441), + (hex!("0100000000333333334444444455000000e10000000000000e20"), 0x0154e1), + (hex!("0100000000333333334444444455000000e10000000000004610"), 0x015581), + (hex!("0100000000333333334444444455000000e100000000000051d0"), 0x015621), + (hex!("0100000000333333334444444455000000e10000000000005f10"), 0x0156c1), + (hex!("0100000000333333334444444455000000e20000000000000e30"), 0x015761), + (hex!("0100000000333333334444444455000000e20000000000007a90"), 0x015801), + (hex!("0100000000333333334444444455000000e30000000000000e40"), 0x0158a1), + (hex!("0100000000333333334444444455000000e30000000000005ae0"), 0x015941), + (hex!("0100000000333333334444444455000000e40000000000000e50"), 0x0159e1), + (hex!("0100000000333333334444444455000000e50000000000000e60"), 0x015a81), + (hex!("0100000000333333334444444455000000e50000000000004700"), 0x015b21), + (hex!("0100000000333333334444444455000000e500000000000065d0"), 0x015bc1), + (hex!("0100000000333333334444444455000000e60000000000000e70"), 0x015c61), + (hex!("0100000000333333334444444455000000e60000000000004fd0"), 0x015d01), + (hex!("0100000000333333334444444455000000e70000000000000e80"), 0x015da1), + (hex!("0100000000333333334444444455000000e70000000000005150"), 0x015e41), + (hex!("0100000000333333334444444455000000e70000000000005920"), 0x015ee1), + (hex!("0100000000333333334444444455000000e80000000000000e90"), 0x015f81), + (hex!("0100000000333333334444444455000000e80000000000004320"), 0x016021), + (hex!("0100000000333333334444444455000000e80000000000005ec0"), 0x0160c1), + (hex!("0100000000333333334444444455000000e90000000000000ea0"), 0x016161), + (hex!("0100000000333333334444444455000000e900000000000043b0"), 0x016201), + (hex!("0100000000333333334444444455000000ea0000000000000eb0"), 0x0162a1), + (hex!("0100000000333333334444444455000000ea0000000000003ea0"), 0x016341), + (hex!("0100000000333333334444444455000000ea0000000000004f50"), 0x0163e1), + (hex!("0100000000333333334444444455000000ea0000000000007520"), 0x016481), + (hex!("0100000000333333334444444455000000eb0000000000000ec0"), 0x016521), + (hex!("0100000000333333334444444455000000ec0000000000000ed0"), 0x0165c1), + (hex!("0100000000333333334444444455000000ec0000000000006670"), 0x016661), + (hex!("0100000000333333334444444455000000ed0000000000000ee0"), 0x016701), + (hex!("0100000000333333334444444455000000ee0000000000000ef0"), 0x0167a1), + (hex!("0100000000333333334444444455000000ee0000000000004d10"), 0x016841), + (hex!("0100000000333333334444444455000000ef0000000000000f00"), 0x0168e1), + (hex!("0100000000333333334444444455000000f00000000000000f10"), 0x016981), + (hex!("0100000000333333334444444455000000f00000000000007220"), 0x016a21), + (hex!("0100000000333333334444444455000000f00000000000007540"), 0x016ac1), + (hex!("0100000000333333334444444455000000f10000000000000f20"), 0x016b61), + (hex!("0100000000333333334444444455000000f100000000000066f0"), 0x016c01), + (hex!("0100000000333333334444444455000000f20000000000000f30"), 0x016ca1), + (hex!("0100000000333333334444444455000000f20000000000007810"), 0x016d41), + (hex!("0100000000333333334444444455000000f30000000000000f40"), 0x016de1), + (hex!("0100000000333333334444444455000000f30000000000007b70"), 0x016e81), + (hex!("0100000000333333334444444455000000f40000000000000f50"), 0x016f21), + (hex!("0100000000333333334444444455000000f400000000000059c0"), 0x016fc1), + (hex!("0100000000333333334444444455000000f50000000000000f60"), 0x017061), + (hex!("0100000000333333334444444455000000f50000000000003fb0"), 0x017101), + (hex!("0100000000333333334444444455000000f50000000000005740"), 0x0171a1), + (hex!("0100000000333333334444444455000000f500000000000064d0"), 0x017241), + (hex!("0100000000333333334444444455000000f50000000000006960"), 0x0172e1), + (hex!("0100000000333333334444444455000000f60000000000000f70"), 0x017381), + (hex!("0100000000333333334444444455000000f60000000000006d00"), 0x017421), + (hex!("0100000000333333334444444455000000f70000000000000f80"), 0x0174c1), + (hex!("0100000000333333334444444455000000f80000000000000f90"), 0x017561), + (hex!("0100000000333333334444444455000000f90000000000000fa0"), 0x017601), + (hex!("0100000000333333334444444455000000fa0000000000000fb0"), 0x0176a1), + (hex!("0100000000333333334444444455000000fa00000000000067b0"), 0x017741), + (hex!("0100000000333333334444444455000000fb0000000000000fc0"), 0x0177e1), + (hex!("0100000000333333334444444455000000fb0000000000004eb0"), 0x017881), + (hex!("0100000000333333334444444455000000fb0000000000006ef0"), 0x017921), + (hex!("0100000000333333334444444455000000fc0000000000000fd0"), 0x0179c1), + (hex!("0100000000333333334444444455000000fc0000000000004470"), 0x017a61), + (hex!("0100000000333333334444444455000000fc0000000000005940"), 0x017b01), + (hex!("0100000000333333334444444455000000fd0000000000000fe0"), 0x017ba1), + (hex!("0100000000333333334444444455000000fe0000000000000ff0"), 0x017c41), + (hex!("0100000000333333334444444455000000ff0000000000001000"), 0x017ce1), + (hex!("0100000000333333334444444455000000ff0000000000005690"), 0x017d81), + (hex!("0100000000333333334444444455000001000000000000001010"), 0x017e21), + (hex!("0100000000333333334444444455000001000000000000005210"), 0x017ec1), + (hex!("01000000003333333344444444550000010000000000000070a0"), 0x017f61), + (hex!("0100000000333333334444444455000001010000000000001020"), 0x018001), + (hex!("0100000000333333334444444455000001010000000000006b80"), 0x0180a1), + (hex!("0100000000333333334444444455000001020000000000001030"), 0x018141), + (hex!("0100000000333333334444444455000001030000000000001040"), 0x0181e1), + (hex!("0100000000333333334444444455000001030000000000004c80"), 0x018281), + (hex!("0100000000333333334444444455000001040000000000001050"), 0x018321), + (hex!("0100000000333333334444444455000001040000000000004850"), 0x0183c1), + (hex!("01000000003333333344444444550000010400000000000057b0"), 0x018461), + (hex!("0100000000333333334444444455000001050000000000001060"), 0x018501), + (hex!("01000000003333333344444444550000010500000000000048d0"), 0x0185a1), + (hex!("0100000000333333334444444455000001050000000000007870"), 0x018641), + (hex!("0100000000333333334444444455000001060000000000001070"), 0x0186e1), + (hex!("0100000000333333334444444455000001060000000000004f90"), 0x018781), + (hex!("0100000000333333334444444455000001060000000000006270"), 0x018821), + (hex!("0100000000333333334444444455000001070000000000001080"), 0x0188c1), + (hex!("01000000003333333344444444550000010700000000000063b0"), 0x018961), + (hex!("0100000000333333334444444455000001080000000000001090"), 0x018a01), + (hex!("01000000003333333344444444550000010900000000000010a0"), 0x018aa1), + (hex!("0100000000333333334444444455000001090000000000006f40"), 0x018b41), + (hex!("01000000003333333344444444550000010a00000000000010b0"), 0x018be1), + (hex!("01000000003333333344444444550000010a0000000000006640"), 0x018c81), + (hex!("01000000003333333344444444550000010b00000000000010c0"), 0x018d21), + (hex!("01000000003333333344444444550000010c00000000000010d0"), 0x018dc1), + (hex!("01000000003333333344444444550000010d00000000000010e0"), 0x018e61), + (hex!("01000000003333333344444444550000010e00000000000010f0"), 0x018f01), + (hex!("01000000003333333344444444550000010e0000000000005c40"), 0x018fa1), + (hex!("01000000003333333344444444550000010e0000000000007ba0"), 0x019041), + (hex!("01000000003333333344444444550000010f0000000000001100"), 0x0190e1), + (hex!("01000000003333333344444444550000010f0000000000005c30"), 0x019181), + (hex!("0100000000333333334444444455000001100000000000001110"), 0x019221), + (hex!("0100000000333333334444444455000001100000000000007640"), 0x0192c1), + (hex!("0100000000333333334444444455000001110000000000001120"), 0x019361), + (hex!("01000000003333333344444444550000011100000000000052c0"), 0x019401), + (hex!("0100000000333333334444444455000001110000000000005710"), 0x0194a1), + (hex!("0100000000333333334444444455000001110000000000006a00"), 0x019541), + (hex!("0100000000333333334444444455000001120000000000001130"), 0x0195e1), + (hex!("0100000000333333334444444455000001130000000000001140"), 0x019681), + (hex!("0100000000333333334444444455000001140000000000001150"), 0x019721), + (hex!("0100000000333333334444444455000001140000000000003fa0"), 0x0197c1), + (hex!("01000000003333333344444444550000011400000000000054b0"), 0x019861), + (hex!("0100000000333333334444444455000001140000000000006070"), 0x019901), + (hex!("0100000000333333334444444455000001150000000000001160"), 0x0199a1), + (hex!("0100000000333333334444444455000001150000000000005320"), 0x019a41), + (hex!("0100000000333333334444444455000001150000000000006600"), 0x019ae1), + (hex!("0100000000333333334444444455000001150000000000006df0"), 0x019b81), + (hex!("01000000003333333344444444550000011500000000000079c0"), 0x019c21), + (hex!("0100000000333333334444444455000001160000000000001170"), 0x019cc1), + (hex!("0100000000333333334444444455000001170000000000001180"), 0x019d61), + (hex!("0100000000333333334444444455000001170000000000004a60"), 0x019e01), + (hex!("01000000003333333344444444550000011700000000000063c0"), 0x019ea1), + (hex!("0100000000333333334444444455000001180000000000001190"), 0x019f41), + (hex!("0100000000333333334444444455000001180000000000004530"), 0x019fe1), + (hex!("01000000003333333344444444550000011800000000000077c0"), 0x01a081), + (hex!("01000000003333333344444444550000011900000000000011a0"), 0x01a121), + (hex!("01000000003333333344444444550000011a00000000000011b0"), 0x01a1c1), + (hex!("01000000003333333344444444550000011a00000000000041c0"), 0x01a261), + (hex!("01000000003333333344444444550000011a00000000000061e0"), 0x01a301), + (hex!("01000000003333333344444444550000011b00000000000011c0"), 0x01a3a1), + (hex!("01000000003333333344444444550000011c00000000000011d0"), 0x01a441), + (hex!("01000000003333333344444444550000011c0000000000005f90"), 0x01a4e1), + (hex!("01000000003333333344444444550000011d00000000000011e0"), 0x01a581), + (hex!("01000000003333333344444444550000011d0000000000004160"), 0x01a621), + (hex!("01000000003333333344444444550000011e00000000000011f0"), 0x01a6c1), + (hex!("01000000003333333344444444550000011e00000000000056d0"), 0x01a761), + (hex!("01000000003333333344444444550000011f0000000000001200"), 0x01a801), + (hex!("01000000003333333344444444550000011f0000000000004510"), 0x01a8a1), + (hex!("0100000000333333334444444455000001200000000000001210"), 0x01a941), + (hex!("0100000000333333334444444455000001210000000000001220"), 0x01a9e1), + (hex!("0100000000333333334444444455000001210000000000005140"), 0x01aa81), + (hex!("0100000000333333334444444455000001210000000000006710"), 0x01ab21), + (hex!("0100000000333333334444444455000001210000000000006f50"), 0x01abc1), + (hex!("0100000000333333334444444455000001220000000000001230"), 0x01ac61), + (hex!("0100000000333333334444444455000001220000000000005570"), 0x01ad01), + (hex!("0100000000333333334444444455000001220000000000007ac0"), 0x01ada1), + (hex!("0100000000333333334444444455000001230000000000001240"), 0x01ae41), + (hex!("0100000000333333334444444455000001240000000000001250"), 0x01aee1), + (hex!("0100000000333333334444444455000001240000000000006cd0"), 0x01af81), + (hex!("0100000000333333334444444455000001250000000000001260"), 0x01b021), + (hex!("01000000003333333344444444550000012500000000000046b0"), 0x01b0c1), + (hex!("0100000000333333334444444455000001250000000000005eb0"), 0x01b161), + (hex!("0100000000333333334444444455000001260000000000001270"), 0x01b201), + (hex!("0100000000333333334444444455000001260000000000004630"), 0x01b2a1), + (hex!("0100000000333333334444444455000001270000000000001280"), 0x01b341), + (hex!("0100000000333333334444444455000001270000000000004ff0"), 0x01b3e1), + (hex!("0100000000333333334444444455000001270000000000006ec0"), 0x01b481), + (hex!("0100000000333333334444444455000001280000000000001290"), 0x01b521), + (hex!("01000000003333333344444444550000012900000000000012a0"), 0x01b5c1), + (hex!("0100000000333333334444444455000001290000000000005f60"), 0x01b661), + (hex!("01000000003333333344444444550000012a00000000000012b0"), 0x01b701), + (hex!("01000000003333333344444444550000012a0000000000005480"), 0x01b7a1), + (hex!("01000000003333333344444444550000012b00000000000012c0"), 0x01b841), + (hex!("01000000003333333344444444550000012b00000000000065a0"), 0x01b8e1), + (hex!("01000000003333333344444444550000012b00000000000066c0"), 0x01b981), + (hex!("01000000003333333344444444550000012c00000000000012d0"), 0x01ba21), + (hex!("01000000003333333344444444550000012c00000000000064b0"), 0x01bac1), + (hex!("01000000003333333344444444550000012d00000000000012e0"), 0x01bb61), + (hex!("01000000003333333344444444550000012d00000000000049c0"), 0x01bc01), + (hex!("01000000003333333344444444550000012d0000000000004bf0"), 0x01bca1), + (hex!("01000000003333333344444444550000012e00000000000012f0"), 0x01bd41), + (hex!("01000000003333333344444444550000012e0000000000005ed0"), 0x01bde1), + (hex!("01000000003333333344444444550000012f0000000000001300"), 0x01be81), + (hex!("01000000003333333344444444550000012f00000000000049a0"), 0x01bf21), + (hex!("0100000000333333334444444455000001300000000000001310"), 0x01bfc1), + (hex!("0100000000333333334444444455000001300000000000007840"), 0x01c061), + (hex!("0100000000333333334444444455000001310000000000001320"), 0x01c101), + (hex!("0100000000333333334444444455000001310000000000005f70"), 0x01c1a1), + (hex!("0100000000333333334444444455000001320000000000001330"), 0x01c241), + (hex!("0100000000333333334444444455000001320000000000005a00"), 0x01c2e1), + (hex!("0100000000333333334444444455000001330000000000001340"), 0x01c381), + (hex!("0100000000333333334444444455000001330000000000006c70"), 0x01c421), + (hex!("0100000000333333334444444455000001340000000000001350"), 0x01c4c1), + (hex!("0100000000333333334444444455000001340000000000005c60"), 0x01c561), + (hex!("0100000000333333334444444455000001350000000000001360"), 0x01c601), + (hex!("0100000000333333334444444455000001350000000000004f10"), 0x01c6a1), + (hex!("0100000000333333334444444455000001360000000000001370"), 0x01c741), + (hex!("0100000000333333334444444455000001360000000000004c60"), 0x01c7e1), + (hex!("0100000000333333334444444455000001370000000000001380"), 0x01c881), + (hex!("0100000000333333334444444455000001380000000000001390"), 0x01c921), + (hex!("01000000003333333344444444550000013900000000000013a0"), 0x01c9c1), + (hex!("0100000000333333334444444455000001390000000000004ea0"), 0x01ca61), + (hex!("01000000003333333344444444550000013a00000000000013b0"), 0x01cb01), + (hex!("01000000003333333344444444550000013a0000000000007350"), 0x01cba1), + (hex!("01000000003333333344444444550000013b00000000000013c0"), 0x01cc41), + (hex!("01000000003333333344444444550000013c00000000000013d0"), 0x01cce1), + (hex!("01000000003333333344444444550000013c0000000000007050"), 0x01cd81), + (hex!("01000000003333333344444444550000013d00000000000013e0"), 0x01ce21), + (hex!("01000000003333333344444444550000013d0000000000006bd0"), 0x01cec1), + (hex!("01000000003333333344444444550000013e00000000000013f0"), 0x01cf61), + (hex!("01000000003333333344444444550000013e00000000000058e0"), 0x01d001), + (hex!("01000000003333333344444444550000013f0000000000001400"), 0x01d0a1), + (hex!("01000000003333333344444444550000013f0000000000004740"), 0x01d141), + (hex!("0100000000333333334444444455000001400000000000001410"), 0x01d1e1), + (hex!("0100000000333333334444444455000001400000000000003f10"), 0x01d281), + (hex!("0100000000333333334444444455000001400000000000006d40"), 0x01d321), + (hex!("01000000003333333344444444550000014000000000000072d0"), 0x01d3c1), + (hex!("0100000000333333334444444455000001410000000000001420"), 0x01d461), + (hex!("0100000000333333334444444455000001420000000000001430"), 0x01d501), + (hex!("0100000000333333334444444455000001430000000000001440"), 0x01d5a1), + (hex!("0100000000333333334444444455000001440000000000001450"), 0x01d641), + (hex!("0100000000333333334444444455000001450000000000001460"), 0x01d6e1), + (hex!("0100000000333333334444444455000001460000000000001470"), 0x01d781), + (hex!("01000000003333333344444444550000014600000000000055c0"), 0x01d821), + (hex!("0100000000333333334444444455000001470000000000001480"), 0x01d8c1), + (hex!("0100000000333333334444444455000001470000000000004570"), 0x01d961), + (hex!("0100000000333333334444444455000001470000000000004be0"), 0x01da01), + (hex!("0100000000333333334444444455000001480000000000001490"), 0x01daa1), + (hex!("0100000000333333334444444455000001480000000000005360"), 0x01db41), + (hex!("01000000003333333344444444550000014900000000000014a0"), 0x01dbe1), + (hex!("01000000003333333344444444550000014a00000000000014b0"), 0x01dc81), + (hex!("01000000003333333344444444550000014a00000000000053d0"), 0x01dd21), + (hex!("01000000003333333344444444550000014b00000000000014c0"), 0x01ddc1), + (hex!("01000000003333333344444444550000014b0000000000005950"), 0x01de61), + (hex!("01000000003333333344444444550000014c00000000000014d0"), 0x01df01), + (hex!("01000000003333333344444444550000014c0000000000004f60"), 0x01dfa1), + (hex!("01000000003333333344444444550000014d00000000000014e0"), 0x01e041), + (hex!("01000000003333333344444444550000014d0000000000004520"), 0x01e0e1), + (hex!("01000000003333333344444444550000014d0000000000005200"), 0x01e181), + (hex!("01000000003333333344444444550000014e00000000000014f0"), 0x01e221), + (hex!("01000000003333333344444444550000014e0000000000005bd0"), 0x01e2c1), + (hex!("01000000003333333344444444550000014f0000000000001500"), 0x01e361), + (hex!("01000000003333333344444444550000014f00000000000060d0"), 0x01e401), + (hex!("0100000000333333334444444455000001500000000000001510"), 0x01e4a1), + (hex!("01000000003333333344444444550000015000000000000075e0"), 0x01e541), + (hex!("0100000000333333334444444455000001510000000000001520"), 0x01e5e1), + (hex!("0100000000333333334444444455000001510000000000005c00"), 0x01e681), + (hex!("0100000000333333334444444455000001510000000000006af0"), 0x01e721), + (hex!("0100000000333333334444444455000001510000000000007b80"), 0x01e7c1), + (hex!("0100000000333333334444444455000001520000000000001530"), 0x01e861), + (hex!("0100000000333333334444444455000001520000000000004c70"), 0x01e901), + (hex!("0100000000333333334444444455000001530000000000001540"), 0x01e9a1), + (hex!("0100000000333333334444444455000001540000000000001550"), 0x01ea41), + (hex!("0100000000333333334444444455000001540000000000007cd0"), 0x01eae1), + (hex!("0100000000333333334444444455000001550000000000001560"), 0x01eb81), + (hex!("0100000000333333334444444455000001550000000000004ae0"), 0x01ec21), + (hex!("01000000003333333344444444550000015500000000000068c0"), 0x01ecc1), + (hex!("0100000000333333334444444455000001560000000000001570"), 0x01ed61), + (hex!("01000000003333333344444444550000015600000000000064a0"), 0x01ee01), + (hex!("0100000000333333334444444455000001570000000000001580"), 0x01eea1), + (hex!("0100000000333333334444444455000001580000000000001590"), 0x01ef41), + (hex!("0100000000333333334444444455000001580000000000006d30"), 0x01efe1), + (hex!("01000000003333333344444444550000015800000000000074f0"), 0x01f081), + (hex!("01000000003333333344444444550000015900000000000015a0"), 0x01f121), + (hex!("01000000003333333344444444550000015900000000000053a0"), 0x01f1c1), + (hex!("01000000003333333344444444550000015900000000000055e0"), 0x01f261), + (hex!("0100000000333333334444444455000001590000000000006210"), 0x01f301), + (hex!("01000000003333333344444444550000015900000000000067c0"), 0x01f3a1), + (hex!("01000000003333333344444444550000015a00000000000015b0"), 0x01f441), + (hex!("01000000003333333344444444550000015b00000000000015c0"), 0x01f4e1), + (hex!("01000000003333333344444444550000015c00000000000015d0"), 0x01f581), + (hex!("01000000003333333344444444550000015c0000000000004d80"), 0x01f621), + (hex!("01000000003333333344444444550000015c00000000000073f0"), 0x01f6c1), + (hex!("01000000003333333344444444550000015d00000000000015e0"), 0x01f761), + (hex!("01000000003333333344444444550000015e00000000000015f0"), 0x01f801), + (hex!("01000000003333333344444444550000015e0000000000004120"), 0x01f8a1), + (hex!("01000000003333333344444444550000015e0000000000004350"), 0x01f941), + (hex!("01000000003333333344444444550000015e0000000000007c50"), 0x01f9e1), + (hex!("01000000003333333344444444550000015f0000000000001600"), 0x01fa81), + (hex!("0100000000333333334444444455000001600000000000001610"), 0x01fb21), + (hex!("0100000000333333334444444455000001600000000000004840"), 0x01fbc1), + (hex!("0100000000333333334444444455000001600000000000004b10"), 0x01fc61), + (hex!("0100000000333333334444444455000001600000000000007060"), 0x01fd01), + (hex!("0100000000333333334444444455000001610000000000001620"), 0x01fda1), + (hex!("0100000000333333334444444455000001610000000000005300"), 0x01fe41), + (hex!("0100000000333333334444444455000001620000000000001630"), 0x01fee1), + (hex!("0100000000333333334444444455000001620000000000006530"), 0x01ff81), + (hex!("0100000000333333334444444455000001630000000000001640"), 0x020021), + (hex!("0100000000333333334444444455000001640000000000001650"), 0x0200c1), + (hex!("0100000000333333334444444455000001650000000000001660"), 0x020161), + (hex!("0100000000333333334444444455000001660000000000001670"), 0x020201), + (hex!("0100000000333333334444444455000001670000000000001680"), 0x0202a1), + (hex!("0100000000333333334444444455000001670000000000007310"), 0x020341), + (hex!("0100000000333333334444444455000001680000000000001690"), 0x0203e1), + (hex!("0100000000333333334444444455000001680000000000007b50"), 0x020481), + (hex!("01000000003333333344444444550000016900000000000016a0"), 0x020521), + (hex!("01000000003333333344444444550000016900000000000049d0"), 0x0205c1), + (hex!("01000000003333333344444444550000016a00000000000016b0"), 0x020661), + (hex!("01000000003333333344444444550000016a00000000000078b0"), 0x020701), + (hex!("01000000003333333344444444550000016b00000000000016c0"), 0x0207a1), + (hex!("01000000003333333344444444550000016b0000000000004100"), 0x020841), + (hex!("01000000003333333344444444550000016c00000000000016d0"), 0x0208e1), + (hex!("01000000003333333344444444550000016c0000000000006e00"), 0x020981), + (hex!("01000000003333333344444444550000016d00000000000016e0"), 0x020a21), + (hex!("01000000003333333344444444550000016e00000000000016f0"), 0x020ac1), + (hex!("01000000003333333344444444550000016e0000000000004ac0"), 0x020b61), + (hex!("01000000003333333344444444550000016e0000000000007820"), 0x020c01), + (hex!("01000000003333333344444444550000016f0000000000001700"), 0x020ca1), + (hex!("0100000000333333334444444455000001700000000000001710"), 0x020d41), + (hex!("0100000000333333334444444455000001700000000000005830"), 0x020de1), + (hex!("0100000000333333334444444455000001710000000000001720"), 0x020e81), + (hex!("01000000003333333344444444550000017100000000000072f0"), 0x020f21), + (hex!("0100000000333333334444444455000001720000000000001730"), 0x020fc1), + (hex!("0100000000333333334444444455000001720000000000004870"), 0x021061), + (hex!("01000000003333333344444444550000017200000000000070b0"), 0x021101), + (hex!("0100000000333333334444444455000001730000000000001740"), 0x0211a1), + (hex!("0100000000333333334444444455000001740000000000001750"), 0x021241), + (hex!("0100000000333333334444444455000001750000000000001760"), 0x0212e1), + (hex!("0100000000333333334444444455000001750000000000005670"), 0x021381), + (hex!("0100000000333333334444444455000001750000000000005870"), 0x021421), + (hex!("0100000000333333334444444455000001760000000000001770"), 0x0214c1), + (hex!("0100000000333333334444444455000001770000000000001780"), 0x021561), + (hex!("0100000000333333334444444455000001770000000000005000"), 0x021601), + (hex!("0100000000333333334444444455000001770000000000007090"), 0x0216a1), + (hex!("0100000000333333334444444455000001780000000000001790"), 0x021741), + (hex!("01000000003333333344444444550000017800000000000048a0"), 0x0217e1), + (hex!("0100000000333333334444444455000001780000000000006bf0"), 0x021881), + (hex!("01000000003333333344444444550000017900000000000017a0"), 0x021921), + (hex!("01000000003333333344444444550000017900000000000057d0"), 0x0219c1), + (hex!("0100000000333333334444444455000001790000000000006660"), 0x021a61), + (hex!("01000000003333333344444444550000017a00000000000017b0"), 0x021b01), + (hex!("01000000003333333344444444550000017a0000000000004970"), 0x021ba1), + (hex!("01000000003333333344444444550000017a0000000000005dc0"), 0x021c41), + (hex!("01000000003333333344444444550000017b00000000000017c0"), 0x021ce1), + (hex!("01000000003333333344444444550000017b0000000000004ee0"), 0x021d81), + (hex!("01000000003333333344444444550000017b00000000000054c0"), 0x021e21), + (hex!("01000000003333333344444444550000017c00000000000017d0"), 0x021ec1), + (hex!("01000000003333333344444444550000017c0000000000003fc0"), 0x021f61), + (hex!("01000000003333333344444444550000017c00000000000063e0"), 0x022001), + (hex!("01000000003333333344444444550000017c0000000000006520"), 0x0220a1), + (hex!("01000000003333333344444444550000017d00000000000017e0"), 0x022141), + (hex!("01000000003333333344444444550000017d0000000000006220"), 0x0221e1), + (hex!("01000000003333333344444444550000017d0000000000007120"), 0x022281), + (hex!("01000000003333333344444444550000017e00000000000017f0"), 0x022321), + (hex!("01000000003333333344444444550000017f0000000000001800"), 0x0223c1), + (hex!("0100000000333333334444444455000001800000000000001810"), 0x022461), + (hex!("0100000000333333334444444455000001810000000000001820"), 0x022501), + (hex!("01000000003333333344444444550000018100000000000041f0"), 0x0225a1), + (hex!("0100000000333333334444444455000001810000000000007590"), 0x022641), + (hex!("0100000000333333334444444455000001820000000000001830"), 0x0226e1), + (hex!("0100000000333333334444444455000001820000000000004ce0"), 0x022781), + (hex!("0100000000333333334444444455000001830000000000001840"), 0x022821), + (hex!("01000000003333333344444444550000018300000000000042c0"), 0x0228c1), + (hex!("0100000000333333334444444455000001840000000000001850"), 0x022961), + (hex!("0100000000333333334444444455000001840000000000004f70"), 0x022a01), + (hex!("0100000000333333334444444455000001850000000000001860"), 0x022aa1), + (hex!("0100000000333333334444444455000001850000000000006470"), 0x022b41), + (hex!("0100000000333333334444444455000001850000000000007500"), 0x022be1), + (hex!("0100000000333333334444444455000001860000000000001870"), 0x022c81), + (hex!("0100000000333333334444444455000001860000000000004770"), 0x022d21), + (hex!("0100000000333333334444444455000001870000000000001880"), 0x022dc1), + (hex!("0100000000333333334444444455000001870000000000006a30"), 0x022e61), + (hex!("0100000000333333334444444455000001880000000000001890"), 0x022f01), + (hex!("0100000000333333334444444455000001880000000000007410"), 0x022fa1), + (hex!("01000000003333333344444444550000018900000000000018a0"), 0x023041), + (hex!("01000000003333333344444444550000018900000000000044d0"), 0x0230e1), + (hex!("0100000000333333334444444455000001890000000000005ac0"), 0x023181), + (hex!("01000000003333333344444444550000018a00000000000018b0"), 0x023221), + (hex!("01000000003333333344444444550000018a0000000000006260"), 0x0232c1), + (hex!("01000000003333333344444444550000018a0000000000006d70"), 0x023361), + (hex!("01000000003333333344444444550000018b00000000000018c0"), 0x023401), + (hex!("01000000003333333344444444550000018b0000000000004aa0"), 0x0234a1), + (hex!("01000000003333333344444444550000018b0000000000006fd0"), 0x023541), + (hex!("01000000003333333344444444550000018c00000000000018d0"), 0x0235e1), + (hex!("01000000003333333344444444550000018c00000000000051b0"), 0x023681), + (hex!("01000000003333333344444444550000018c0000000000006650"), 0x023721), + (hex!("01000000003333333344444444550000018d00000000000018e0"), 0x0237c1), + (hex!("01000000003333333344444444550000018e00000000000018f0"), 0x023861), + (hex!("01000000003333333344444444550000018e00000000000041d0"), 0x023901), + (hex!("01000000003333333344444444550000018f0000000000001900"), 0x0239a1), + (hex!("01000000003333333344444444550000018f0000000000007600"), 0x023a41), + (hex!("0100000000333333334444444455000001900000000000001910"), 0x023ae1), + (hex!("0100000000333333334444444455000001900000000000005410"), 0x023b81), + (hex!("0100000000333333334444444455000001900000000000006760"), 0x023c21), + (hex!("0100000000333333334444444455000001910000000000001920"), 0x023cc1), + (hex!("0100000000333333334444444455000001920000000000001930"), 0x023d61), + (hex!("0100000000333333334444444455000001920000000000004ca0"), 0x023e01), + (hex!("0100000000333333334444444455000001920000000000005d80"), 0x023ea1), + (hex!("0100000000333333334444444455000001920000000000005fd0"), 0x023f41), + (hex!("01000000003333333344444444550000019200000000000070d0"), 0x023fe1), + (hex!("0100000000333333334444444455000001930000000000001940"), 0x024081), + (hex!("0100000000333333334444444455000001930000000000004010"), 0x024121), + (hex!("0100000000333333334444444455000001930000000000007ca0"), 0x0241c1), + (hex!("0100000000333333334444444455000001940000000000001950"), 0x024261), + (hex!("0100000000333333334444444455000001950000000000001960"), 0x024301), + (hex!("0100000000333333334444444455000001950000000000005380"), 0x0243a1), + (hex!("0100000000333333334444444455000001960000000000001970"), 0x024441), + (hex!("0100000000333333334444444455000001960000000000006de0"), 0x0244e1), + (hex!("0100000000333333334444444455000001970000000000001980"), 0x024581), + (hex!("01000000003333333344444444550000019700000000000048f0"), 0x024621), + (hex!("0100000000333333334444444455000001980000000000001990"), 0x0246c1), + (hex!("0100000000333333334444444455000001980000000000006510"), 0x024761), + (hex!("01000000003333333344444444550000019900000000000019a0"), 0x024801), + (hex!("0100000000333333334444444455000001990000000000007570"), 0x0248a1), + (hex!("0100000000333333334444444455000001990000000000007580"), 0x024941), + (hex!("01000000003333333344444444550000019a00000000000019b0"), 0x0249e1), + (hex!("01000000003333333344444444550000019a0000000000004050"), 0x024a81), + (hex!("01000000003333333344444444550000019a0000000000004ba0"), 0x024b21), + (hex!("01000000003333333344444444550000019a0000000000005540"), 0x024bc1), + (hex!("01000000003333333344444444550000019a00000000000061c0"), 0x024c61), + (hex!("01000000003333333344444444550000019a0000000000007c60"), 0x024d01), + (hex!("01000000003333333344444444550000019b00000000000019c0"), 0x024da1), + (hex!("01000000003333333344444444550000019b0000000000006240"), 0x024e41), + (hex!("01000000003333333344444444550000019c00000000000019d0"), 0x024ee1), + (hex!("01000000003333333344444444550000019d00000000000019e0"), 0x024f81), + (hex!("01000000003333333344444444550000019d0000000000004640"), 0x025021), + (hex!("01000000003333333344444444550000019d00000000000052a0"), 0x0250c1), + (hex!("01000000003333333344444444550000019d00000000000052b0"), 0x025161), + (hex!("01000000003333333344444444550000019e00000000000019f0"), 0x025201), + (hex!("01000000003333333344444444550000019f0000000000001a00"), 0x0252a1), + (hex!("01000000003333333344444444550000019f0000000000006b20"), 0x025341), + (hex!("0100000000333333334444444455000001a00000000000001a10"), 0x0253e1), + (hex!("0100000000333333334444444455000001a10000000000001a20"), 0x025481), + (hex!("0100000000333333334444444455000001a10000000000005460"), 0x025521), + (hex!("0100000000333333334444444455000001a10000000000005d20"), 0x0255c1), + (hex!("0100000000333333334444444455000001a100000000000068f0"), 0x025661), + (hex!("0100000000333333334444444455000001a20000000000001a30"), 0x025701), + (hex!("0100000000333333334444444455000001a20000000000007170"), 0x0257a1), + (hex!("0100000000333333334444444455000001a30000000000001a40"), 0x025841), + (hex!("0100000000333333334444444455000001a40000000000001a50"), 0x0258e1), + (hex!("0100000000333333334444444455000001a50000000000001a60"), 0x025981), + (hex!("0100000000333333334444444455000001a60000000000001a70"), 0x025a21), + (hex!("0100000000333333334444444455000001a70000000000001a80"), 0x025ac1), + (hex!("0100000000333333334444444455000001a70000000000005a90"), 0x025b61), + (hex!("0100000000333333334444444455000001a70000000000006440"), 0x025c01), + (hex!("0100000000333333334444444455000001a80000000000001a90"), 0x025ca1), + (hex!("0100000000333333334444444455000001a80000000000004800"), 0x025d41), + (hex!("0100000000333333334444444455000001a90000000000001aa0"), 0x025de1), + (hex!("0100000000333333334444444455000001aa0000000000001ab0"), 0x025e81), + (hex!("0100000000333333334444444455000001aa0000000000005b60"), 0x025f21), + (hex!("0100000000333333334444444455000001ab0000000000001ac0"), 0x025fc1), + (hex!("0100000000333333334444444455000001ab0000000000006700"), 0x026061), + (hex!("0100000000333333334444444455000001ab00000000000071d0"), 0x026101), + (hex!("0100000000333333334444444455000001ac0000000000001ad0"), 0x0261a1), + (hex!("0100000000333333334444444455000001ac0000000000007380"), 0x026241), + (hex!("0100000000333333334444444455000001ad0000000000001ae0"), 0x0262e1), + (hex!("0100000000333333334444444455000001ad0000000000006350"), 0x026381), + (hex!("0100000000333333334444444455000001ae0000000000001af0"), 0x026421), + (hex!("0100000000333333334444444455000001af0000000000001b00"), 0x0264c1), + (hex!("0100000000333333334444444455000001af0000000000007390"), 0x026561), + (hex!("0100000000333333334444444455000001b00000000000001b10"), 0x026601), + (hex!("0100000000333333334444444455000001b10000000000001b20"), 0x0266a1), + (hex!("0100000000333333334444444455000001b10000000000005cc0"), 0x026741), + (hex!("0100000000333333334444444455000001b20000000000001b30"), 0x0267e1), + (hex!("0100000000333333334444444455000001b20000000000004fb0"), 0x026881), + (hex!("0100000000333333334444444455000001b30000000000001b40"), 0x026921), + (hex!("0100000000333333334444444455000001b40000000000001b50"), 0x0269c1), + (hex!("0100000000333333334444444455000001b50000000000001b60"), 0x026a61), + (hex!("0100000000333333334444444455000001b60000000000001b70"), 0x026b01), + (hex!("0100000000333333334444444455000001b600000000000048e0"), 0x026ba1), + (hex!("0100000000333333334444444455000001b70000000000001b80"), 0x026c41), + (hex!("0100000000333333334444444455000001b70000000000005ca0"), 0x026ce1), + (hex!("0100000000333333334444444455000001b70000000000007900"), 0x026d81), + (hex!("0100000000333333334444444455000001b80000000000001b90"), 0x026e21), + (hex!("0100000000333333334444444455000001b80000000000004d90"), 0x026ec1), + (hex!("0100000000333333334444444455000001b90000000000001ba0"), 0x026f61), + (hex!("0100000000333333334444444455000001b90000000000003f40"), 0x027001), + (hex!("0100000000333333334444444455000001ba0000000000001bb0"), 0x0270a1), + (hex!("0100000000333333334444444455000001ba00000000000042a0"), 0x027141), + (hex!("0100000000333333334444444455000001ba00000000000067f0"), 0x0271e1), + (hex!("0100000000333333334444444455000001ba00000000000073a0"), 0x027281), + (hex!("0100000000333333334444444455000001bb0000000000001bc0"), 0x027321), + (hex!("0100000000333333334444444455000001bb0000000000004a00"), 0x0273c1), + (hex!("0100000000333333334444444455000001bb0000000000005e00"), 0x027461), + (hex!("0100000000333333334444444455000001bc0000000000001bd0"), 0x027501), + (hex!("0100000000333333334444444455000001bc0000000000004230"), 0x0275a1), + (hex!("0100000000333333334444444455000001bc0000000000005860"), 0x027641), + (hex!("0100000000333333334444444455000001bd0000000000001be0"), 0x0276e1), + (hex!("0100000000333333334444444455000001bd0000000000007c70"), 0x027781), + (hex!("0100000000333333334444444455000001be0000000000001bf0"), 0x027821), + (hex!("0100000000333333334444444455000001be0000000000007770"), 0x0278c1), + (hex!("0100000000333333334444444455000001be0000000000007cf0"), 0x027961), + (hex!("0100000000333333334444444455000001bf0000000000001c00"), 0x027a01), + (hex!("0100000000333333334444444455000001bf0000000000006490"), 0x027aa1), + (hex!("0100000000333333334444444455000001c00000000000001c10"), 0x027b41), + (hex!("0100000000333333334444444455000001c10000000000001c20"), 0x027be1), + (hex!("0100000000333333334444444455000001c10000000000004600"), 0x027c81), + (hex!("0100000000333333334444444455000001c20000000000001c30"), 0x027d21), + (hex!("0100000000333333334444444455000001c20000000000006e30"), 0x027dc1), + (hex!("0100000000333333334444444455000001c30000000000001c40"), 0x027e61), + (hex!("0100000000333333334444444455000001c40000000000001c50"), 0x027f01), + (hex!("0100000000333333334444444455000001c50000000000001c60"), 0x027fa1), + (hex!("0100000000333333334444444455000001c60000000000001c70"), 0x028041), + (hex!("0100000000333333334444444455000001c60000000000004240"), 0x0280e1), + (hex!("0100000000333333334444444455000001c60000000000005bb0"), 0x028181), + (hex!("0100000000333333334444444455000001c70000000000001c80"), 0x028221), + (hex!("0100000000333333334444444455000001c80000000000001c90"), 0x0282c1), + (hex!("0100000000333333334444444455000001c90000000000001ca0"), 0x028361), + (hex!("0100000000333333334444444455000001c90000000000006730"), 0x028401), + (hex!("0100000000333333334444444455000001ca0000000000001cb0"), 0x0284a1), + (hex!("0100000000333333334444444455000001ca00000000000070f0"), 0x028541), + (hex!("0100000000333333334444444455000001cb0000000000001cc0"), 0x0285e1), + (hex!("0100000000333333334444444455000001cb00000000000071a0"), 0x028681), + (hex!("0100000000333333334444444455000001cc0000000000001cd0"), 0x028721), + (hex!("0100000000333333334444444455000001cc0000000000005280"), 0x0287c1), + (hex!("0100000000333333334444444455000001cc0000000000005d90"), 0x028861), + (hex!("0100000000333333334444444455000001cd0000000000001ce0"), 0x028901), + (hex!("0100000000333333334444444455000001cd00000000000069b0"), 0x0289a1), + (hex!("0100000000333333334444444455000001ce0000000000001cf0"), 0x028a41), + (hex!("0100000000333333334444444455000001ce0000000000004540"), 0x028ae1), + (hex!("0100000000333333334444444455000001cf0000000000001d00"), 0x028b81), + (hex!("0100000000333333334444444455000001cf00000000000076a0"), 0x028c21), + (hex!("0100000000333333334444444455000001d00000000000001d10"), 0x028cc1), + (hex!("0100000000333333334444444455000001d000000000000060a0"), 0x028d61), + (hex!("0100000000333333334444444455000001d10000000000001d20"), 0x028e01), + (hex!("0100000000333333334444444455000001d20000000000001d30"), 0x028ea1), + (hex!("0100000000333333334444444455000001d30000000000001d40"), 0x028f41), + (hex!("0100000000333333334444444455000001d30000000000004000"), 0x028fe1), + (hex!("0100000000333333334444444455000001d30000000000004140"), 0x029081), + (hex!("0100000000333333334444444455000001d30000000000006790"), 0x029121), + (hex!("0100000000333333334444444455000001d40000000000001d50"), 0x0291c1), + (hex!("0100000000333333334444444455000001d50000000000001d60"), 0x029261), + (hex!("0100000000333333334444444455000001d60000000000001d70"), 0x029301), + (hex!("0100000000333333334444444455000001d60000000000004b50"), 0x0293a1), + (hex!("0100000000333333334444444455000001d60000000000007430"), 0x029441), + (hex!("0100000000333333334444444455000001d70000000000001d80"), 0x0294e1), + (hex!("0100000000333333334444444455000001d70000000000006920"), 0x029581), + (hex!("0100000000333333334444444455000001d80000000000001d90"), 0x029621), + (hex!("0100000000333333334444444455000001d80000000000005b30"), 0x0296c1), + (hex!("0100000000333333334444444455000001d90000000000001da0"), 0x029761), + (hex!("0100000000333333334444444455000001da0000000000001db0"), 0x029801), + (hex!("0100000000333333334444444455000001da0000000000004af0"), 0x0298a1), + (hex!("0100000000333333334444444455000001da0000000000007240"), 0x029941), + (hex!("0100000000333333334444444455000001da0000000000007470"), 0x0299e1), + (hex!("0100000000333333334444444455000001db0000000000001dc0"), 0x029a81), + (hex!("0100000000333333334444444455000001db00000000000045d0"), 0x029b21), + (hex!("0100000000333333334444444455000001dc0000000000001dd0"), 0x029bc1), + (hex!("0100000000333333334444444455000001dd0000000000001de0"), 0x029c61), + (hex!("0100000000333333334444444455000001dd0000000000004bb0"), 0x029d01), + (hex!("0100000000333333334444444455000001dd0000000000004cd0"), 0x029da1), + (hex!("0100000000333333334444444455000001dd0000000000006100"), 0x029e41), + (hex!("0100000000333333334444444455000001dd0000000000007bb0"), 0x029ee1), + (hex!("0100000000333333334444444455000001de0000000000001df0"), 0x029f81), + (hex!("0100000000333333334444444455000001de0000000000004260"), 0x02a021), + (hex!("0100000000333333334444444455000001de0000000000006040"), 0x02a0c1), + (hex!("0100000000333333334444444455000001df0000000000001e00"), 0x02a161), + (hex!("0100000000333333334444444455000001df0000000000005fa0"), 0x02a201), + (hex!("0100000000333333334444444455000001df0000000000006a70"), 0x02a2a1), + (hex!("0100000000333333334444444455000001df0000000000006dc0"), 0x02a341), + (hex!("0100000000333333334444444455000001e00000000000001e10"), 0x02a3e1), + (hex!("0100000000333333334444444455000001e00000000000007010"), 0x02a481), + (hex!("0100000000333333334444444455000001e10000000000001e20"), 0x02a521), + (hex!("0100000000333333334444444455000001e10000000000005720"), 0x02a5c1), + (hex!("0100000000333333334444444455000001e10000000000006830"), 0x02a661), + (hex!("0100000000333333334444444455000001e20000000000001e30"), 0x02a701), + (hex!("0100000000333333334444444455000001e20000000000005100"), 0x02a7a1), + (hex!("0100000000333333334444444455000001e30000000000001e40"), 0x02a841), + (hex!("0100000000333333334444444455000001e40000000000001e50"), 0x02a8e1), + (hex!("0100000000333333334444444455000001e40000000000003f30"), 0x02a981), + (hex!("0100000000333333334444444455000001e40000000000005220"), 0x02aa21), + (hex!("0100000000333333334444444455000001e50000000000001e60"), 0x02aac1), + (hex!("0100000000333333334444444455000001e50000000000006f60"), 0x02ab61), + (hex!("0100000000333333334444444455000001e60000000000001e70"), 0x02ac01), + (hex!("0100000000333333334444444455000001e60000000000006c80"), 0x02aca1), + (hex!("0100000000333333334444444455000001e70000000000001e80"), 0x02ad41), + (hex!("0100000000333333334444444455000001e80000000000001e90"), 0x02ade1), + (hex!("0100000000333333334444444455000001e80000000000004e30"), 0x02ae81), + (hex!("0100000000333333334444444455000001e90000000000001ea0"), 0x02af21), + (hex!("0100000000333333334444444455000001e90000000000005470"), 0x02afc1), + (hex!("0100000000333333334444444455000001ea0000000000001eb0"), 0x02b061), + (hex!("0100000000333333334444444455000001ea0000000000007980"), 0x02b101), + (hex!("0100000000333333334444444455000001eb0000000000001ec0"), 0x02b1a1), + (hex!("0100000000333333334444444455000001eb0000000000004390"), 0x02b241), + (hex!("0100000000333333334444444455000001eb0000000000005970"), 0x02b2e1), + (hex!("0100000000333333334444444455000001ec0000000000001ed0"), 0x02b381), + (hex!("0100000000333333334444444455000001ec0000000000005d50"), 0x02b421), + (hex!("0100000000333333334444444455000001ec00000000000076e0"), 0x02b4c1), + (hex!("0100000000333333334444444455000001ed0000000000001ee0"), 0x02b561), + (hex!("0100000000333333334444444455000001ed0000000000006190"), 0x02b601), + (hex!("0100000000333333334444444455000001ee0000000000001ef0"), 0x02b6a1), + (hex!("0100000000333333334444444455000001ee0000000000004900"), 0x02b741), + (hex!("0100000000333333334444444455000001ef0000000000001f00"), 0x02b7e1), + (hex!("0100000000333333334444444455000001ef0000000000006c60"), 0x02b881), + (hex!("0100000000333333334444444455000001f00000000000001f10"), 0x02b921), + (hex!("0100000000333333334444444455000001f00000000000006950"), 0x02b9c1), + (hex!("0100000000333333334444444455000001f10000000000001f20"), 0x02ba61), + (hex!("0100000000333333334444444455000001f10000000000006400"), 0x02bb01), + (hex!("0100000000333333334444444455000001f20000000000001f30"), 0x02bba1), + (hex!("0100000000333333334444444455000001f20000000000006f00"), 0x02bc41), + (hex!("0100000000333333334444444455000001f20000000000007b10"), 0x02bce1), + (hex!("0100000000333333334444444455000001f30000000000001f40"), 0x02bd81), + (hex!("0100000000333333334444444455000001f40000000000001f50"), 0x02be21), + (hex!("0100000000333333334444444455000001f50000000000001f60"), 0x02bec1), + (hex!("0100000000333333334444444455000001f500000000000044f0"), 0x02bf61), + (hex!("0100000000333333334444444455000001f60000000000001f70"), 0x02c001), + (hex!("0100000000333333334444444455000001f70000000000001f80"), 0x02c0a1), + (hex!("0100000000333333334444444455000001f70000000000004ad0"), 0x02c141), + (hex!("0100000000333333334444444455000001f80000000000001f90"), 0x02c1e1), + (hex!("0100000000333333334444444455000001f90000000000001fa0"), 0x02c281), + (hex!("0100000000333333334444444455000001f90000000000003f60"), 0x02c321), + (hex!("0100000000333333334444444455000001f90000000000004a80"), 0x02c3c1), + (hex!("0100000000333333334444444455000001fa0000000000001fb0"), 0x02c461), + (hex!("0100000000333333334444444455000001fa0000000000006f90"), 0x02c501), + (hex!("0100000000333333334444444455000001fb0000000000001fc0"), 0x02c5a1), + (hex!("0100000000333333334444444455000001fc0000000000001fd0"), 0x02c641), + (hex!("0100000000333333334444444455000001fc0000000000004a90"), 0x02c6e1), + (hex!("0100000000333333334444444455000001fd0000000000001fe0"), 0x02c781), + (hex!("0100000000333333334444444455000001fd0000000000005f50"), 0x02c821), + (hex!("0100000000333333334444444455000001fe0000000000001ff0"), 0x02c8c1), + (hex!("0100000000333333334444444455000001ff0000000000002000"), 0x02c961), + (hex!("0100000000333333334444444455000002000000000000002010"), 0x02ca01), + (hex!("0100000000333333334444444455000002000000000000005f00"), 0x02caa1), + (hex!("0100000000333333334444444455000002000000000000006840"), 0x02cb41), + (hex!("0100000000333333334444444455000002010000000000002020"), 0x02cbe1), + (hex!("0100000000333333334444444455000002020000000000002030"), 0x02cc81), + (hex!("0100000000333333334444444455000002030000000000002040"), 0x02cd21), + (hex!("0100000000333333334444444455000002040000000000002050"), 0x02cdc1), + (hex!("01000000003333333344444444550000020400000000000051f0"), 0x02ce61), + (hex!("0100000000333333334444444455000002050000000000002060"), 0x02cf01), + (hex!("0100000000333333334444444455000002060000000000002070"), 0x02cfa1), + (hex!("0100000000333333334444444455000002060000000000005c80"), 0x02d041), + (hex!("01000000003333333344444444550000020600000000000061d0"), 0x02d0e1), + (hex!("01000000003333333344444444550000020600000000000078c0"), 0x02d181), + (hex!("0100000000333333334444444455000002070000000000002080"), 0x02d221), + (hex!("0100000000333333334444444455000002070000000000006ba0"), 0x02d2c1), + (hex!("0100000000333333334444444455000002080000000000002090"), 0x02d361), + (hex!("01000000003333333344444444550000020900000000000020a0"), 0x02d401), + (hex!("01000000003333333344444444550000020900000000000067a0"), 0x02d4a1), + (hex!("01000000003333333344444444550000020a00000000000020b0"), 0x02d541), + (hex!("01000000003333333344444444550000020a0000000000004950"), 0x02d5e1), + (hex!("01000000003333333344444444550000020a0000000000004de0"), 0x02d681), + (hex!("01000000003333333344444444550000020b00000000000020c0"), 0x02d721), + (hex!("01000000003333333344444444550000020b0000000000004b00"), 0x02d7c1), + (hex!("01000000003333333344444444550000020c00000000000020d0"), 0x02d861), + (hex!("01000000003333333344444444550000020d00000000000020e0"), 0x02d901), + (hex!("01000000003333333344444444550000020e00000000000020f0"), 0x02d9a1), + (hex!("01000000003333333344444444550000020f0000000000002100"), 0x02da41), + (hex!("0100000000333333334444444455000002100000000000002110"), 0x02dae1), + (hex!("0100000000333333334444444455000002110000000000002120"), 0x02db81), + (hex!("0100000000333333334444444455000002110000000000004490"), 0x02dc21), + (hex!("0100000000333333334444444455000002120000000000002130"), 0x02dcc1), + (hex!("0100000000333333334444444455000002130000000000002140"), 0x02dd61), + (hex!("01000000003333333344444444550000021300000000000046d0"), 0x02de01), + (hex!("01000000003333333344444444550000021300000000000046e0"), 0x02dea1), + (hex!("0100000000333333334444444455000002130000000000004b70"), 0x02df41), + (hex!("0100000000333333334444444455000002140000000000002150"), 0x02dfe1), + (hex!("0100000000333333334444444455000002140000000000006c50"), 0x02e081), + (hex!("0100000000333333334444444455000002150000000000002160"), 0x02e121), + (hex!("01000000003333333344444444550000021500000000000043c0"), 0x02e1c1), + (hex!("0100000000333333334444444455000002160000000000002170"), 0x02e261), + (hex!("01000000003333333344444444550000021600000000000055b0"), 0x02e301), + (hex!("0100000000333333334444444455000002160000000000006150"), 0x02e3a1), + (hex!("0100000000333333334444444455000002170000000000002180"), 0x02e441), + (hex!("01000000003333333344444444550000021700000000000053b0"), 0x02e4e1), + (hex!("0100000000333333334444444455000002170000000000007460"), 0x02e581), + (hex!("0100000000333333334444444455000002180000000000002190"), 0x02e621), + (hex!("01000000003333333344444444550000021900000000000021a0"), 0x02e6c1), + (hex!("01000000003333333344444444550000021a00000000000021b0"), 0x02e761), + (hex!("01000000003333333344444444550000021a0000000000007650"), 0x02e801), + (hex!("01000000003333333344444444550000021b00000000000021c0"), 0x02e8a1), + (hex!("01000000003333333344444444550000021b0000000000004b20"), 0x02e941), + (hex!("01000000003333333344444444550000021c00000000000021d0"), 0x02e9e1), + (hex!("01000000003333333344444444550000021c0000000000007610"), 0x02ea81), + (hex!("01000000003333333344444444550000021d00000000000021e0"), 0x02eb21), + (hex!("01000000003333333344444444550000021d0000000000005f40"), 0x02ebc1), + (hex!("01000000003333333344444444550000021e00000000000021f0"), 0x02ec61), + (hex!("01000000003333333344444444550000021e0000000000005a50"), 0x02ed01), + (hex!("01000000003333333344444444550000021e0000000000005ff0"), 0x02eda1), + (hex!("01000000003333333344444444550000021f0000000000002200"), 0x02ee41), + (hex!("01000000003333333344444444550000021f00000000000043a0"), 0x02eee1), + (hex!("01000000003333333344444444550000021f0000000000004cb0"), 0x02ef81), + (hex!("01000000003333333344444444550000021f0000000000004e00"), 0x02f021), + (hex!("0100000000333333334444444455000002200000000000002210"), 0x02f0c1), + (hex!("0100000000333333334444444455000002210000000000002220"), 0x02f161), + (hex!("0100000000333333334444444455000002210000000000006290"), 0x02f201), + (hex!("0100000000333333334444444455000002210000000000007230"), 0x02f2a1), + (hex!("0100000000333333334444444455000002220000000000002230"), 0x02f341), + (hex!("0100000000333333334444444455000002220000000000006ea0"), 0x02f3e1), + (hex!("0100000000333333334444444455000002230000000000002240"), 0x02f481), + (hex!("0100000000333333334444444455000002230000000000004710"), 0x02f521), + (hex!("0100000000333333334444444455000002240000000000002250"), 0x02f5c1), + (hex!("0100000000333333334444444455000002250000000000002260"), 0x02f661), + (hex!("0100000000333333334444444455000002260000000000002270"), 0x02f701), + (hex!("0100000000333333334444444455000002260000000000005b40"), 0x02f7a1), + (hex!("0100000000333333334444444455000002260000000000006300"), 0x02f841), + (hex!("0100000000333333334444444455000002270000000000002280"), 0x02f8e1), + (hex!("0100000000333333334444444455000002270000000000005b80"), 0x02f981), + (hex!("0100000000333333334444444455000002280000000000002290"), 0x02fa21), + (hex!("0100000000333333334444444455000002280000000000003ed0"), 0x02fac1), + (hex!("0100000000333333334444444455000002280000000000004550"), 0x02fb61), + (hex!("01000000003333333344444444550000022800000000000077d0"), 0x02fc01), + (hex!("01000000003333333344444444550000022900000000000022a0"), 0x02fca1), + (hex!("0100000000333333334444444455000002290000000000006480"), 0x02fd41), + (hex!("01000000003333333344444444550000022a00000000000022b0"), 0x02fde1), + (hex!("01000000003333333344444444550000022a0000000000005450"), 0x02fe81), + (hex!("01000000003333333344444444550000022b00000000000022c0"), 0x02ff21), + (hex!("01000000003333333344444444550000022b0000000000006dd0"), 0x02ffc1), + (hex!("01000000003333333344444444550000022c00000000000022d0"), 0x030061), + (hex!("01000000003333333344444444550000022c0000000000006890"), 0x030101), + (hex!("01000000003333333344444444550000022d00000000000022e0"), 0x0301a1), + (hex!("01000000003333333344444444550000022e00000000000022f0"), 0x030241), + (hex!("01000000003333333344444444550000022e0000000000004f20"), 0x0302e1), + (hex!("01000000003333333344444444550000022f0000000000002300"), 0x030381), + (hex!("01000000003333333344444444550000022f0000000000005260"), 0x030421), + (hex!("01000000003333333344444444550000022f00000000000053f0"), 0x0304c1), + (hex!("0100000000333333334444444455000002300000000000002310"), 0x030561), + (hex!("01000000003333333344444444550000023000000000000050e0"), 0x030601), + (hex!("0100000000333333334444444455000002310000000000002320"), 0x0306a1), + (hex!("0100000000333333334444444455000002310000000000007800"), 0x030741), + (hex!("0100000000333333334444444455000002320000000000002330"), 0x0307e1), + (hex!("0100000000333333334444444455000002330000000000002340"), 0x030881), + (hex!("0100000000333333334444444455000002330000000000004d70"), 0x030921), + (hex!("0100000000333333334444444455000002330000000000005cf0"), 0x0309c1), + (hex!("0100000000333333334444444455000002340000000000002350"), 0x030a61), + (hex!("0100000000333333334444444455000002350000000000002360"), 0x030b01), + (hex!("0100000000333333334444444455000002350000000000006970"), 0x030ba1), + (hex!("0100000000333333334444444455000002360000000000002370"), 0x030c41), + (hex!("0100000000333333334444444455000002360000000000005270"), 0x030ce1), + (hex!("0100000000333333334444444455000002370000000000002380"), 0x030d81), + (hex!("0100000000333333334444444455000002370000000000005d70"), 0x030e21), + (hex!("0100000000333333334444444455000002380000000000002390"), 0x030ec1), + (hex!("01000000003333333344444444550000023800000000000069a0"), 0x030f61), + (hex!("01000000003333333344444444550000023900000000000023a0"), 0x031001), + (hex!("01000000003333333344444444550000023900000000000052e0"), 0x0310a1), + (hex!("0100000000333333334444444455000002390000000000005a10"), 0x031141), + (hex!("0100000000333333334444444455000002390000000000007440"), 0x0311e1), + (hex!("01000000003333333344444444550000023a00000000000023b0"), 0x031281), + (hex!("01000000003333333344444444550000023a0000000000003f00"), 0x031321), + (hex!("01000000003333333344444444550000023a0000000000004430"), 0x0313c1), + (hex!("01000000003333333344444444550000023a0000000000007070"), 0x031461), + (hex!("01000000003333333344444444550000023a00000000000074a0"), 0x031501), + (hex!("01000000003333333344444444550000023b00000000000023c0"), 0x0315a1), + (hex!("01000000003333333344444444550000023b0000000000004730"), 0x031641), + (hex!("01000000003333333344444444550000023b00000000000068b0"), 0x0316e1), + (hex!("01000000003333333344444444550000023c00000000000023d0"), 0x031781), + (hex!("01000000003333333344444444550000023c0000000000004680"), 0x031821), + (hex!("01000000003333333344444444550000023d00000000000023e0"), 0x0318c1), + (hex!("01000000003333333344444444550000023d00000000000059a0"), 0x031961), + (hex!("01000000003333333344444444550000023e00000000000023f0"), 0x031a01), + (hex!("01000000003333333344444444550000023f0000000000002400"), 0x031aa1), + (hex!("0100000000333333334444444455000002400000000000002410"), 0x031b41), + (hex!("0100000000333333334444444455000002400000000000004920"), 0x031be1), + (hex!("01000000003333333344444444550000024000000000000066e0"), 0x031c81), + (hex!("01000000003333333344444444550000024000000000000076f0"), 0x031d21), + (hex!("01000000003333333344444444550000024000000000000078e0"), 0x031dc1), + (hex!("0100000000333333334444444455000002410000000000002420"), 0x031e61), + (hex!("0100000000333333334444444455000002420000000000002430"), 0x031f01), + (hex!("0100000000333333334444444455000002420000000000006590"), 0x031fa1), + (hex!("0100000000333333334444444455000002430000000000002440"), 0x032041), + (hex!("0100000000333333334444444455000002430000000000004d00"), 0x0320e1), + (hex!("0100000000333333334444444455000002440000000000002450"), 0x032181), + (hex!("0100000000333333334444444455000002440000000000005f80"), 0x032221), + (hex!("0100000000333333334444444455000002450000000000002460"), 0x0322c1), + (hex!("0100000000333333334444444455000002450000000000004940"), 0x032361), + (hex!("0100000000333333334444444455000002460000000000002470"), 0x032401), + (hex!("0100000000333333334444444455000002470000000000002480"), 0x0324a1), + (hex!("0100000000333333334444444455000002470000000000004dd0"), 0x032541), + (hex!("0100000000333333334444444455000002470000000000005930"), 0x0325e1), + (hex!("01000000003333333344444444550000024700000000000061b0"), 0x032681), + (hex!("0100000000333333334444444455000002470000000000007740"), 0x032721), + (hex!("0100000000333333334444444455000002480000000000002490"), 0x0327c1), + (hex!("0100000000333333334444444455000002480000000000004890"), 0x032861), + (hex!("01000000003333333344444444550000024900000000000024a0"), 0x032901), + (hex!("01000000003333333344444444550000024a00000000000024b0"), 0x0329a1), + (hex!("01000000003333333344444444550000024b00000000000024c0"), 0x032a41), + (hex!("01000000003333333344444444550000024c00000000000024d0"), 0x032ae1), + (hex!("01000000003333333344444444550000024d00000000000024e0"), 0x032b81), + (hex!("01000000003333333344444444550000024d0000000000004070"), 0x032c21), + (hex!("01000000003333333344444444550000024e00000000000024f0"), 0x032cc1), + (hex!("01000000003333333344444444550000024e00000000000066a0"), 0x032d61), + (hex!("01000000003333333344444444550000024e0000000000006ab0"), 0x032e01), + (hex!("01000000003333333344444444550000024f0000000000002500"), 0x032ea1), + (hex!("0100000000333333334444444455000002500000000000002510"), 0x032f41), + (hex!("0100000000333333334444444455000002510000000000002520"), 0x032fe1), + (hex!("0100000000333333334444444455000002510000000000007320"), 0x033081), + (hex!("0100000000333333334444444455000002520000000000002530"), 0x033121), + (hex!("0100000000333333334444444455000002520000000000006410"), 0x0331c1), + (hex!("0100000000333333334444444455000002530000000000002540"), 0x033261), + (hex!("0100000000333333334444444455000002530000000000005110"), 0x033301), + (hex!("0100000000333333334444444455000002540000000000002550"), 0x0333a1), + (hex!("01000000003333333344444444550000025400000000000040c0"), 0x033441), + (hex!("0100000000333333334444444455000002540000000000006a40"), 0x0334e1), + (hex!("0100000000333333334444444455000002550000000000002560"), 0x033581), + (hex!("0100000000333333334444444455000002550000000000005190"), 0x033621), + (hex!("0100000000333333334444444455000002560000000000002570"), 0x0336c1), + (hex!("01000000003333333344444444550000025600000000000061f0"), 0x033761), + (hex!("0100000000333333334444444455000002570000000000002580"), 0x033801), + (hex!("0100000000333333334444444455000002580000000000002590"), 0x0338a1), + (hex!("01000000003333333344444444550000025800000000000043d0"), 0x033941), + (hex!("01000000003333333344444444550000025900000000000025a0"), 0x0339e1), + (hex!("0100000000333333334444444455000002590000000000006bb0"), 0x033a81), + (hex!("01000000003333333344444444550000025a00000000000025b0"), 0x033b21), + (hex!("01000000003333333344444444550000025a0000000000005fb0"), 0x033bc1), + (hex!("01000000003333333344444444550000025a00000000000064c0"), 0x033c61), + (hex!("01000000003333333344444444550000025b00000000000025c0"), 0x033d01), + (hex!("01000000003333333344444444550000025b0000000000005c10"), 0x033da1), + (hex!("01000000003333333344444444550000025c00000000000025d0"), 0x033e41), + (hex!("01000000003333333344444444550000025c0000000000007d00"), 0x033ee1), + (hex!("01000000003333333344444444550000025d00000000000025e0"), 0x033f81), + (hex!("01000000003333333344444444550000025e00000000000025f0"), 0x034021), + (hex!("01000000003333333344444444550000025e00000000000045e0"), 0x0340c1), + (hex!("01000000003333333344444444550000025e0000000000006ee0"), 0x034161), + (hex!("01000000003333333344444444550000025f0000000000002600"), 0x034201), + (hex!("01000000003333333344444444550000025f00000000000050b0"), 0x0342a1), + (hex!("01000000003333333344444444550000025f0000000000007690"), 0x034341), + (hex!("0100000000333333334444444455000002600000000000002610"), 0x0343e1), + (hex!("0100000000333333334444444455000002600000000000007b60"), 0x034481), + (hex!("0100000000333333334444444455000002610000000000002620"), 0x034521), + (hex!("0100000000333333334444444455000002620000000000002630"), 0x0345c1), + (hex!("0100000000333333334444444455000002630000000000002640"), 0x034661), + (hex!("0100000000333333334444444455000002640000000000002650"), 0x034701), + (hex!("0100000000333333334444444455000002650000000000002660"), 0x0347a1), + (hex!("0100000000333333334444444455000002650000000000006180"), 0x034841), + (hex!("0100000000333333334444444455000002660000000000002670"), 0x0348e1), + (hex!("0100000000333333334444444455000002660000000000005430"), 0x034981), + (hex!("0100000000333333334444444455000002660000000000007a60"), 0x034a21), + (hex!("0100000000333333334444444455000002670000000000002680"), 0x034ac1), + (hex!("01000000003333333344444444550000026700000000000077f0"), 0x034b61), + (hex!("0100000000333333334444444455000002680000000000002690"), 0x034c01), + (hex!("01000000003333333344444444550000026900000000000026a0"), 0x034ca1), + (hex!("01000000003333333344444444550000026a00000000000026b0"), 0x034d41), + (hex!("01000000003333333344444444550000026a0000000000007530"), 0x034de1), + (hex!("01000000003333333344444444550000026b00000000000026c0"), 0x034e81), + (hex!("01000000003333333344444444550000026b00000000000058b0"), 0x034f21), + (hex!("01000000003333333344444444550000026b00000000000066b0"), 0x034fc1), + (hex!("01000000003333333344444444550000026b0000000000006b10"), 0x035061), + (hex!("01000000003333333344444444550000026c00000000000026d0"), 0x035101), + (hex!("01000000003333333344444444550000026d00000000000026e0"), 0x0351a1), + (hex!("01000000003333333344444444550000026d0000000000004210"), 0x035241), + (hex!("01000000003333333344444444550000026d0000000000005490"), 0x0352e1), + (hex!("01000000003333333344444444550000026d0000000000005e60"), 0x035381), + (hex!("01000000003333333344444444550000026d00000000000068e0"), 0x035421), + (hex!("01000000003333333344444444550000026d0000000000007020"), 0x0354c1), + (hex!("01000000003333333344444444550000026d0000000000007300"), 0x035561), + (hex!("01000000003333333344444444550000026e00000000000026f0"), 0x035601), + (hex!("01000000003333333344444444550000026f0000000000002700"), 0x0356a1), + (hex!("01000000003333333344444444550000026f0000000000004910"), 0x035741), + (hex!("0100000000333333334444444455000002700000000000002710"), 0x0357e1), + (hex!("0100000000333333334444444455000002710000000000002720"), 0x035881), + (hex!("01000000003333333344444444550000027100000000000050c0"), 0x035921), + (hex!("0100000000333333334444444455000002720000000000002730"), 0x0359c1), + (hex!("0100000000333333334444444455000002730000000000002740"), 0x035a61), + (hex!("0100000000333333334444444455000002740000000000002750"), 0x035b01), + (hex!("0100000000333333334444444455000002740000000000007490"), 0x035ba1), + (hex!("0100000000333333334444444455000002750000000000002760"), 0x035c41), + (hex!("0100000000333333334444444455000002760000000000002770"), 0x035ce1), + (hex!("0100000000333333334444444455000002760000000000004790"), 0x035d81), + (hex!("0100000000333333334444444455000002770000000000002780"), 0x035e21), + (hex!("01000000003333333344444444550000027700000000000050a0"), 0x035ec1), + (hex!("0100000000333333334444444455000002780000000000002790"), 0x035f61), + (hex!("0100000000333333334444444455000002780000000000004330"), 0x036001), + (hex!("0100000000333333334444444455000002780000000000006b00"), 0x0360a1), + (hex!("01000000003333333344444444550000027900000000000027a0"), 0x036141), + (hex!("01000000003333333344444444550000027a00000000000027b0"), 0x0361e1), + (hex!("01000000003333333344444444550000027b00000000000027c0"), 0x036281), + (hex!("01000000003333333344444444550000027b0000000000004930"), 0x036321), + (hex!("01000000003333333344444444550000027b0000000000006250"), 0x0363c1), + (hex!("01000000003333333344444444550000027c00000000000027d0"), 0x036461), + (hex!("01000000003333333344444444550000027d00000000000027e0"), 0x036501), + (hex!("01000000003333333344444444550000027d0000000000005ce0"), 0x0365a1), + (hex!("01000000003333333344444444550000027d0000000000005fe0"), 0x036641), + (hex!("01000000003333333344444444550000027e00000000000027f0"), 0x0366e1), + (hex!("01000000003333333344444444550000027f0000000000002800"), 0x036781), + (hex!("01000000003333333344444444550000027f0000000000003e90"), 0x036821), + (hex!("01000000003333333344444444550000027f0000000000007910"), 0x0368c1), + (hex!("0100000000333333334444444455000002800000000000002810"), 0x036961), + (hex!("0100000000333333334444444455000002800000000000004990"), 0x036a01), + (hex!("0100000000333333334444444455000002800000000000006160"), 0x036aa1), + (hex!("0100000000333333334444444455000002800000000000006740"), 0x036b41), + (hex!("0100000000333333334444444455000002810000000000002820"), 0x036be1), + (hex!("0100000000333333334444444455000002820000000000002830"), 0x036c81), + (hex!("0100000000333333334444444455000002820000000000005170"), 0x036d21), + (hex!("0100000000333333334444444455000002830000000000002840"), 0x036dc1), + (hex!("0100000000333333334444444455000002840000000000002850"), 0x036e61), + (hex!("0100000000333333334444444455000002840000000000004810"), 0x036f01), + (hex!("0100000000333333334444444455000002840000000000006aa0"), 0x036fa1), + (hex!("0100000000333333334444444455000002850000000000002860"), 0x037041), + (hex!("0100000000333333334444444455000002860000000000002870"), 0x0370e1), + (hex!("0100000000333333334444444455000002860000000000005080"), 0x037181), + (hex!("0100000000333333334444444455000002870000000000002880"), 0x037221), + (hex!("0100000000333333334444444455000002870000000000004e60"), 0x0372c1), + (hex!("0100000000333333334444444455000002880000000000002890"), 0x037361), + (hex!("0100000000333333334444444455000002880000000000005060"), 0x037401), + (hex!("0100000000333333334444444455000002880000000000006f20"), 0x0374a1), + (hex!("01000000003333333344444444550000028900000000000028a0"), 0x037541), + (hex!("01000000003333333344444444550000028900000000000047e0"), 0x0375e1), + (hex!("01000000003333333344444444550000028a00000000000028b0"), 0x037681), + (hex!("01000000003333333344444444550000028a0000000000005ab0"), 0x037721), + (hex!("01000000003333333344444444550000028a0000000000007130"), 0x0377c1), + (hex!("01000000003333333344444444550000028a0000000000007660"), 0x037861), + (hex!("01000000003333333344444444550000028b00000000000028c0"), 0x037901), + (hex!("01000000003333333344444444550000028b00000000000054e0"), 0x0379a1), + (hex!("01000000003333333344444444550000028c00000000000028d0"), 0x037a41), + (hex!("01000000003333333344444444550000028c00000000000046f0"), 0x037ae1), + (hex!("01000000003333333344444444550000028c00000000000061a0"), 0x037b81), + (hex!("01000000003333333344444444550000028d00000000000028e0"), 0x037c21), + (hex!("01000000003333333344444444550000028e00000000000028f0"), 0x037cc1), + (hex!("01000000003333333344444444550000028e0000000000004130"), 0x037d61), + (hex!("01000000003333333344444444550000028f0000000000002900"), 0x037e01), + (hex!("01000000003333333344444444550000028f0000000000007510"), 0x037ea1), + (hex!("0100000000333333334444444455000002900000000000002910"), 0x037f41), + (hex!("0100000000333333334444444455000002900000000000004a40"), 0x037fe1), + (hex!("0100000000333333334444444455000002910000000000002920"), 0x038081), + (hex!("0100000000333333334444444455000002920000000000002930"), 0x038121), + (hex!("0100000000333333334444444455000002920000000000004e90"), 0x0381c1), + (hex!("0100000000333333334444444455000002930000000000002940"), 0x038261), + (hex!("0100000000333333334444444455000002930000000000006880"), 0x038301), + (hex!("0100000000333333334444444455000002940000000000002950"), 0x0383a1), + (hex!("0100000000333333334444444455000002940000000000007bc0"), 0x038441), + (hex!("0100000000333333334444444455000002950000000000002960"), 0x0384e1), + (hex!("0100000000333333334444444455000002960000000000002970"), 0x038581), + (hex!("01000000003333333344444444550000029600000000000059d0"), 0x038621), + (hex!("0100000000333333334444444455000002970000000000002980"), 0x0386c1), + (hex!("0100000000333333334444444455000002970000000000004a50"), 0x038761), + (hex!("0100000000333333334444444455000002970000000000005f20"), 0x038801), + (hex!("01000000003333333344444444550000029700000000000068d0"), 0x0388a1), + (hex!("0100000000333333334444444455000002980000000000002990"), 0x038941), + (hex!("0100000000333333334444444455000002980000000000004370"), 0x0389e1), + (hex!("0100000000333333334444444455000002980000000000004420"), 0x038a81), + (hex!("01000000003333333344444444550000029900000000000029a0"), 0x038b21), + (hex!("01000000003333333344444444550000029a00000000000029b0"), 0x038bc1), + (hex!("01000000003333333344444444550000029a0000000000006010"), 0x038c61), + (hex!("01000000003333333344444444550000029a0000000000006980"), 0x038d01), + (hex!("01000000003333333344444444550000029b00000000000029c0"), 0x038da1), + (hex!("01000000003333333344444444550000029c00000000000029d0"), 0x038e41), + (hex!("01000000003333333344444444550000029c0000000000007480"), 0x038ee1), + (hex!("01000000003333333344444444550000029d00000000000029e0"), 0x038f81), + (hex!("01000000003333333344444444550000029d0000000000005030"), 0x039021), + (hex!("01000000003333333344444444550000029d0000000000007780"), 0x0390c1), + (hex!("01000000003333333344444444550000029d0000000000007a50"), 0x039161), + (hex!("01000000003333333344444444550000029e00000000000029f0"), 0x039201), + (hex!("01000000003333333344444444550000029e00000000000074b0"), 0x0392a1), + (hex!("01000000003333333344444444550000029f0000000000002a00"), 0x039341), + (hex!("0100000000333333334444444455000002a00000000000002a10"), 0x0393e1), + (hex!("0100000000333333334444444455000002a10000000000002a20"), 0x039481), + (hex!("0100000000333333334444444455000002a20000000000002a30"), 0x039521), + (hex!("0100000000333333334444444455000002a20000000000004c50"), 0x0395c1), + (hex!("0100000000333333334444444455000002a20000000000006f10"), 0x039661), + (hex!("0100000000333333334444444455000002a30000000000002a40"), 0x039701), + (hex!("0100000000333333334444444455000002a40000000000002a50"), 0x0397a1), + (hex!("0100000000333333334444444455000002a40000000000005d60"), 0x039841), + (hex!("0100000000333333334444444455000002a50000000000002a60"), 0x0398e1), + (hex!("0100000000333333334444444455000002a50000000000005440"), 0x039981), + (hex!("0100000000333333334444444455000002a50000000000005890"), 0x039a21), + (hex!("0100000000333333334444444455000002a60000000000002a70"), 0x039ac1), + (hex!("0100000000333333334444444455000002a70000000000002a80"), 0x039b61), + (hex!("0100000000333333334444444455000002a700000000000054a0"), 0x039c01), + (hex!("0100000000333333334444444455000002a70000000000007280"), 0x039ca1), + (hex!("0100000000333333334444444455000002a80000000000002a90"), 0x039d41), + (hex!("0100000000333333334444444455000002a90000000000002aa0"), 0x039de1), + (hex!("0100000000333333334444444455000002aa0000000000002ab0"), 0x039e81), + (hex!("0100000000333333334444444455000002ab0000000000002ac0"), 0x039f21), + (hex!("0100000000333333334444444455000002ab0000000000006c90"), 0x039fc1), + (hex!("0100000000333333334444444455000002ac0000000000002ad0"), 0x03a061), + (hex!("0100000000333333334444444455000002ac0000000000006db0"), 0x03a101), + (hex!("0100000000333333334444444455000002ad0000000000002ae0"), 0x03a1a1), + (hex!("0100000000333333334444444455000002ad00000000000065e0"), 0x03a241), + (hex!("0100000000333333334444444455000002ad0000000000007b40"), 0x03a2e1), + (hex!("0100000000333333334444444455000002ae0000000000002af0"), 0x03a381), + (hex!("0100000000333333334444444455000002ae0000000000004d20"), 0x03a421), + (hex!("0100000000333333334444444455000002ae0000000000006f30"), 0x03a4c1), + (hex!("0100000000333333334444444455000002af0000000000002b00"), 0x03a561), + (hex!("0100000000333333334444444455000002b00000000000002b10"), 0x03a601), + (hex!("0100000000333333334444444455000002b00000000000004560"), 0x03a6a1), + (hex!("0100000000333333334444444455000002b00000000000005800"), 0x03a741), + (hex!("0100000000333333334444444455000002b00000000000005a60"), 0x03a7e1), + (hex!("0100000000333333334444444455000002b10000000000002b20"), 0x03a881), + (hex!("0100000000333333334444444455000002b10000000000007b30"), 0x03a921), + (hex!("0100000000333333334444444455000002b20000000000002b30"), 0x03a9c1), + (hex!("0100000000333333334444444455000002b20000000000004440"), 0x03aa61), + (hex!("0100000000333333334444444455000002b20000000000004f80"), 0x03ab01), + (hex!("0100000000333333334444444455000002b20000000000005020"), 0x03aba1), + (hex!("0100000000333333334444444455000002b30000000000002b40"), 0x03ac41), + (hex!("0100000000333333334444444455000002b40000000000002b50"), 0x03ace1), + (hex!("0100000000333333334444444455000002b50000000000002b60"), 0x03ad81), + (hex!("0100000000333333334444444455000002b500000000000059e0"), 0x03ae21), + (hex!("0100000000333333334444444455000002b60000000000002b70"), 0x03aec1), + (hex!("0100000000333333334444444455000002b70000000000002b80"), 0x03af61), + (hex!("0100000000333333334444444455000002b80000000000002b90"), 0x03b001), + (hex!("0100000000333333334444444455000002b80000000000004590"), 0x03b0a1), + (hex!("0100000000333333334444444455000002b800000000000047d0"), 0x03b141), + (hex!("0100000000333333334444444455000002b80000000000006030"), 0x03b1e1), + (hex!("0100000000333333334444444455000002b80000000000006a20"), 0x03b281), + (hex!("0100000000333333334444444455000002b80000000000006a90"), 0x03b321), + (hex!("0100000000333333334444444455000002b90000000000002ba0"), 0x03b3c1), + (hex!("0100000000333333334444444455000002ba0000000000002bb0"), 0x03b461), + (hex!("0100000000333333334444444455000002ba0000000000006e80"), 0x03b501), + (hex!("0100000000333333334444444455000002bb0000000000002bc0"), 0x03b5a1), + (hex!("0100000000333333334444444455000002bc0000000000002bd0"), 0x03b641), + (hex!("0100000000333333334444444455000002bc0000000000004b30"), 0x03b6e1), + (hex!("0100000000333333334444444455000002bd0000000000002be0"), 0x03b781), + (hex!("0100000000333333334444444455000002bd0000000000005e10"), 0x03b821), + (hex!("0100000000333333334444444455000002be0000000000002bf0"), 0x03b8c1), + (hex!("0100000000333333334444444455000002bf0000000000002c00"), 0x03b961), + (hex!("0100000000333333334444444455000002c00000000000002c10"), 0x03ba01), + (hex!("0100000000333333334444444455000002c10000000000002c20"), 0x03baa1), + (hex!("0100000000333333334444444455000002c10000000000003ef0"), 0x03bb41), + (hex!("0100000000333333334444444455000002c20000000000002c30"), 0x03bbe1), + (hex!("0100000000333333334444444455000002c200000000000056e0"), 0x03bc81), + (hex!("0100000000333333334444444455000002c30000000000002c40"), 0x03bd21), + (hex!("0100000000333333334444444455000002c30000000000004b60"), 0x03bdc1), + (hex!("0100000000333333334444444455000002c40000000000002c50"), 0x03be61), + (hex!("0100000000333333334444444455000002c400000000000045f0"), 0x03bf01), + (hex!("0100000000333333334444444455000002c40000000000005290"), 0x03bfa1), + (hex!("0100000000333333334444444455000002c50000000000002c60"), 0x03c041), + (hex!("0100000000333333334444444455000002c60000000000002c70"), 0x03c0e1), + (hex!("0100000000333333334444444455000002c60000000000006ae0"), 0x03c181), + (hex!("0100000000333333334444444455000002c70000000000002c80"), 0x03c221), + (hex!("0100000000333333334444444455000002c70000000000005680"), 0x03c2c1), + (hex!("0100000000333333334444444455000002c70000000000006e10"), 0x03c361), + (hex!("0100000000333333334444444455000002c80000000000002c90"), 0x03c401), + (hex!("0100000000333333334444444455000002c90000000000002ca0"), 0x03c4a1), + (hex!("0100000000333333334444444455000002ca0000000000002cb0"), 0x03c541), + (hex!("0100000000333333334444444455000002cb0000000000002cc0"), 0x03c5e1), + (hex!("0100000000333333334444444455000002cc0000000000002cd0"), 0x03c681), + (hex!("0100000000333333334444444455000002cc0000000000005b50"), 0x03c721), + (hex!("0100000000333333334444444455000002cd0000000000002ce0"), 0x03c7c1), + (hex!("0100000000333333334444444455000002ce0000000000002cf0"), 0x03c861), + (hex!("0100000000333333334444444455000002ce00000000000043f0"), 0x03c901), + (hex!("0100000000333333334444444455000002ce0000000000006420"), 0x03c9a1), + (hex!("0100000000333333334444444455000002cf0000000000002d00"), 0x03ca41), + (hex!("0100000000333333334444444455000002d00000000000002d10"), 0x03cae1), + (hex!("0100000000333333334444444455000002d10000000000002d20"), 0x03cb81), + (hex!("0100000000333333334444444455000002d10000000000005370"), 0x03cc21), + (hex!("0100000000333333334444444455000002d20000000000002d30"), 0x03ccc1), + (hex!("0100000000333333334444444455000002d20000000000005ef0"), 0x03cd61), + (hex!("0100000000333333334444444455000002d20000000000006570"), 0x03ce01), + (hex!("0100000000333333334444444455000002d30000000000002d40"), 0x03cea1), + (hex!("0100000000333333334444444455000002d30000000000007360"), 0x03cf41), + (hex!("0100000000333333334444444455000002d40000000000002d50"), 0x03cfe1), + (hex!("0100000000333333334444444455000002d400000000000079a0"), 0x03d081), + (hex!("0100000000333333334444444455000002d50000000000002d60"), 0x03d121), + (hex!("0100000000333333334444444455000002d50000000000004250"), 0x03d1c1), + (hex!("0100000000333333334444444455000002d50000000000006050"), 0x03d261), + (hex!("0100000000333333334444444455000002d60000000000002d70"), 0x03d301), + (hex!("0100000000333333334444444455000002d60000000000007080"), 0x03d3a1), + (hex!("0100000000333333334444444455000002d70000000000002d80"), 0x03d441), + (hex!("0100000000333333334444444455000002d80000000000002d90"), 0x03d4e1), + (hex!("0100000000333333334444444455000002d80000000000007110"), 0x03d581), + (hex!("0100000000333333334444444455000002d800000000000073c0"), 0x03d621), + (hex!("0100000000333333334444444455000002d800000000000075a0"), 0x03d6c1), + (hex!("0100000000333333334444444455000002d90000000000002da0"), 0x03d761), + (hex!("0100000000333333334444444455000002d90000000000004860"), 0x03d801), + (hex!("0100000000333333334444444455000002d90000000000006b60"), 0x03d8a1), + (hex!("0100000000333333334444444455000002da0000000000002db0"), 0x03d941), + (hex!("0100000000333333334444444455000002da0000000000006630"), 0x03d9e1), + (hex!("0100000000333333334444444455000002db0000000000002dc0"), 0x03da81), + (hex!("0100000000333333334444444455000002dc0000000000002dd0"), 0x03db21), + (hex!("0100000000333333334444444455000002dc0000000000004830"), 0x03dbc1), + (hex!("0100000000333333334444444455000002dd0000000000002de0"), 0x03dc61), + (hex!("0100000000333333334444444455000002de0000000000002df0"), 0x03dd01), + (hex!("0100000000333333334444444455000002de0000000000004f00"), 0x03dda1), + (hex!("0100000000333333334444444455000002df0000000000002e00"), 0x03de41), + (hex!("0100000000333333334444444455000002e00000000000002e10"), 0x03dee1), + (hex!("0100000000333333334444444455000002e10000000000002e20"), 0x03df81), + (hex!("0100000000333333334444444455000002e10000000000006e90"), 0x03e021), + (hex!("0100000000333333334444444455000002e20000000000002e30"), 0x03e0c1), + (hex!("0100000000333333334444444455000002e200000000000053e0"), 0x03e161), + (hex!("0100000000333333334444444455000002e30000000000002e40"), 0x03e201), + (hex!("0100000000333333334444444455000002e30000000000006020"), 0x03e2a1), + (hex!("0100000000333333334444444455000002e30000000000006540"), 0x03e341), + (hex!("0100000000333333334444444455000002e40000000000002e50"), 0x03e3e1), + (hex!("0100000000333333334444444455000002e50000000000002e60"), 0x03e481), + (hex!("0100000000333333334444444455000002e50000000000005180"), 0x03e521), + (hex!("0100000000333333334444444455000002e50000000000007bf0"), 0x03e5c1), + (hex!("0100000000333333334444444455000002e60000000000002e70"), 0x03e661), + (hex!("0100000000333333334444444455000002e60000000000005350"), 0x03e701), + (hex!("0100000000333333334444444455000002e60000000000007960"), 0x03e7a1), + (hex!("0100000000333333334444444455000002e70000000000002e80"), 0x03e841), + (hex!("0100000000333333334444444455000002e80000000000002e90"), 0x03e8e1), + (hex!("0100000000333333334444444455000002e90000000000002ea0"), 0x03e981), + (hex!("0100000000333333334444444455000002ea0000000000002eb0"), 0x03ea21), + (hex!("0100000000333333334444444455000002eb0000000000002ec0"), 0x03eac1), + (hex!("0100000000333333334444444455000002ec0000000000002ed0"), 0x03eb61), + (hex!("0100000000333333334444444455000002ec0000000000006c10"), 0x03ec01), + (hex!("0100000000333333334444444455000002ed0000000000002ee0"), 0x03eca1), + (hex!("0100000000333333334444444455000002ed0000000000005590"), 0x03ed41), + (hex!("0100000000333333334444444455000002ed0000000000005cd0"), 0x03ede1), + (hex!("0100000000333333334444444455000002ed0000000000006910"), 0x03ee81), + (hex!("0100000000333333334444444455000002ee0000000000002ef0"), 0x03ef21), + (hex!("0100000000333333334444444455000002ef0000000000002f00"), 0x03efc1), + (hex!("0100000000333333334444444455000002ef0000000000004ed0"), 0x03f061), + (hex!("0100000000333333334444444455000002f00000000000002f10"), 0x03f101), + (hex!("0100000000333333334444444455000002f00000000000004cf0"), 0x03f1a1), + (hex!("0100000000333333334444444455000002f00000000000005d10"), 0x03f241), + (hex!("0100000000333333334444444455000002f00000000000006860"), 0x03f2e1), + (hex!("0100000000333333334444444455000002f00000000000006b50"), 0x03f381), + (hex!("0100000000333333334444444455000002f00000000000007100"), 0x03f421), + (hex!("0100000000333333334444444455000002f00000000000007aa0"), 0x03f4c1), + (hex!("0100000000333333334444444455000002f10000000000002f20"), 0x03f561), + (hex!("0100000000333333334444444455000002f20000000000002f30"), 0x03f601), + (hex!("0100000000333333334444444455000002f200000000000044b0"), 0x03f6a1), + (hex!("0100000000333333334444444455000002f30000000000002f40"), 0x03f741), + (hex!("0100000000333333334444444455000002f300000000000075b0"), 0x03f7e1), + (hex!("0100000000333333334444444455000002f40000000000002f50"), 0x03f881), + (hex!("0100000000333333334444444455000002f400000000000060f0"), 0x03f921), + (hex!("0100000000333333334444444455000002f50000000000002f60"), 0x03f9c1), + (hex!("0100000000333333334444444455000002f50000000000007210"), 0x03fa61), + (hex!("0100000000333333334444444455000002f60000000000002f70"), 0x03fb01), + (hex!("0100000000333333334444444455000002f60000000000006610"), 0x03fba1), + (hex!("0100000000333333334444444455000002f70000000000002f80"), 0x03fc41), + (hex!("0100000000333333334444444455000002f70000000000007560"), 0x03fce1), + (hex!("0100000000333333334444444455000002f80000000000002f90"), 0x03fd81), + (hex!("0100000000333333334444444455000002f80000000000006320"), 0x03fe21), + (hex!("0100000000333333334444444455000002f90000000000002fa0"), 0x03fec1), + (hex!("0100000000333333334444444455000002f90000000000006e50"), 0x03ff61), + (hex!("0100000000333333334444444455000002fa0000000000002fb0"), 0x040001), + (hex!("0100000000333333334444444455000002fb0000000000002fc0"), 0x0400a1), + (hex!("0100000000333333334444444455000002fb0000000000004780"), 0x040141), + (hex!("0100000000333333334444444455000002fc0000000000002fd0"), 0x0401e1), + (hex!("0100000000333333334444444455000002fd0000000000002fe0"), 0x040281), + (hex!("0100000000333333334444444455000002fd0000000000005600"), 0x040321), + (hex!("0100000000333333334444444455000002fd0000000000006c00"), 0x0403c1), + (hex!("0100000000333333334444444455000002fe0000000000002ff0"), 0x040461), + (hex!("0100000000333333334444444455000002ff0000000000003000"), 0x040501), + (hex!("0100000000333333334444444455000003000000000000003010"), 0x0405a1), + (hex!("0100000000333333334444444455000003000000000000004080"), 0x040641), + (hex!("0100000000333333334444444455000003010000000000003020"), 0x0406e1), + (hex!("0100000000333333334444444455000003010000000000006340"), 0x040781), + (hex!("0100000000333333334444444455000003020000000000003030"), 0x040821), + (hex!("0100000000333333334444444455000003020000000000005b00"), 0x0408c1), + (hex!("0100000000333333334444444455000003020000000000007b20"), 0x040961), + (hex!("0100000000333333334444444455000003030000000000003040"), 0x040a01), + (hex!("01000000003333333344444444550000030300000000000056b0"), 0x040aa1), + (hex!("0100000000333333334444444455000003030000000000006280"), 0x040b41), + (hex!("0100000000333333334444444455000003030000000000007ad0"), 0x040be1), + (hex!("0100000000333333334444444455000003040000000000003050"), 0x040c81), + (hex!("0100000000333333334444444455000003040000000000005c50"), 0x040d21), + (hex!("0100000000333333334444444455000003050000000000003060"), 0x040dc1), + (hex!("01000000003333333344444444550000030500000000000072e0"), 0x040e61), + (hex!("0100000000333333334444444455000003060000000000003070"), 0x040f01), + (hex!("0100000000333333334444444455000003060000000000004360"), 0x040fa1), + (hex!("0100000000333333334444444455000003060000000000004380"), 0x041041), + (hex!("0100000000333333334444444455000003060000000000004820"), 0x0410e1), + (hex!("0100000000333333334444444455000003060000000000006d10"), 0x041181), + (hex!("0100000000333333334444444455000003070000000000003080"), 0x041221), + (hex!("0100000000333333334444444455000003070000000000004450"), 0x0412c1), + (hex!("0100000000333333334444444455000003080000000000003090"), 0x041361), + (hex!("0100000000333333334444444455000003080000000000005ad0"), 0x041401), + (hex!("01000000003333333344444444550000030900000000000030a0"), 0x0414a1), + (hex!("01000000003333333344444444550000030a00000000000030b0"), 0x041541), + (hex!("01000000003333333344444444550000030a0000000000007760"), 0x0415e1), + (hex!("01000000003333333344444444550000030b00000000000030c0"), 0x041681), + (hex!("01000000003333333344444444550000030b0000000000007a80"), 0x041721), + (hex!("01000000003333333344444444550000030c00000000000030d0"), 0x0417c1), + (hex!("01000000003333333344444444550000030d00000000000030e0"), 0x041861), + (hex!("01000000003333333344444444550000030d0000000000003eb0"), 0x041901), + (hex!("01000000003333333344444444550000030e00000000000030f0"), 0x0419a1), + (hex!("01000000003333333344444444550000030f0000000000003100"), 0x041a41), + (hex!("01000000003333333344444444550000030f0000000000004690"), 0x041ae1), + (hex!("01000000003333333344444444550000030f0000000000006900"), 0x041b81), + (hex!("0100000000333333334444444455000003100000000000003110"), 0x041c21), + (hex!("01000000003333333344444444550000031000000000000058a0"), 0x041cc1), + (hex!("0100000000333333334444444455000003110000000000003120"), 0x041d61), + (hex!("0100000000333333334444444455000003110000000000004200"), 0x041e01), + (hex!("0100000000333333334444444455000003120000000000003130"), 0x041ea1), + (hex!("0100000000333333334444444455000003130000000000003140"), 0x041f41), + (hex!("0100000000333333334444444455000003130000000000004d50"), 0x041fe1), + (hex!("0100000000333333334444444455000003130000000000005400"), 0x042081), + (hex!("0100000000333333334444444455000003130000000000005520"), 0x042121), + (hex!("0100000000333333334444444455000003140000000000003150"), 0x0421c1), + (hex!("0100000000333333334444444455000003140000000000006450"), 0x042261), + (hex!("0100000000333333334444444455000003150000000000003160"), 0x042301), + (hex!("01000000003333333344444444550000031500000000000062d0"), 0x0423a1), + (hex!("0100000000333333334444444455000003160000000000003170"), 0x042441), + (hex!("0100000000333333334444444455000003160000000000004c40"), 0x0424e1), + (hex!("0100000000333333334444444455000003160000000000007c80"), 0x042581), + (hex!("0100000000333333334444444455000003170000000000003180"), 0x042621), + (hex!("0100000000333333334444444455000003170000000000004400"), 0x0426c1), + (hex!("0100000000333333334444444455000003170000000000005090"), 0x042761), + (hex!("0100000000333333334444444455000003170000000000006cb0"), 0x042801), + (hex!("0100000000333333334444444455000003180000000000003190"), 0x0428a1), + (hex!("0100000000333333334444444455000003180000000000006560"), 0x042941), + (hex!("01000000003333333344444444550000031900000000000031a0"), 0x0429e1), + (hex!("01000000003333333344444444550000031900000000000052d0"), 0x042a81), + (hex!("01000000003333333344444444550000031900000000000057e0"), 0x042b21), + (hex!("01000000003333333344444444550000031a00000000000031b0"), 0x042bc1), + (hex!("01000000003333333344444444550000031a00000000000071e0"), 0x042c61), + (hex!("01000000003333333344444444550000031b00000000000031c0"), 0x042d01), + (hex!("01000000003333333344444444550000031c00000000000031d0"), 0x042da1), + (hex!("01000000003333333344444444550000031c0000000000004480"), 0x042e41), + (hex!("01000000003333333344444444550000031c0000000000005790"), 0x042ee1), + (hex!("01000000003333333344444444550000031c0000000000007be0"), 0x042f81), + (hex!("01000000003333333344444444550000031d00000000000031e0"), 0x043021), + (hex!("01000000003333333344444444550000031d0000000000005560"), 0x0430c1), + (hex!("01000000003333333344444444550000031e00000000000031f0"), 0x043161), + (hex!("01000000003333333344444444550000031f0000000000003200"), 0x043201), + (hex!("01000000003333333344444444550000031f0000000000004190"), 0x0432a1), + (hex!("0100000000333333334444444455000003200000000000003210"), 0x043341), + (hex!("0100000000333333334444444455000003210000000000003220"), 0x0433e1), + (hex!("0100000000333333334444444455000003220000000000003230"), 0x043481), + (hex!("0100000000333333334444444455000003230000000000003240"), 0x043521), + (hex!("01000000003333333344444444550000032300000000000069d0"), 0x0435c1), + (hex!("0100000000333333334444444455000003240000000000003250"), 0x043661), + (hex!("0100000000333333334444444455000003250000000000003260"), 0x043701), + (hex!("01000000003333333344444444550000032500000000000042b0"), 0x0437a1), + (hex!("01000000003333333344444444550000032500000000000064e0"), 0x043841), + (hex!("0100000000333333334444444455000003260000000000003270"), 0x0438e1), + (hex!("0100000000333333334444444455000003270000000000003280"), 0x043981), + (hex!("0100000000333333334444444455000003270000000000005b20"), 0x043a21), + (hex!("0100000000333333334444444455000003270000000000006330"), 0x043ac1), + (hex!("0100000000333333334444444455000003270000000000006810"), 0x043b61), + (hex!("0100000000333333334444444455000003280000000000003290"), 0x043c01), + (hex!("01000000003333333344444444550000032900000000000032a0"), 0x043ca1), + (hex!("01000000003333333344444444550000032900000000000056f0"), 0x043d41), + (hex!("0100000000333333334444444455000003290000000000005e20"), 0x043de1), + (hex!("0100000000333333334444444455000003290000000000005e70"), 0x043e81), + (hex!("01000000003333333344444444550000032a00000000000032b0"), 0x043f21), + (hex!("01000000003333333344444444550000032b00000000000032c0"), 0x043fc1), + (hex!("01000000003333333344444444550000032b0000000000005500"), 0x044061), + (hex!("01000000003333333344444444550000032b0000000000005a20"), 0x044101), + (hex!("01000000003333333344444444550000032c00000000000032d0"), 0x0441a1), + (hex!("01000000003333333344444444550000032c0000000000004060"), 0x044241), + (hex!("01000000003333333344444444550000032c0000000000004760"), 0x0442e1), + (hex!("01000000003333333344444444550000032d00000000000032e0"), 0x044381), + (hex!("01000000003333333344444444550000032d00000000000068a0"), 0x044421), + (hex!("01000000003333333344444444550000032e00000000000032f0"), 0x0444c1), + (hex!("01000000003333333344444444550000032f0000000000003300"), 0x044561), + (hex!("0100000000333333334444444455000003300000000000003310"), 0x044601), + (hex!("0100000000333333334444444455000003300000000000006e40"), 0x0446a1), + (hex!("0100000000333333334444444455000003310000000000003320"), 0x044741), + (hex!("0100000000333333334444444455000003310000000000004620"), 0x0447e1), + (hex!("0100000000333333334444444455000003320000000000003330"), 0x044881), + (hex!("0100000000333333334444444455000003330000000000003340"), 0x044921), + (hex!("0100000000333333334444444455000003330000000000004b80"), 0x0449c1), + (hex!("0100000000333333334444444455000003340000000000003350"), 0x044a61), + (hex!("0100000000333333334444444455000003350000000000003360"), 0x044b01), + (hex!("0100000000333333334444444455000003360000000000003370"), 0x044ba1), + (hex!("0100000000333333334444444455000003370000000000003380"), 0x044c41), + (hex!("0100000000333333334444444455000003380000000000003390"), 0x044ce1), + (hex!("01000000003333333344444444550000033900000000000033a0"), 0x044d81), + (hex!("0100000000333333334444444455000003390000000000006b90"), 0x044e21), + (hex!("01000000003333333344444444550000033a00000000000033b0"), 0x044ec1), + (hex!("01000000003333333344444444550000033a0000000000007420"), 0x044f61), + (hex!("01000000003333333344444444550000033b00000000000033c0"), 0x045001), + (hex!("01000000003333333344444444550000033b0000000000007620"), 0x0450a1), + (hex!("01000000003333333344444444550000033c00000000000033d0"), 0x045141), + (hex!("01000000003333333344444444550000033c0000000000006b30"), 0x0451e1), + (hex!("01000000003333333344444444550000033d00000000000033e0"), 0x045281), + (hex!("01000000003333333344444444550000033e00000000000033f0"), 0x045321), + (hex!("01000000003333333344444444550000033e00000000000048b0"), 0x0453c1), + (hex!("01000000003333333344444444550000033e0000000000004e70"), 0x045461), + (hex!("01000000003333333344444444550000033f0000000000003400"), 0x045501), + (hex!("01000000003333333344444444550000033f0000000000006380"), 0x0455a1), + (hex!("0100000000333333334444444455000003400000000000003410"), 0x045641), + (hex!("0100000000333333334444444455000003410000000000003420"), 0x0456e1), + (hex!("0100000000333333334444444455000003410000000000006090"), 0x045781), + (hex!("0100000000333333334444444455000003420000000000003430"), 0x045821), + (hex!("01000000003333333344444444550000034200000000000073d0"), 0x0458c1), + (hex!("0100000000333333334444444455000003430000000000003440"), 0x045961), + (hex!("0100000000333333334444444455000003430000000000006370"), 0x045a01), + (hex!("01000000003333333344444444550000034300000000000075c0"), 0x045aa1), + (hex!("0100000000333333334444444455000003440000000000003450"), 0x045b41), + (hex!("0100000000333333334444444455000003450000000000003460"), 0x045be1), + (hex!("0100000000333333334444444455000003460000000000003470"), 0x045c81), + (hex!("01000000003333333344444444550000034600000000000055f0"), 0x045d21), + (hex!("0100000000333333334444444455000003470000000000003480"), 0x045dc1), + (hex!("0100000000333333334444444455000003470000000000003fe0"), 0x045e61), + (hex!("0100000000333333334444444455000003480000000000003490"), 0x045f01), + (hex!("0100000000333333334444444455000003480000000000007990"), 0x045fa1), + (hex!("01000000003333333344444444550000034900000000000034a0"), 0x046041), + (hex!("0100000000333333334444444455000003490000000000004410"), 0x0460e1), + (hex!("01000000003333333344444444550000034a00000000000034b0"), 0x046181), + (hex!("01000000003333333344444444550000034a00000000000062a0"), 0x046221), + (hex!("01000000003333333344444444550000034a0000000000007260"), 0x0462c1), + (hex!("01000000003333333344444444550000034b00000000000034c0"), 0x046361), + (hex!("01000000003333333344444444550000034b0000000000005760"), 0x046401), + (hex!("01000000003333333344444444550000034b0000000000006200"), 0x0464a1), + (hex!("01000000003333333344444444550000034c00000000000034d0"), 0x046541), + (hex!("01000000003333333344444444550000034d00000000000034e0"), 0x0465e1), + (hex!("01000000003333333344444444550000034e00000000000034f0"), 0x046681), + (hex!("01000000003333333344444444550000034e0000000000007790"), 0x046721), + (hex!("01000000003333333344444444550000034f0000000000003500"), 0x0467c1), + (hex!("0100000000333333334444444455000003500000000000003510"), 0x046861), + (hex!("0100000000333333334444444455000003510000000000003520"), 0x046901), + (hex!("0100000000333333334444444455000003520000000000003530"), 0x0469a1), + (hex!("01000000003333333344444444550000035200000000000056a0"), 0x046a41), + (hex!("0100000000333333334444444455000003530000000000003540"), 0x046ae1), + (hex!("0100000000333333334444444455000003540000000000003550"), 0x046b81), + (hex!("01000000003333333344444444550000035400000000000047b0"), 0x046c21), + (hex!("0100000000333333334444444455000003550000000000003560"), 0x046cc1), + (hex!("0100000000333333334444444455000003550000000000004500"), 0x046d61), + (hex!("0100000000333333334444444455000003560000000000003570"), 0x046e01), + (hex!("0100000000333333334444444455000003560000000000004fc0"), 0x046ea1), + (hex!("0100000000333333334444444455000003560000000000007160"), 0x046f41), + (hex!("0100000000333333334444444455000003560000000000007400"), 0x046fe1), + (hex!("0100000000333333334444444455000003570000000000003580"), 0x047081), + (hex!("0100000000333333334444444455000003580000000000003590"), 0x047121), + (hex!("0100000000333333334444444455000003580000000000005a80"), 0x0471c1), + (hex!("01000000003333333344444444550000035900000000000035a0"), 0x047261), + (hex!("01000000003333333344444444550000035900000000000073b0"), 0x047301), + (hex!("01000000003333333344444444550000035a00000000000035b0"), 0x0473a1), + (hex!("01000000003333333344444444550000035a0000000000004c20"), 0x047441), + (hex!("01000000003333333344444444550000035b00000000000035c0"), 0x0474e1), + (hex!("01000000003333333344444444550000035b0000000000005120"), 0x047581), + (hex!("01000000003333333344444444550000035c00000000000035d0"), 0x047621), + (hex!("01000000003333333344444444550000035c0000000000004300"), 0x0476c1), + (hex!("01000000003333333344444444550000035c0000000000005a40"), 0x047761), + (hex!("01000000003333333344444444550000035c0000000000006620"), 0x047801), + (hex!("01000000003333333344444444550000035c0000000000006ed0"), 0x0478a1), + (hex!("01000000003333333344444444550000035d00000000000035e0"), 0x047941), + (hex!("01000000003333333344444444550000035d0000000000005df0"), 0x0479e1), + (hex!("01000000003333333344444444550000035e00000000000035f0"), 0x047a81), + (hex!("01000000003333333344444444550000035f0000000000003600"), 0x047b21), + (hex!("01000000003333333344444444550000035f00000000000058d0"), 0x047bc1), + (hex!("0100000000333333334444444455000003600000000000003610"), 0x047c61), + (hex!("0100000000333333334444444455000003600000000000007b90"), 0x047d01), + (hex!("0100000000333333334444444455000003610000000000003620"), 0x047da1), + (hex!("0100000000333333334444444455000003610000000000006ad0"), 0x047e41), + (hex!("0100000000333333334444444455000003620000000000003630"), 0x047ee1), + (hex!("01000000003333333344444444550000036200000000000063a0"), 0x047f81), + (hex!("0100000000333333334444444455000003630000000000003640"), 0x048021), + (hex!("0100000000333333334444444455000003630000000000007250"), 0x0480c1), + (hex!("0100000000333333334444444455000003640000000000003650"), 0x048161), + (hex!("0100000000333333334444444455000003640000000000005510"), 0x048201), + (hex!("0100000000333333334444444455000003640000000000007850"), 0x0482a1), + (hex!("0100000000333333334444444455000003650000000000003660"), 0x048341), + (hex!("0100000000333333334444444455000003660000000000003670"), 0x0483e1), + (hex!("0100000000333333334444444455000003660000000000004650"), 0x048481), + (hex!("01000000003333333344444444550000036600000000000050d0"), 0x048521), + (hex!("0100000000333333334444444455000003660000000000006eb0"), 0x0485c1), + (hex!("0100000000333333334444444455000003670000000000003680"), 0x048661), + (hex!("01000000003333333344444444550000036700000000000071f0"), 0x048701), + (hex!("0100000000333333334444444455000003680000000000003690"), 0x0487a1), + (hex!("01000000003333333344444444550000036900000000000036a0"), 0x048841), + (hex!("0100000000333333334444444455000003690000000000005c70"), 0x0488e1), + (hex!("01000000003333333344444444550000036a00000000000036b0"), 0x048981), + (hex!("01000000003333333344444444550000036a00000000000071b0"), 0x048a21), + (hex!("01000000003333333344444444550000036b00000000000036c0"), 0x048ac1), + (hex!("01000000003333333344444444550000036b0000000000004670"), 0x048b61), + (hex!("01000000003333333344444444550000036c00000000000036d0"), 0x048c01), + (hex!("01000000003333333344444444550000036c0000000000004750"), 0x048ca1), + (hex!("01000000003333333344444444550000036c0000000000006fa0"), 0x048d41), + (hex!("01000000003333333344444444550000036d00000000000036e0"), 0x048de1), + (hex!("01000000003333333344444444550000036d0000000000003f70"), 0x048e81), + (hex!("01000000003333333344444444550000036d0000000000004b90"), 0x048f21), + (hex!("01000000003333333344444444550000036d00000000000057a0"), 0x048fc1), + (hex!("01000000003333333344444444550000036e00000000000036f0"), 0x049061), + (hex!("01000000003333333344444444550000036e00000000000075d0"), 0x049101), + (hex!("01000000003333333344444444550000036f0000000000003700"), 0x0491a1), + (hex!("0100000000333333334444444455000003700000000000003710"), 0x049241), + (hex!("0100000000333333334444444455000003700000000000005aa0"), 0x0492e1), + (hex!("0100000000333333334444444455000003710000000000003720"), 0x049381), + (hex!("0100000000333333334444444455000003710000000000005130"), 0x049421), + (hex!("0100000000333333334444444455000003710000000000006fc0"), 0x0494c1), + (hex!("0100000000333333334444444455000003710000000000007b00"), 0x049561), + (hex!("0100000000333333334444444455000003720000000000003730"), 0x049601), + (hex!("01000000003333333344444444550000037200000000000054d0"), 0x0496a1), + (hex!("0100000000333333334444444455000003730000000000003740"), 0x049741), + (hex!("0100000000333333334444444455000003730000000000004220"), 0x0497e1), + (hex!("0100000000333333334444444455000003740000000000003750"), 0x049881), + (hex!("0100000000333333334444444455000003740000000000004720"), 0x049921), + (hex!("0100000000333333334444444455000003750000000000003760"), 0x0499c1), + (hex!("0100000000333333334444444455000003750000000000004110"), 0x049a61), + (hex!("0100000000333333334444444455000003760000000000003770"), 0x049b01), + (hex!("0100000000333333334444444455000003770000000000003780"), 0x049ba1), + (hex!("0100000000333333334444444455000003780000000000003790"), 0x049c41), + (hex!("0100000000333333334444444455000003780000000000004b40"), 0x049ce1), + (hex!("0100000000333333334444444455000003780000000000005660"), 0x049d81), + (hex!("0100000000333333334444444455000003780000000000005ea0"), 0x049e21), + (hex!("01000000003333333344444444550000037900000000000037a0"), 0x049ec1), + (hex!("01000000003333333344444444550000037a00000000000037b0"), 0x049f61), + (hex!("01000000003333333344444444550000037b00000000000037c0"), 0x04a001), + (hex!("01000000003333333344444444550000037c00000000000037d0"), 0x04a0a1), + (hex!("01000000003333333344444444550000037c0000000000004340"), 0x04a141), + (hex!("01000000003333333344444444550000037c0000000000005230"), 0x04a1e1), + (hex!("01000000003333333344444444550000037d00000000000037e0"), 0x04a281), + (hex!("01000000003333333344444444550000037d00000000000051e0"), 0x04a321), + (hex!("01000000003333333344444444550000037e00000000000037f0"), 0x04a3c1), + (hex!("01000000003333333344444444550000037e0000000000004090"), 0x04a461), + (hex!("01000000003333333344444444550000037e0000000000005c20"), 0x04a501), + (hex!("01000000003333333344444444550000037f0000000000003800"), 0x04a5a1), + (hex!("0100000000333333334444444455000003800000000000003810"), 0x04a641), + (hex!("0100000000333333334444444455000003800000000000007630"), 0x04a6e1), + (hex!("0100000000333333334444444455000003810000000000003820"), 0x04a781), + (hex!("0100000000333333334444444455000003820000000000003830"), 0x04a821), + (hex!("0100000000333333334444444455000003820000000000004170"), 0x04a8c1), + (hex!("0100000000333333334444444455000003830000000000003840"), 0x04a961), + (hex!("0100000000333333334444444455000003840000000000003850"), 0x04aa01), + (hex!("0100000000333333334444444455000003850000000000003860"), 0x04aaa1), + (hex!("0100000000333333334444444455000003850000000000004180"), 0x04ab41), + (hex!("0100000000333333334444444455000003850000000000005c90"), 0x04abe1), + (hex!("0100000000333333334444444455000003850000000000005da0"), 0x04ac81), + (hex!("0100000000333333334444444455000003850000000000006ff0"), 0x04ad21), + (hex!("0100000000333333334444444455000003860000000000003870"), 0x04adc1), + (hex!("01000000003333333344444444550000038600000000000065c0"), 0x04ae61), + (hex!("0100000000333333334444444455000003870000000000003880"), 0x04af01), + (hex!("0100000000333333334444444455000003870000000000007cc0"), 0x04afa1), + (hex!("0100000000333333334444444455000003880000000000003890"), 0x04b041), + (hex!("01000000003333333344444444550000038900000000000038a0"), 0x04b0e1), + (hex!("01000000003333333344444444550000038a00000000000038b0"), 0x04b181), + (hex!("01000000003333333344444444550000038a00000000000073e0"), 0x04b221), + (hex!("01000000003333333344444444550000038b00000000000038c0"), 0x04b2c1), + (hex!("01000000003333333344444444550000038c00000000000038d0"), 0x04b361), + (hex!("01000000003333333344444444550000038d00000000000038e0"), 0x04b401), + (hex!("01000000003333333344444444550000038d00000000000069f0"), 0x04b4a1), + (hex!("01000000003333333344444444550000038d0000000000007680"), 0x04b541), + (hex!("01000000003333333344444444550000038e00000000000038f0"), 0x04b5e1), + (hex!("01000000003333333344444444550000038f0000000000003900"), 0x04b681), + (hex!("01000000003333333344444444550000038f00000000000045b0"), 0x04b721), + (hex!("01000000003333333344444444550000038f0000000000007180"), 0x04b7c1), + (hex!("0100000000333333334444444455000003900000000000003910"), 0x04b861), + (hex!("0100000000333333334444444455000003910000000000003920"), 0x04b901), + (hex!("0100000000333333334444444455000003910000000000004a20"), 0x04b9a1), + (hex!("0100000000333333334444444455000003920000000000003930"), 0x04ba41), + (hex!("01000000003333333344444444550000039200000000000059b0"), 0x04bae1), + (hex!("0100000000333333334444444455000003930000000000003940"), 0x04bb81), + (hex!("0100000000333333334444444455000003930000000000006cc0"), 0x04bc21), + (hex!("0100000000333333334444444455000003940000000000003950"), 0x04bcc1), + (hex!("01000000003333333344444444550000039400000000000056c0"), 0x04bd61), + (hex!("0100000000333333334444444455000003950000000000003960"), 0x04be01), + (hex!("0100000000333333334444444455000003950000000000004cc0"), 0x04bea1), + (hex!("0100000000333333334444444455000003950000000000007720"), 0x04bf41), + (hex!("0100000000333333334444444455000003960000000000003970"), 0x04bfe1), + (hex!("0100000000333333334444444455000003960000000000004da0"), 0x04c081), + (hex!("0100000000333333334444444455000003960000000000004df0"), 0x04c121), + (hex!("0100000000333333334444444455000003960000000000004f30"), 0x04c1c1), + (hex!("01000000003333333344444444550000039600000000000050f0"), 0x04c261), + (hex!("0100000000333333334444444455000003960000000000007940"), 0x04c301), + (hex!("0100000000333333334444444455000003970000000000003980"), 0x04c3a1), + (hex!("0100000000333333334444444455000003970000000000005850"), 0x04c441), + (hex!("0100000000333333334444444455000003970000000000007bd0"), 0x04c4e1), + (hex!("0100000000333333334444444455000003980000000000003990"), 0x04c581), + (hex!("0100000000333333334444444455000003980000000000004c00"), 0x04c621), + (hex!("0100000000333333334444444455000003980000000000005580"), 0x04c6c1), + (hex!("01000000003333333344444444550000039900000000000039a0"), 0x04c761), + (hex!("0100000000333333334444444455000003990000000000005820"), 0x04c801), + (hex!("01000000003333333344444444550000039a00000000000039b0"), 0x04c8a1), + (hex!("01000000003333333344444444550000039b00000000000039c0"), 0x04c941), + (hex!("01000000003333333344444444550000039b0000000000004c10"), 0x04c9e1), + (hex!("01000000003333333344444444550000039b0000000000006460"), 0x04ca81), + (hex!("01000000003333333344444444550000039c00000000000039d0"), 0x04cb21), + (hex!("01000000003333333344444444550000039d00000000000039e0"), 0x04cbc1), + (hex!("01000000003333333344444444550000039d00000000000044c0"), 0x04cc61), + (hex!("01000000003333333344444444550000039d00000000000049e0"), 0x04cd01), + (hex!("01000000003333333344444444550000039e00000000000039f0"), 0x04cda1), + (hex!("01000000003333333344444444550000039f0000000000003a00"), 0x04ce41), + (hex!("0100000000333333334444444455000003a00000000000003a10"), 0x04cee1), + (hex!("0100000000333333334444444455000003a10000000000003a20"), 0x04cf81), + (hex!("0100000000333333334444444455000003a10000000000006a80"), 0x04d021), + (hex!("0100000000333333334444444455000003a20000000000003a30"), 0x04d0c1), + (hex!("0100000000333333334444444455000003a200000000000062b0"), 0x04d161), + (hex!("0100000000333333334444444455000003a30000000000003a40"), 0x04d201), + (hex!("0100000000333333334444444455000003a30000000000006ce0"), 0x04d2a1), + (hex!("0100000000333333334444444455000003a40000000000003a50"), 0x04d341), + (hex!("0100000000333333334444444455000003a50000000000003a60"), 0x04d3e1), + (hex!("0100000000333333334444444455000003a60000000000003a70"), 0x04d481), + (hex!("0100000000333333334444444455000003a60000000000007750"), 0x04d521), + (hex!("0100000000333333334444444455000003a70000000000003a80"), 0x04d5c1), + (hex!("0100000000333333334444444455000003a70000000000005b10"), 0x04d661), + (hex!("0100000000333333334444444455000003a80000000000003a90"), 0x04d701), + (hex!("0100000000333333334444444455000003a80000000000006c20"), 0x04d7a1), + (hex!("0100000000333333334444444455000003a90000000000003aa0"), 0x04d841), + (hex!("0100000000333333334444444455000003a90000000000005b70"), 0x04d8e1), + (hex!("0100000000333333334444444455000003a900000000000070e0"), 0x04d981), + (hex!("0100000000333333334444444455000003aa0000000000003ab0"), 0x04da21), + (hex!("0100000000333333334444444455000003aa00000000000049f0"), 0x04dac1), + (hex!("0100000000333333334444444455000003aa0000000000004d60"), 0x04db61), + (hex!("0100000000333333334444444455000003ab0000000000003ac0"), 0x04dc01), + (hex!("0100000000333333334444444455000003ac0000000000003ad0"), 0x04dca1), + (hex!("0100000000333333334444444455000003ac0000000000004580"), 0x04dd41), + (hex!("0100000000333333334444444455000003ad0000000000003ae0"), 0x04dde1), + (hex!("0100000000333333334444444455000003ae0000000000003af0"), 0x04de81), + (hex!("0100000000333333334444444455000003af0000000000003b00"), 0x04df21), + (hex!("0100000000333333334444444455000003b00000000000003b10"), 0x04dfc1), + (hex!("0100000000333333334444444455000003b10000000000003b20"), 0x04e061), + (hex!("0100000000333333334444444455000003b10000000000003fd0"), 0x04e101), + (hex!("0100000000333333334444444455000003b20000000000003b30"), 0x04e1a1), + (hex!("0100000000333333334444444455000003b30000000000003b40"), 0x04e241), + (hex!("0100000000333333334444444455000003b40000000000003b50"), 0x04e2e1), + (hex!("0100000000333333334444444455000003b40000000000007450"), 0x04e381), + (hex!("0100000000333333334444444455000003b50000000000003b60"), 0x04e421), + (hex!("0100000000333333334444444455000003b60000000000003b70"), 0x04e4c1), + (hex!("0100000000333333334444444455000003b70000000000003b80"), 0x04e561), + (hex!("0100000000333333334444444455000003b70000000000006d50"), 0x04e601), + (hex!("0100000000333333334444444455000003b80000000000003b90"), 0x04e6a1), + (hex!("0100000000333333334444444455000003b800000000000057c0"), 0x04e741), + (hex!("0100000000333333334444444455000003b800000000000078a0"), 0x04e7e1), + (hex!("0100000000333333334444444455000003b90000000000003ba0"), 0x04e881), + (hex!("0100000000333333334444444455000003b90000000000006750"), 0x04e921), + (hex!("0100000000333333334444444455000003ba0000000000003bb0"), 0x04e9c1), + (hex!("0100000000333333334444444455000003ba0000000000007a10"), 0x04ea61), + (hex!("0100000000333333334444444455000003ba0000000000007a20"), 0x04eb01), + (hex!("0100000000333333334444444455000003bb0000000000003bc0"), 0x04eba1), + (hex!("0100000000333333334444444455000003bb0000000000005bc0"), 0x04ec41), + (hex!("0100000000333333334444444455000003bc0000000000003bd0"), 0x04ece1), + (hex!("0100000000333333334444444455000003bc0000000000005e80"), 0x04ed81), + (hex!("0100000000333333334444444455000003bc0000000000007ab0"), 0x04ee21), + (hex!("0100000000333333334444444455000003bd0000000000003be0"), 0x04eec1), + (hex!("0100000000333333334444444455000003bd00000000000049b0"), 0x04ef61), + (hex!("0100000000333333334444444455000003be0000000000003bf0"), 0x04f001), + (hex!("0100000000333333334444444455000003be0000000000005780"), 0x04f0a1), + (hex!("0100000000333333334444444455000003be0000000000007930"), 0x04f141), + (hex!("0100000000333333334444444455000003bf0000000000003c00"), 0x04f1e1), + (hex!("0100000000333333334444444455000003bf0000000000005de0"), 0x04f281), + (hex!("0100000000333333334444444455000003bf00000000000060b0"), 0x04f321), + (hex!("0100000000333333334444444455000003bf00000000000060c0"), 0x04f3c1), + (hex!("0100000000333333334444444455000003bf0000000000006a50"), 0x04f461), + (hex!("0100000000333333334444444455000003c00000000000003c10"), 0x04f501), + (hex!("0100000000333333334444444455000003c00000000000004030"), 0x04f5a1), + (hex!("0100000000333333334444444455000003c10000000000003c20"), 0x04f641), + (hex!("0100000000333333334444444455000003c20000000000003c30"), 0x04f6e1), + (hex!("0100000000333333334444444455000003c200000000000040b0"), 0x04f781), + (hex!("0100000000333333334444444455000003c30000000000003c40"), 0x04f821), + (hex!("0100000000333333334444444455000003c40000000000003c50"), 0x04f8c1), + (hex!("0100000000333333334444444455000003c40000000000005ba0"), 0x04f961), + (hex!("0100000000333333334444444455000003c50000000000003c60"), 0x04fa01), + (hex!("0100000000333333334444444455000003c60000000000003c70"), 0x04faa1), + (hex!("0100000000333333334444444455000003c70000000000003c80"), 0x04fb41), + (hex!("0100000000333333334444444455000003c70000000000004270"), 0x04fbe1), + (hex!("0100000000333333334444444455000003c80000000000003c90"), 0x04fc81), + (hex!("0100000000333333334444444455000003c80000000000006e70"), 0x04fd21), + (hex!("0100000000333333334444444455000003c90000000000003ca0"), 0x04fdc1), + (hex!("0100000000333333334444444455000003ca0000000000003cb0"), 0x04fe61), + (hex!("0100000000333333334444444455000003ca0000000000006e20"), 0x04ff01), + (hex!("0100000000333333334444444455000003ca0000000000007c20"), 0x04ffa1), + (hex!("0100000000333333334444444455000003cb0000000000003cc0"), 0x050041), + (hex!("0100000000333333334444444455000003cc0000000000003cd0"), 0x0500e1), + (hex!("0100000000333333334444444455000003cc0000000000006120"), 0x050181), + (hex!("0100000000333333334444444455000003cc0000000000007950"), 0x050221), + (hex!("0100000000333333334444444455000003cd0000000000003ce0"), 0x0502c1), + (hex!("0100000000333333334444444455000003ce0000000000003cf0"), 0x050361), + (hex!("0100000000333333334444444455000003cf0000000000003d00"), 0x050401), + (hex!("0100000000333333334444444455000003d00000000000003d10"), 0x0504a1), + (hex!("0100000000333333334444444455000003d10000000000003d20"), 0x050541), + (hex!("0100000000333333334444444455000003d10000000000005e50"), 0x0505e1), + (hex!("0100000000333333334444444455000003d10000000000007880"), 0x050681), + (hex!("0100000000333333334444444455000003d20000000000003d30"), 0x050721), + (hex!("0100000000333333334444444455000003d20000000000005d00"), 0x0507c1), + (hex!("0100000000333333334444444455000003d30000000000003d40"), 0x050861), + (hex!("0100000000333333334444444455000003d30000000000005d40"), 0x050901), + (hex!("0100000000333333334444444455000003d300000000000063f0"), 0x0509a1), + (hex!("0100000000333333334444444455000003d40000000000003d50"), 0x050a41), + (hex!("0100000000333333334444444455000003d40000000000005700"), 0x050ae1), + (hex!("0100000000333333334444444455000003d400000000000078f0"), 0x050b81), + (hex!("0100000000333333334444444455000003d50000000000003d60"), 0x050c21), + (hex!("0100000000333333334444444455000003d60000000000003d70"), 0x050cc1), + (hex!("0100000000333333334444444455000003d70000000000003d80"), 0x050d61), + (hex!("0100000000333333334444444455000003d80000000000003d90"), 0x050e01), + (hex!("0100000000333333334444444455000003d80000000000006690"), 0x050ea1), + (hex!("0100000000333333334444444455000003d90000000000003da0"), 0x050f41), + (hex!("0100000000333333334444444455000003d900000000000076d0"), 0x050fe1), + (hex!("0100000000333333334444444455000003da0000000000003db0"), 0x051081), + (hex!("0100000000333333334444444455000003db0000000000003dc0"), 0x051121), + (hex!("0100000000333333334444444455000003db0000000000004a30"), 0x0511c1), + (hex!("0100000000333333334444444455000003db0000000000005390"), 0x051261), + (hex!("0100000000333333334444444455000003dc0000000000003dd0"), 0x051301), + (hex!("0100000000333333334444444455000003dc0000000000006d60"), 0x0513a1), + (hex!("0100000000333333334444444455000003dd0000000000003de0"), 0x051441), + (hex!("0100000000333333334444444455000003de0000000000003df0"), 0x0514e1), + (hex!("0100000000333333334444444455000003df0000000000003e00"), 0x051581), + (hex!("0100000000333333334444444455000003df0000000000005240"), 0x051621), + (hex!("0100000000333333334444444455000003df0000000000005610"), 0x0516c1), + (hex!("0100000000333333334444444455000003e00000000000003e10"), 0x051761), + (hex!("0100000000333333334444444455000003e00000000000006500"), 0x051801), + (hex!("0100000000333333334444444455000003e10000000000003e20"), 0x0518a1), + (hex!("0100000000333333334444444455000003e10000000000006a10"), 0x051941), + (hex!("0100000000333333334444444455000003e10000000000007c10"), 0x0519e1), + (hex!("0100000000333333334444444455000003e20000000000003e30"), 0x051a81), + (hex!("0100000000333333334444444455000003e20000000000006310"), 0x051b21), + (hex!("0100000000333333334444444455000003e30000000000003e40"), 0x051bc1), + (hex!("0100000000333333334444444455000003e40000000000003e50"), 0x051c61), + (hex!("0100000000333333334444444455000003e40000000000006780"), 0x051d01), + (hex!("0100000000333333334444444455000003e40000000000007ce0"), 0x051da1), + (hex!("0100000000333333334444444455000003e50000000000003e60"), 0x051e41), + (hex!("0100000000333333334444444455000003e60000000000003e70"), 0x051ee1), + (hex!("0100000000333333334444444455000003e60000000000005040"), 0x051f81), + (hex!("0100000000333333334444444455000003e60000000000005bf0"), 0x052021), + (hex!("0100000000333333334444444455000003e70000000000003e80"), 0x0520c1), + (hex!("0100000000333333334444444455000003e70000000000003f50"), 0x052161), ]; diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 1275250bf0..2cb1e55b26 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -345,14 +345,19 @@ impl InMemoryLayer { let cursor = inner.file.block_cursor(); - let mut keys: Vec<(&Key, &VecMap)> = inner.index.iter().collect(); - keys.sort_by_key(|k| k.0); + // Sort the keys because delta layer writer expects them sorted. + // + // NOTE: this sort can take up significant time if the layer has millions of + // keys. To speed up all the comparisons we convert the key to i128 and + // keep the value as a reference. + let mut keys: Vec<_> = inner.index.iter().map(|(k, m)| (k.to_i128(), m)).collect(); + keys.sort_unstable_by_key(|k| k.0); let ctx = RequestContextBuilder::extend(ctx) .page_content_kind(PageContentKind::InMemoryLayer) .build(); for (key, vec_map) in keys.iter() { - let key = **key; + let key = Key::from_i128(*key); // Write all page versions for (lsn, pos) in vec_map.as_slice() { cursor.read_blob_into_buf(*pos, &mut buf, &ctx).await?; From e6470ee92efec67a4bc845bb149a7b06260456dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 6 Nov 2023 15:00:07 +0100 Subject: [PATCH 08/63] Add API description for safekeeper copy endpoint (#5770) Adds a yaml API description for a new endpoint that allows creation of a new timeline as the copy of an existing one. Part of #5282 --- safekeeper/src/http/openapi_spec.yaml | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/safekeeper/src/http/openapi_spec.yaml b/safekeeper/src/http/openapi_spec.yaml index 51ce7589a0..a617e0310c 100644 --- a/safekeeper/src/http/openapi_spec.yaml +++ b/safekeeper/src/http/openapi_spec.yaml @@ -86,6 +86,41 @@ paths: default: $ref: "#/components/responses/GenericError" + /v1/tenant/{tenant_id}/timeline/{source_timeline_id}/copy: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex + - name: source_timeline_id + in: path + required: true + schema: + type: string + format: hex + + post: + tags: + - "Timeline" + summary: Register new timeline as copy of existing timeline + description: "" + operationId: v1CopyTenantTimeline + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/TimelineCopyRequest" + responses: + "201": + description: Timeline created + # TODO: return timeline info? + "403": + $ref: "#/components/responses/ForbiddenError" + default: + $ref: "#/components/responses/GenericError" + /v1/tenant/{tenant_id}/timeline/{timeline_id}: parameters: @@ -210,6 +245,18 @@ components: type: integer minimum: 0 + TimelineCopyRequest: + type: object + required: + - target_timeline_id + - until_lsn + properties: + target_timeline_id: + type: string + format: hex + until_lsn: + type: string + SkTimelineInfo: type: object required: From 85cd97af61ffc92f0e4c7ad7ee4992af0e98dce2 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 6 Nov 2023 14:03:22 +0000 Subject: [PATCH 09/63] pageserver: add `InProgress` tenant map state, use a sync lock for the map (#5367) ## Problem Follows on from #5299 - We didn't have a generic way to protect a tenant undergoing changes: `Tenant` had states, but for our arbitrary transitions between secondary/attached, we need a general way to say "reserve this tenant ID, and don't allow any other ops on it, but don't try and report it as being in any particular state". - The TenantsMap structure was behind an async RwLock, but it was never correct to hold it across await points: that would block any other changes for all tenants. ## Summary of changes - Add the `TenantSlot::InProgress` value. This means: - Incoming administrative operations on the tenant should retry later - Anything trying to read the live state of the tenant (e.g. a page service reader) should retry later or block. - Store TenantsMap in `std::sync::RwLock` - Provide an extended `get_active_tenant_with_timeout` for page_service to use, which will wait on InProgress slots as well as non-active tenants. Closes: https://github.com/neondatabase/neon/issues/5378 --------- Co-authored-by: Christian Schwarz --- libs/utils/src/lib.rs | 3 + libs/utils/src/timeout.rs | 37 + pageserver/src/consumption_metrics.rs | 2 +- pageserver/src/consumption_metrics/metrics.rs | 1 - pageserver/src/disk_usage_eviction_task.rs | 2 +- pageserver/src/http/routes.rs | 77 +- pageserver/src/metrics.rs | 29 + pageserver/src/page_service.rs | 134 +- pageserver/src/tenant.rs | 63 +- pageserver/src/tenant/delete.rs | 52 +- pageserver/src/tenant/mgr.rs | 1203 ++++++++++++----- .../src/tenant/timeline/eviction_task.rs | 15 +- test_runner/fixtures/neon_fixtures.py | 17 + test_runner/regress/test_neon_cli.py | 7 + .../regress/test_pageserver_restart.py | 5 + test_runner/regress/test_tenant_delete.py | 8 + test_runner/regress/test_tenant_detach.py | 76 +- 17 files changed, 1142 insertions(+), 589 deletions(-) create mode 100644 libs/utils/src/timeout.rs diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 1e34034023..ff5d774285 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -77,6 +77,9 @@ pub mod completion; /// Reporting utilities pub mod error; +/// async timeout helper +pub mod timeout; + pub mod sync; /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages diff --git a/libs/utils/src/timeout.rs b/libs/utils/src/timeout.rs new file mode 100644 index 0000000000..11fa417242 --- /dev/null +++ b/libs/utils/src/timeout.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use tokio_util::sync::CancellationToken; + +pub enum TimeoutCancellableError { + Timeout, + Cancelled, +} + +/// Wrap [`tokio::time::timeout`] with a CancellationToken. +/// +/// This wrapper is appropriate for any long running operation in a task +/// that ought to respect a CancellationToken (which means most tasks). +/// +/// The only time you should use a bare tokio::timeout is when the future `F` +/// itself respects a CancellationToken: otherwise, always use this wrapper +/// with your CancellationToken to ensure that your task does not hold up +/// graceful shutdown. +pub async fn timeout_cancellable( + duration: Duration, + cancel: &CancellationToken, + future: F, +) -> Result +where + F: std::future::Future, +{ + tokio::select!( + r = tokio::time::timeout(duration, future) => { + r.map_err(|_| TimeoutCancellableError::Timeout) + + }, + _ = cancel.cancelled() => { + Err(TimeoutCancellableError::Cancelled) + + } + ) +} diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 061045eb76..9e8377c1f1 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -266,7 +266,7 @@ async fn calculate_synthetic_size_worker( continue; } - if let Ok(tenant) = mgr::get_tenant(tenant_id, true).await { + if let Ok(tenant) = mgr::get_tenant(tenant_id, true) { // TODO should we use concurrent_background_tasks_rate_limit() here, like the other background tasks? // We can put in some prioritization for consumption metrics. // Same for the loop that fetches computed metrics. diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index c22f218976..2989e15e8e 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -202,7 +202,6 @@ pub(super) async fn collect_all_metrics( None } else { crate::tenant::mgr::get_tenant(id, true) - .await .ok() .map(|tenant| (id, tenant)) } diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index bc4bf51862..642cafad28 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -545,7 +545,7 @@ async fn collect_eviction_candidates( if cancel.is_cancelled() { return Ok(EvictionCandidates::Cancelled); } - let tenant = match tenant::mgr::get_tenant(*tenant_id, true).await { + let tenant = match tenant::mgr::get_tenant(*tenant_id, true) { Ok(tenant) => tenant, Err(e) => { // this can happen if tenant has lifecycle transition after we fetched it diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index db728f243e..820b8eae46 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -35,7 +35,8 @@ use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; use crate::tenant::config::{LocationConf, TenantConfOpt}; use crate::tenant::mgr::{ - GetTenantError, SetNewTenantConfigError, TenantMapInsertError, TenantStateError, + GetTenantError, SetNewTenantConfigError, TenantMapError, TenantMapInsertError, TenantSlotError, + TenantSlotUpsertError, TenantStateError, }; use crate::tenant::size::ModelInputs; use crate::tenant::storage_layer::LayerAccessStatsReset; @@ -146,28 +147,59 @@ impl From for ApiError { impl From for ApiError { fn from(tmie: TenantMapInsertError) -> ApiError { match tmie { - TenantMapInsertError::StillInitializing | TenantMapInsertError::ShuttingDown => { - ApiError::ResourceUnavailable(format!("{tmie}").into()) - } - TenantMapInsertError::TenantAlreadyExists(id, state) => { - ApiError::Conflict(format!("tenant {id} already exists, state: {state:?}")) - } - TenantMapInsertError::TenantExistsSecondary(id) => { - ApiError::Conflict(format!("tenant {id} already exists as secondary")) - } + TenantMapInsertError::SlotError(e) => e.into(), + TenantMapInsertError::SlotUpsertError(e) => e.into(), TenantMapInsertError::Other(e) => ApiError::InternalServerError(e), } } } +impl From for ApiError { + fn from(e: TenantSlotError) -> ApiError { + use TenantSlotError::*; + match e { + NotFound(tenant_id) => { + ApiError::NotFound(anyhow::anyhow!("NotFound: tenant {tenant_id}").into()) + } + e @ (AlreadyExists(_, _) | Conflict(_)) => ApiError::Conflict(format!("{e}")), + InProgress => { + ApiError::ResourceUnavailable("Tenant is being modified concurrently".into()) + } + MapState(e) => e.into(), + } + } +} + +impl From for ApiError { + fn from(e: TenantSlotUpsertError) -> ApiError { + use TenantSlotUpsertError::*; + match e { + InternalError(e) => ApiError::InternalServerError(anyhow::anyhow!("{e}")), + MapState(e) => e.into(), + } + } +} + +impl From for ApiError { + fn from(e: TenantMapError) -> ApiError { + use TenantMapError::*; + match e { + StillInitializing | ShuttingDown => { + ApiError::ResourceUnavailable(format!("{e}").into()) + } + } + } +} + impl From for ApiError { fn from(tse: TenantStateError) -> ApiError { match tse { - TenantStateError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid).into()), TenantStateError::IsStopping(_) => { ApiError::ResourceUnavailable("Tenant is stopping".into()) } - _ => ApiError::InternalServerError(anyhow::Error::new(tse)), + TenantStateError::SlotError(e) => e.into(), + TenantStateError::SlotUpsertError(e) => e.into(), + TenantStateError::Other(e) => ApiError::InternalServerError(anyhow!(e)), } } } @@ -188,6 +220,7 @@ impl From for ApiError { // (We can produce this variant only in `mgr::get_tenant(..., active=true)` calls). ApiError::ResourceUnavailable("Tenant not yet active".into()) } + GetTenantError::MapState(e) => ApiError::ResourceUnavailable(format!("{e}").into()), } } } @@ -242,6 +275,9 @@ impl From for ApiError { Get(g) => ApiError::from(g), e @ AlreadyInProgress => ApiError::Conflict(e.to_string()), Timeline(t) => ApiError::from(t), + NotAttached => ApiError::NotFound(anyhow::anyhow!("Tenant is not attached").into()), + SlotError(e) => e.into(), + SlotUpsertError(e) => e.into(), Other(o) => ApiError::InternalServerError(o), e @ InvalidState(_) => ApiError::PreconditionFailed(e.to_string().into_boxed_str()), } @@ -368,7 +404,7 @@ async fn timeline_create_handler( let state = get_state(&request); async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; match tenant.create_timeline( new_timeline_id, request_data.ancestor_timeline_id.map(TimelineId::from), @@ -418,7 +454,7 @@ async fn timeline_list_handler( let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let response_data = async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; let timelines = tenant.list_timelines(); let mut response_data = Vec::with_capacity(timelines.len()); @@ -457,7 +493,7 @@ async fn timeline_detail_handler( let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let timeline_info = async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; let timeline = tenant .get_timeline(timeline_id, false) @@ -713,7 +749,7 @@ async fn tenant_status( check_permission(&request, Some(tenant_id))?; let tenant_info = async { - let tenant = mgr::get_tenant(tenant_id, false).await?; + let tenant = mgr::get_tenant(tenant_id, false)?; // Calculate total physical size of all timelines let mut current_physical_size = 0; @@ -776,7 +812,7 @@ async fn tenant_size_handler( let headers = request.headers(); let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; // this can be long operation let inputs = tenant @@ -1033,7 +1069,7 @@ async fn get_tenant_config_handler( let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; - let tenant = mgr::get_tenant(tenant_id, false).await?; + let tenant = mgr::get_tenant(tenant_id, false)?; let response = HashMap::from([ ( @@ -1092,7 +1128,7 @@ async fn put_tenant_location_config_handler( .await { match e { - TenantStateError::NotFound(_) => { + TenantStateError::SlotError(TenantSlotError::NotFound(_)) => { // This API is idempotent: a NotFound on a detach is fine. } _ => return Err(e.into()), @@ -1130,7 +1166,6 @@ async fn handle_tenant_break( let tenant_id: TenantId = parse_request_param(&r, "tenant_id")?; let tenant = crate::tenant::mgr::get_tenant(tenant_id, true) - .await .map_err(|_| ApiError::Conflict(String::from("no active tenant found")))?; tenant.set_broken("broken from test".to_owned()).await; @@ -1435,7 +1470,7 @@ async fn active_timeline_of_active_tenant( tenant_id: TenantId, timeline_id: TimelineId, ) -> Result, ApiError> { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; tenant .get_timeline(timeline_id, true) .map_err(|e| ApiError::NotFound(e.into())) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 3e15be67dc..429ab801d9 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -962,6 +962,32 @@ static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy = Lazy .expect("failed to define a metric") }); +pub(crate) struct TenantManagerMetrics { + pub(crate) tenant_slots: UIntGauge, + pub(crate) tenant_slot_writes: IntCounter, + pub(crate) unexpected_errors: IntCounter, +} + +pub(crate) static TENANT_MANAGER: Lazy = Lazy::new(|| { + TenantManagerMetrics { + tenant_slots: register_uint_gauge!( + "pageserver_tenant_manager_slots", + "How many slots currently exist, including all attached, secondary and in-progress operations", + ) + .expect("failed to define a metric"), + tenant_slot_writes: register_int_counter!( + "pageserver_tenant_manager_slot_writes", + "Writes to a tenant slot, including all of create/attach/detach/delete" + ) + .expect("failed to define a metric"), + unexpected_errors: register_int_counter!( + "pageserver_tenant_manager_unexpected_errors_total", + "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug." + ) + .expect("failed to define a metric"), +} +}); + pub(crate) struct DeletionQueueMetrics { pub(crate) keys_submitted: IntCounter, pub(crate) keys_dropped: IntCounter, @@ -1884,6 +1910,9 @@ pub fn preinitialize_metrics() { // Deletion queue stats Lazy::force(&DELETION_QUEUE); + // Tenant manager stats + Lazy::force(&TENANT_MANAGER); + // countervecs [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT] .into_iter() diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 334aee3dd1..6086d0b063 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -14,7 +14,6 @@ use async_compression::tokio::write::GzipEncoder; use bytes::Buf; use bytes::Bytes; use futures::Stream; -use pageserver_api::models::TenantState; use pageserver_api::models::{ PagestreamBeMessage, PagestreamDbSizeRequest, PagestreamDbSizeResponse, PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse, @@ -55,16 +54,20 @@ use crate::metrics; use crate::metrics::LIVE_CONNECTIONS_COUNT; use crate::task_mgr; use crate::task_mgr::TaskKind; -use crate::tenant; use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::mgr; -use crate::tenant::mgr::GetTenantError; -use crate::tenant::{Tenant, Timeline}; +use crate::tenant::mgr::get_active_tenant_with_timeout; +use crate::tenant::mgr::GetActiveTenantError; +use crate::tenant::Timeline; use crate::trace::Tracer; use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID; use postgres_ffi::BLCKSZ; +// How long we may block waiting for a [`TenantSlot::InProgress`]` and/or a [`Tenant`] which +// is not yet in state [`TenantState::Active`]. +const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(5000); + /// Read the end of a tar archive. /// /// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each. @@ -378,7 +381,12 @@ impl PageServerHandler { debug_assert_current_span_has_tenant_and_timeline_id(); // Make request tracer if needed - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = mgr::get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; let mut tracer = if tenant.get_trace_read_requests() { let connection_id = ConnectionId::generate(); let path = tenant @@ -529,7 +537,12 @@ impl PageServerHandler { // Create empty timeline info!("creating new timeline"); - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; let timeline = tenant .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx) .await?; @@ -587,7 +600,9 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let last_record_lsn = timeline.get_last_record_lsn(); if last_record_lsn != start_lsn { return Err(QueryError::Other( @@ -795,7 +810,9 @@ impl PageServerHandler { let started = std::time::Instant::now(); // check that the timeline exists - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); if let Some(lsn) = lsn { // Backup was requested at a particular LSN. Wait for it to arrive. @@ -894,6 +911,25 @@ impl PageServerHandler { .expect("claims presence already checked"); check_permission(claims, tenant_id) } + + /// Shorthand for getting a reference to a Timeline of an Active tenant. + async fn get_active_tenant_timeline( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> Result, GetActiveTimelineError> { + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await + .map_err(GetActiveTimelineError::Tenant)?; + let timeline = tenant + .get_timeline(timeline_id, true) + .map_err(|e| GetActiveTimelineError::Timeline(anyhow::anyhow!(e)))?; + Ok(timeline) + } } #[async_trait::async_trait] @@ -1051,7 +1087,9 @@ where .record("timeline_id", field::display(timeline_id)); self.check_permission(Some(tenant_id))?; - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let end_of_timeline = timeline.get_last_record_rlsn(); @@ -1235,7 +1273,12 @@ where self.check_permission(Some(tenant_id))?; - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; pgb.write_message_noflush(&BeMessage::RowDescription(&[ RowDescriptor::int8_col(b"checkpoint_distance"), RowDescriptor::int8_col(b"checkpoint_timeout"), @@ -1281,67 +1324,13 @@ where } } -#[derive(thiserror::Error, Debug)] -enum GetActiveTenantError { - #[error( - "Timed out waiting {wait_time:?} for tenant active state. Latest state: {latest_state:?}" - )] - WaitForActiveTimeout { - latest_state: TenantState, - wait_time: Duration, - }, - #[error(transparent)] - NotFound(GetTenantError), - #[error(transparent)] - WaitTenantActive(tenant::WaitToBecomeActiveError), -} - impl From for QueryError { fn from(e: GetActiveTenantError) -> Self { match e { GetActiveTenantError::WaitForActiveTimeout { .. } => QueryError::Disconnected( ConnectionError::Io(io::Error::new(io::ErrorKind::TimedOut, e.to_string())), ), - GetActiveTenantError::WaitTenantActive(e) => QueryError::Other(anyhow::Error::new(e)), - GetActiveTenantError::NotFound(e) => QueryError::Other(anyhow::Error::new(e)), - } - } -} - -/// Get active tenant. -/// -/// If the tenant is Loading, waits for it to become Active, for up to 30 s. That -/// ensures that queries don't fail immediately after pageserver startup, because -/// all tenants are still loading. -async fn get_active_tenant_with_timeout( - tenant_id: TenantId, - _ctx: &RequestContext, /* require get a context to support cancellation in the future */ -) -> Result, GetActiveTenantError> { - let tenant = match mgr::get_tenant(tenant_id, false).await { - Ok(tenant) => tenant, - Err(e @ GetTenantError::NotFound(_)) => return Err(GetActiveTenantError::NotFound(e)), - Err(GetTenantError::NotActive(_)) => { - unreachable!("we're calling get_tenant with active_only=false") - } - Err(GetTenantError::Broken(_)) => { - unreachable!("we're calling get_tenant with active_only=false") - } - }; - let wait_time = Duration::from_secs(30); - match tokio::time::timeout(wait_time, tenant.wait_to_become_active()).await { - Ok(Ok(())) => Ok(tenant), - // no .context(), the error message is good enough and some tests depend on it - Ok(Err(e)) => Err(GetActiveTenantError::WaitTenantActive(e)), - Err(_) => { - let latest_state = tenant.current_state(); - if latest_state == TenantState::Active { - Ok(tenant) - } else { - Err(GetActiveTenantError::WaitForActiveTimeout { - latest_state, - wait_time, - }) - } + e => QueryError::Other(anyhow::anyhow!(e)), } } } @@ -1362,18 +1351,3 @@ impl From for QueryError { } } } - -/// Shorthand for getting a reference to a Timeline of an Active tenant. -async fn get_active_tenant_timeline( - tenant_id: TenantId, - timeline_id: TimelineId, - ctx: &RequestContext, -) -> Result, GetActiveTimelineError> { - let tenant = get_active_tenant_with_timeout(tenant_id, ctx) - .await - .map_err(GetActiveTimelineError::Tenant)?; - let timeline = tenant - .get_timeline(timeline_id, true) - .map_err(|e| GetActiveTimelineError::Timeline(anyhow::anyhow!(e)))?; - Ok(timeline) -} diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index ad538a82fd..a738633d5e 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -55,6 +55,8 @@ use self::config::TenantConf; use self::delete::DeleteTenantFlow; use self::metadata::LoadMetadataError; use self::metadata::TimelineMetadata; +use self::mgr::GetActiveTenantError; +use self::mgr::GetTenantError; use self::mgr::TenantsMap; use self::remote_timeline_client::RemoteTimelineClient; use self::timeline::uninit::TimelineUninitMark; @@ -263,6 +265,12 @@ pub struct Tenant { pub(crate) gate: Gate, } +impl std::fmt::Debug for Tenant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ({})", self.tenant_id, self.current_state()) + } +} + pub(crate) enum WalRedoManager { Prod(PostgresRedoManager), #[cfg(test)] @@ -368,34 +376,6 @@ impl Debug for SetStoppingError { } } -#[derive(Debug, thiserror::Error)] -pub(crate) enum WaitToBecomeActiveError { - WillNotBecomeActive { - tenant_id: TenantId, - state: TenantState, - }, - TenantDropped { - tenant_id: TenantId, - }, -} - -impl std::fmt::Display for WaitToBecomeActiveError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - WaitToBecomeActiveError::WillNotBecomeActive { tenant_id, state } => { - write!( - f, - "Tenant {} will not become active. Current state: {:?}", - tenant_id, state - ) - } - WaitToBecomeActiveError::TenantDropped { tenant_id } => { - write!(f, "Tenant {tenant_id} will not become active (dropped)") - } - } - } -} - #[derive(thiserror::Error, Debug)] pub enum CreateTimelineError { #[error("a timeline with the given ID already exists")] @@ -537,7 +517,7 @@ impl Tenant { resources: TenantSharedResources, attached_conf: AttachedTenantConf, init_order: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { @@ -1850,6 +1830,7 @@ impl Tenant { } Err(SetStoppingError::AlreadyStopping(other)) => { // give caller the option to wait for this this shutdown + info!("Tenant::shutdown: AlreadyStopping"); return Err(other); } }; @@ -2048,7 +2029,7 @@ impl Tenant { self.state.subscribe() } - pub(crate) async fn wait_to_become_active(&self) -> Result<(), WaitToBecomeActiveError> { + pub(crate) async fn wait_to_become_active(&self) -> Result<(), GetActiveTenantError> { let mut receiver = self.state.subscribe(); loop { let current_state = receiver.borrow_and_update().clone(); @@ -2056,11 +2037,9 @@ impl Tenant { TenantState::Loading | TenantState::Attaching | TenantState::Activating(_) => { // in these states, there's a chance that we can reach ::Active receiver.changed().await.map_err( - |_e: tokio::sync::watch::error::RecvError| { - WaitToBecomeActiveError::TenantDropped { - tenant_id: self.tenant_id, - } - }, + |_e: tokio::sync::watch::error::RecvError| + // Tenant existed but was dropped: report it as non-existent + GetActiveTenantError::NotFound(GetTenantError::NotFound(self.tenant_id)) )?; } TenantState::Active { .. } => { @@ -2068,10 +2047,7 @@ impl Tenant { } TenantState::Broken { .. } | TenantState::Stopping { .. } => { // There's no chance the tenant can transition back into ::Active - return Err(WaitToBecomeActiveError::WillNotBecomeActive { - tenant_id: self.tenant_id, - state: current_state, - }); + return Err(GetActiveTenantError::WillNotBecomeActive(current_state)); } } } @@ -2137,6 +2113,9 @@ where } impl Tenant { + pub fn get_tenant_id(&self) -> TenantId { + self.tenant_id + } pub fn tenant_specific_overrides(&self) -> TenantConfOpt { self.tenant_conf.read().unwrap().tenant_conf } @@ -4266,11 +4245,7 @@ mod tests { metadata_bytes[8] ^= 1; std::fs::write(metadata_path, metadata_bytes)?; - let err = harness - .try_load_local(&ctx) - .await - .err() - .expect("should fail"); + let err = harness.try_load_local(&ctx).await.expect_err("should fail"); // get all the stack with all .context, not only the last one let message = format!("{err:#}"); let expected = "failed to load metadata"; diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 87b48e4bee..066f239ff0 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -21,7 +21,7 @@ use crate::{ }; use super::{ - mgr::{GetTenantError, TenantsMap}, + mgr::{GetTenantError, TenantSlotError, TenantSlotUpsertError, TenantsMap}, remote_timeline_client::{FAILED_REMOTE_OP_RETRIES, FAILED_UPLOAD_WARN_THRESHOLD}, span, timeline::delete::DeleteTimelineFlow, @@ -33,12 +33,21 @@ pub(crate) enum DeleteTenantError { #[error("GetTenant {0}")] Get(#[from] GetTenantError), + #[error("Tenant not attached")] + NotAttached, + #[error("Invalid state {0}. Expected Active or Broken")] InvalidState(TenantState), #[error("Tenant deletion is already in progress")] AlreadyInProgress, + #[error("Tenant map slot error {0}")] + SlotError(#[from] TenantSlotError), + + #[error("Tenant map slot upsert error {0}")] + SlotUpsertError(#[from] TenantSlotUpsertError), + #[error("Timeline {0}")] Timeline(#[from] DeleteTimelineError), @@ -273,12 +282,12 @@ impl DeleteTenantFlow { pub(crate) async fn run( conf: &'static PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, - tenant_id: TenantId, + tenants: &'static std::sync::RwLock, + tenant: Arc, ) -> Result<(), DeleteTenantError> { span::debug_assert_current_span_has_tenant_id(); - let (tenant, mut guard) = Self::prepare(tenants, tenant_id).await?; + let mut guard = Self::prepare(&tenant).await?; if let Err(e) = Self::run_inner(&mut guard, conf, remote_storage.as_ref(), &tenant).await { tenant.set_broken(format!("{e:#}")).await; @@ -378,7 +387,7 @@ impl DeleteTenantFlow { guard: DeletionGuard, tenant: &Arc, preload: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, init_order: Option, ctx: &RequestContext, ) -> Result<(), DeleteTenantError> { @@ -405,15 +414,8 @@ impl DeleteTenantFlow { } async fn prepare( - tenants: &tokio::sync::RwLock, - tenant_id: TenantId, - ) -> Result<(Arc, tokio::sync::OwnedMutexGuard), DeleteTenantError> { - let m = tenants.read().await; - - let tenant = m - .get(&tenant_id) - .ok_or(GetTenantError::NotFound(tenant_id))?; - + tenant: &Arc, + ) -> Result, DeleteTenantError> { // FIXME: unsure about active only. Our init jobs may not be cancellable properly, // so at least for now allow deletions only for active tenants. TODO recheck // Broken and Stopping is needed for retries. @@ -447,14 +449,14 @@ impl DeleteTenantFlow { ))); } - Ok((Arc::clone(tenant), guard)) + Ok(guard) } fn schedule_background( guard: OwnedMutexGuard, conf: &'static PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, tenant: Arc, ) { let tenant_id = tenant.tenant_id; @@ -487,7 +489,7 @@ impl DeleteTenantFlow { mut guard: OwnedMutexGuard, conf: &PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, tenant: &Arc, ) -> Result<(), DeleteTenantError> { // Tree sort timelines, schedule delete for them. Mention retries from the console side. @@ -535,10 +537,18 @@ impl DeleteTenantFlow { .await .context("cleanup_remaining_fs_traces")?; - let mut locked = tenants.write().await; - if locked.remove(&tenant.tenant_id).is_none() { - warn!("Tenant got removed from tenants map during deletion"); - }; + { + let mut locked = tenants.write().unwrap(); + if locked.remove(&tenant.tenant_id).is_none() { + warn!("Tenant got removed from tenants map during deletion"); + }; + + // FIXME: we should not be modifying this from outside of mgr.rs. + // This will go away when we simplify deletion (https://github.com/neondatabase/neon/issues/5080) + crate::metrics::TENANT_MANAGER + .tenant_slots + .set(locked.len() as u64); + } *guard = Self::Finished; diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 33fdc76f8d..576bcea2ce 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -3,13 +3,16 @@ use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; use rand::{distributions::Alphanumeric, Rng}; -use std::collections::{hash_map, HashMap}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::ops::Deref; use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::fs; +use utils::timeout::{timeout_cancellable, TimeoutCancellableError}; use anyhow::Context; use once_cell::sync::Lazy; -use tokio::sync::RwLock; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::*; @@ -23,6 +26,7 @@ use crate::control_plane_client::{ ControlPlaneClient, ControlPlaneGenerationsApi, RetryForeverError, }; use crate::deletion_queue::DeletionQueueClient; +use crate::metrics::TENANT_MANAGER as METRICS; use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; use crate::tenant::delete::DeleteTenantFlow; @@ -47,10 +51,22 @@ use super::TenantSharedResources; /// that way we avoid having to carefully switch a tenant's ingestion etc on and off during /// its lifetime, and we can preserve some important safety invariants like `Tenant` always /// having a properly acquired generation (Secondary doesn't need a generation) -#[derive(Clone)] pub(crate) enum TenantSlot { Attached(Arc), Secondary, + /// In this state, other administrative operations acting on the TenantId should + /// block, or return a retry indicator equivalent to HTTP 503. + InProgress(utils::completion::Barrier), +} + +impl std::fmt::Debug for TenantSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Attached(tenant) => write!(f, "Attached({})", tenant.current_state()), + Self::Secondary => write!(f, "Secondary"), + Self::InProgress(_) => write!(f, "InProgress"), + } + } } impl TenantSlot { @@ -59,14 +75,7 @@ impl TenantSlot { match self { Self::Attached(t) => Some(t), Self::Secondary => None, - } - } - - /// Consume self and return the `Tenant` that was in this slot if attached, else None - fn into_attached(self) -> Option> { - match self { - Self::Attached(t) => Some(t), - Self::Secondary => None, + Self::InProgress(_) => None, } } } @@ -77,7 +86,7 @@ pub(crate) enum TenantsMap { /// [`init_tenant_mgr`] is not done yet. Initializing, /// [`init_tenant_mgr`] is done, all on-disk tenants have been loaded. - /// New tenants can be added using [`tenant_map_insert`]. + /// New tenants can be added using [`tenant_map_acquire_slot`]. Open(HashMap), /// The pageserver has entered shutdown mode via [`shutdown_all_tenants`]. /// Existing tenants are still accessible, but no new tenants can be created. @@ -97,19 +106,17 @@ impl TenantsMap { } } - /// Get the contents of the map at this tenant ID, even if it is in secondary state. - pub(crate) fn get_slot(&self, tenant_id: &TenantId) -> Option<&TenantSlot> { + pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option { match self { TenantsMap::Initializing => None, - TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.get(tenant_id), + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.remove(tenant_id), } } - pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option> { + + pub(crate) fn len(&self) -> usize { match self { - TenantsMap::Initializing => None, - TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => { - m.remove(tenant_id).and_then(TenantSlot::into_attached) - } + TenantsMap::Initializing => 0, + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.len(), } } } @@ -147,7 +154,8 @@ async fn safe_rename_tenant_dir(path: impl AsRef) -> std::io::Result> = Lazy::new(|| RwLock::new(TenantsMap::Initializing)); +static TENANTS: Lazy> = + Lazy::new(|| std::sync::RwLock::new(TenantsMap::Initializing)); /// Create a directory, including parents. This does no fsyncs and makes /// no guarantees about the persistence of the resulting metadata: for @@ -456,8 +464,9 @@ pub async fn init_tenant_mgr( info!("Processed {} local tenants at startup", tenants.len()); - let mut tenants_map = TENANTS.write().await; + let mut tenants_map = TENANTS.write().unwrap(); assert!(matches!(&*tenants_map, &TenantsMap::Initializing)); + METRICS.tenant_slots.set(tenants.len() as u64); *tenants_map = TenantsMap::Open(tenants); Ok(()) } @@ -472,7 +481,7 @@ pub(crate) fn tenant_spawn( resources: TenantSharedResources, location_conf: AttachedTenantConf, init_order: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { @@ -533,12 +542,12 @@ pub(crate) async fn shutdown_all_tenants() { shutdown_all_tenants0(&TENANTS).await } -async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { +async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { use utils::completion; - // Prevent new tenants from being created. - let tenants_to_shut_down = { - let mut m = tenants.write().await; + // Atomically, 1. extract the list of tenants to shut down and 2. prevent creation of new tenants. + let (in_progress_ops, tenants_to_shut_down) = { + let mut m = tenants.write().unwrap(); match &mut *m { TenantsMap::Initializing => { *m = TenantsMap::ShuttingDown(HashMap::default()); @@ -546,9 +555,28 @@ async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { return; } TenantsMap::Open(tenants) => { - let tenants_clone = tenants.clone(); - *m = TenantsMap::ShuttingDown(std::mem::take(tenants)); - tenants_clone + let mut shutdown_state = HashMap::new(); + let mut in_progress_ops = Vec::new(); + let mut tenants_to_shut_down = Vec::new(); + + for (k, v) in tenants.drain() { + match v { + TenantSlot::Attached(t) => { + tenants_to_shut_down.push(t.clone()); + shutdown_state.insert(k, TenantSlot::Attached(t)); + } + TenantSlot::Secondary => { + shutdown_state.insert(k, TenantSlot::Secondary); + } + TenantSlot::InProgress(notify) => { + // InProgress tenants are not visible in TenantsMap::ShuttingDown: we will + // wait for their notifications to fire in this function. + in_progress_ops.push(notify); + } + } + } + *m = TenantsMap::ShuttingDown(shutdown_state); + (in_progress_ops, tenants_to_shut_down) } TenantsMap::ShuttingDown(_) => { // TODO: it is possible that detach and shutdown happen at the same time. as a @@ -559,25 +587,31 @@ async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { } }; + info!( + "Waiting for {} InProgress tenants and {} Attached tenants to shut down", + in_progress_ops.len(), + tenants_to_shut_down.len() + ); + + for barrier in in_progress_ops { + barrier.wait().await; + } + + info!( + "InProgress tenants shut down, waiting for {} Attached tenants to shut down", + tenants_to_shut_down.len() + ); let started_at = std::time::Instant::now(); let mut join_set = JoinSet::new(); - for (tenant_id, tenant) in tenants_to_shut_down { + for tenant in tenants_to_shut_down { + let tenant_id = tenant.get_tenant_id(); join_set.spawn( async move { let freeze_and_flush = true; let res = { let (_guard, shutdown_progress) = completion::channel(); - match tenant { - TenantSlot::Attached(t) => { - t.shutdown(shutdown_progress, freeze_and_flush).await - } - TenantSlot::Secondary => { - // TODO: once secondary mode downloads are implemented, - // ensure they have all stopped before we reach this point. - Ok(()) - } - } + tenant.shutdown(shutdown_progress, freeze_and_flush).await }; if let Err(other_progress) = res { @@ -649,42 +683,35 @@ pub(crate) async fn create_tenant( resources: TenantSharedResources, ctx: &RequestContext, ) -> Result, TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let location_conf = LocationConf::attached_single(tenant_conf, generation); + let location_conf = LocationConf::attached_single(tenant_conf, generation); - // We're holding the tenants lock in write mode while doing local IO. - // If this section ever becomes contentious, introduce a new `TenantState::Creating` - // and do the work in that state. - super::create_tenant_files(conf, &location_conf, &tenant_id).await?; + let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + let tenant_path = super::create_tenant_files(conf, &location_conf, &tenant_id).await?; - // TODO: tenant directory remains on disk if we bail out from here on. - // See https://github.com/neondatabase/neon/issues/4233 + let created_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Create, + ctx, + )?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 - let tenant_path = conf.tenant_path(&tenant_id); + let created_tenant_id = created_tenant.tenant_id(); + if tenant_id != created_tenant_id { + return Err(TenantMapInsertError::Other(anyhow::anyhow!( + "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {created_tenant_id})", + ))); + } - let created_tenant = tenant_spawn( - conf, - tenant_id, - &tenant_path, - resources, - AttachedTenantConf::try_from(location_conf)?, - None, - &TENANTS, - SpawnMode::Create, - ctx, - )?; - // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. - // See https://github.com/neondatabase/neon/issues/4233 + slot_guard.upsert(TenantSlot::Attached(created_tenant.clone()))?; - let crated_tenant_id = created_tenant.tenant_id(); - anyhow::ensure!( - tenant_id == crated_tenant_id, - "loaded created tenant has unexpected tenant id \ - (expect {tenant_id} != actual {crated_tenant_id})", - ); - Ok(created_tenant) - }) - .await + Ok(created_tenant) } #[derive(Debug, thiserror::Error)] @@ -701,7 +728,7 @@ pub(crate) async fn set_new_tenant_config( tenant_id: TenantId, ) -> Result<(), SetNewTenantConfigError> { info!("configuring tenant {tenant_id}"); - let tenant = get_tenant(tenant_id, true).await?; + let tenant = get_tenant(tenant_id, true)?; // This is a legacy API that only operates on attached tenants: the preferred // API to use is the location_config/ endpoint, which lets the caller provide @@ -727,32 +754,49 @@ pub(crate) async fn upsert_location( ) -> Result<(), anyhow::Error> { info!("configuring tenant location {tenant_id} to state {new_location_config:?}"); - let mut existing_tenant = match get_tenant(tenant_id, false).await { - Ok(t) => Some(t), - Err(GetTenantError::NotFound(_)) => None, - Err(e) => anyhow::bail!(e), - }; + // Special case fast-path for updates to Tenant: if our upsert is only updating configuration, + // then we do not need to set the slot to InProgress, we can just call into the + // existng tenant. + { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Write)?; + match (&new_location_config.mode, peek_slot) { + (LocationMode::Attached(attach_conf), Some(TenantSlot::Attached(tenant))) => { + if attach_conf.generation == tenant.generation { + // A transition from Attached to Attached in the same generation, we may + // take our fast path and just provide the updated configuration + // to the tenant. + tenant.set_new_location_config(AttachedTenantConf::try_from( + new_location_config, + )?); - // If we need to shut down a Tenant, do that first - let shutdown_tenant = match (&new_location_config.mode, &existing_tenant) { - (LocationMode::Secondary(_), Some(t)) => Some(t), - (LocationMode::Attached(attach_conf), Some(t)) => { - if attach_conf.generation != t.generation { - Some(t) - } else { - None + // Persist the new config in the background, to avoid holding up any + // locks while we do so. + // TODO + + return Ok(()); + } else { + // Different generations, fall through to general case + } + } + _ => { + // Not an Attached->Attached transition, fall through to general case } } - _ => None, - }; + } - // TODO: currently we risk concurrent operations interfering with the tenant - // while we await shutdown, but we also should not hold the TenantsMap lock - // across the whole operation. Before we start using this function in production, - // a follow-on change will revise how concurrency is handled in TenantsMap. - // (https://github.com/neondatabase/neon/issues/5378) + // General case for upserts to TenantsMap, excluding the case above: we will substitute an + // InProgress value to the slot while we make whatever changes are required. The state for + // the tenant is inaccessible to the outside world while we are doing this, but that is sensible: + // the state is ill-defined while we're in transition. Transitions are async, but fast: we do + // not do significant I/O, and shutdowns should be prompt via cancellation tokens. + let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::Any)?; - if let Some(tenant) = shutdown_tenant { + if let Some(TenantSlot::Attached(tenant)) = slot_guard.get_old_value() { + // The case where we keep a Tenant alive was covered above in the special case + // for Attached->Attached transitions in the same generation. By this point, + // if we see an attached tenant we know it will be discarded and should be + // shut down. let (_guard, progress) = utils::completion::channel(); match tenant.get_attach_mode() { @@ -774,82 +818,62 @@ pub(crate) async fn upsert_location( barrier.wait().await; } } - existing_tenant = None; + slot_guard.drop_old_value().expect("We just shut it down"); } - if let Some(tenant) = existing_tenant { - // Update the existing tenant - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; - tenant.set_new_location_config(AttachedTenantConf::try_from(new_location_config)?); - } else { - // Upsert a fresh TenantSlot into TenantsMap. Do it within the map write lock, - // and re-check that the state of anything we are replacing is as expected. - tenant_map_upsert_slot(tenant_id, |old_value| async move { - if let Some(TenantSlot::Attached(t)) = old_value { - if !matches!(t.current_state(), TenantState::Stopping { .. }) { - anyhow::bail!("Tenant state changed during location configuration update"); - } - } + let tenant_path = conf.tenant_path(&tenant_id); + let new_slot = match &new_location_config.mode { + LocationMode::Secondary(_) => { let tenant_path = conf.tenant_path(&tenant_id); + // Directory doesn't need to be fsync'd because if we crash it can + // safely be recreated next time this tenant location is configured. + unsafe_create_dir_all(&tenant_path) + .await + .with_context(|| format!("Creating {tenant_path}"))?; - let new_slot = match &new_location_config.mode { - LocationMode::Secondary(_) => { - // Directory doesn't need to be fsync'd because if we crash it can - // safely be recreated next time this tenant location is configured. - unsafe_create_dir_all(&tenant_path) - .await - .with_context(|| format!("Creating {tenant_path}"))?; + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; + TenantSlot::Secondary + } + LocationMode::Attached(_attach_config) => { + let timelines_path = conf.timelines_path(&tenant_id); - TenantSlot::Secondary - } - LocationMode::Attached(_attach_config) => { - // FIXME: should avoid doing this disk I/O inside the TenantsMap lock, - // we have the same problem in load_tenant/attach_tenant. Probably - // need a lock in TenantSlot to fix this. - let timelines_path = conf.timelines_path(&tenant_id); + // Directory doesn't need to be fsync'd because we do not depend on + // it to exist after crashes: it may be recreated when tenant is + // re-attached, see https://github.com/neondatabase/neon/issues/5550 + unsafe_create_dir_all(&timelines_path) + .await + .with_context(|| format!("Creating {timelines_path}"))?; - // Directory doesn't need to be fsync'd because we do not depend on - // it to exist after crashes: it may be recreated when tenant is - // re-attached, see https://github.com/neondatabase/neon/issues/5550 - unsafe_create_dir_all(&timelines_path) - .await - .with_context(|| format!("Creating {timelines_path}"))?; + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; + let tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + TenantSharedResources { + broker_client, + remote_storage, + deletion_queue_client, + }, + AttachedTenantConf::try_from(new_location_config)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + )?; - let tenant = tenant_spawn( - conf, - tenant_id, - &tenant_path, - TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client, - }, - AttachedTenantConf::try_from(new_location_config)?, - None, - &TENANTS, - SpawnMode::Normal, - ctx, - )?; + TenantSlot::Attached(tenant) + } + }; - TenantSlot::Attached(tenant) - } - }; + slot_guard.upsert(new_slot)?; - Ok(new_slot) - }) - .await?; - } Ok(()) } @@ -864,34 +888,167 @@ pub(crate) enum GetTenantError { /// is a stuck error state #[error("Tenant is broken: {0}")] Broken(String), + + // Initializing or shutting down: cannot authoritatively say whether we have this tenant + #[error("Tenant map is not available: {0}")] + MapState(#[from] TenantMapError), } /// Gets the tenant from the in-memory data, erroring if it's absent or is not fitting to the query. /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. /// /// This method is cancel-safe. -pub(crate) async fn get_tenant( +pub(crate) fn get_tenant( tenant_id: TenantId, active_only: bool, ) -> Result, GetTenantError> { - let m = TENANTS.read().await; - let tenant = m - .get(&tenant_id) - .ok_or(GetTenantError::NotFound(tenant_id))?; + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read)?; - match tenant.current_state() { - TenantState::Broken { - reason, - backtrace: _, - } if active_only => Err(GetTenantError::Broken(reason)), - TenantState::Active => Ok(Arc::clone(tenant)), - _ => { - if active_only { - Err(GetTenantError::NotActive(tenant_id)) - } else { - Ok(Arc::clone(tenant)) + match peek_slot { + Some(TenantSlot::Attached(tenant)) => match tenant.current_state() { + TenantState::Broken { + reason, + backtrace: _, + } if active_only => Err(GetTenantError::Broken(reason)), + TenantState::Active => Ok(Arc::clone(tenant)), + _ => { + if active_only { + Err(GetTenantError::NotActive(tenant_id)) + } else { + Ok(Arc::clone(tenant)) + } + } + }, + Some(TenantSlot::InProgress(_)) => Err(GetTenantError::NotActive(tenant_id)), + None | Some(TenantSlot::Secondary) => Err(GetTenantError::NotFound(tenant_id)), + } +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum GetActiveTenantError { + /// We may time out either while TenantSlot is InProgress, or while the Tenant + /// is in a non-Active state + #[error( + "Timed out waiting {wait_time:?} for tenant active state. Latest state: {latest_state:?}" + )] + WaitForActiveTimeout { + latest_state: Option, + wait_time: Duration, + }, + + /// The TenantSlot is absent, or in secondary mode + #[error(transparent)] + NotFound(#[from] GetTenantError), + + /// Cancellation token fired while we were waiting + #[error("cancelled")] + Cancelled, + + /// Tenant exists, but is in a state that cannot become active (e.g. Stopping, Broken) + #[error("will not become active. Current state: {0}")] + WillNotBecomeActive(TenantState), +} + +/// Get a [`Tenant`] in its active state. If the tenant_id is currently in [`TenantSlot::InProgress`] +/// state, then wait for up to `timeout`. If the [`Tenant`] is not currently in [`TenantState::Active`], +/// then wait for up to `timeout` (minus however long we waited for the slot). +pub(crate) async fn get_active_tenant_with_timeout( + tenant_id: TenantId, + timeout: Duration, + cancel: &CancellationToken, +) -> Result, GetActiveTenantError> { + enum WaitFor { + Barrier(utils::completion::Barrier), + Tenant(Arc), + } + + let wait_start = Instant::now(); + let deadline = wait_start + timeout; + + let wait_for = { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read) + .map_err(GetTenantError::MapState)?; + match peek_slot { + Some(TenantSlot::Attached(tenant)) => { + match tenant.current_state() { + TenantState::Active => { + // Fast path: we don't need to do any async waiting. + return Ok(tenant.clone()); + } + _ => WaitFor::Tenant(tenant.clone()), + } + } + Some(TenantSlot::Secondary) => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotActive( + tenant_id, + ))) + } + Some(TenantSlot::InProgress(barrier)) => WaitFor::Barrier(barrier.clone()), + None => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotFound( + tenant_id, + ))) } } + }; + + let tenant = match wait_for { + WaitFor::Barrier(barrier) => { + tracing::debug!("Waiting for tenant InProgress state to pass..."); + timeout_cancellable( + deadline.duration_since(Instant::now()), + cancel, + barrier.wait(), + ) + .await + .map_err(|e| match e { + TimeoutCancellableError::Timeout => GetActiveTenantError::WaitForActiveTimeout { + latest_state: None, + wait_time: wait_start.elapsed(), + }, + TimeoutCancellableError::Cancelled => GetActiveTenantError::Cancelled, + })?; + { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read) + .map_err(GetTenantError::MapState)?; + match peek_slot { + Some(TenantSlot::Attached(tenant)) => tenant.clone(), + _ => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotActive( + tenant_id, + ))) + } + } + } + } + WaitFor::Tenant(tenant) => tenant, + }; + + tracing::debug!("Waiting for tenant to enter active state..."); + match timeout_cancellable( + deadline.duration_since(Instant::now()), + cancel, + tenant.wait_to_become_active(), + ) + .await + { + Ok(Ok(())) => Ok(tenant), + Ok(Err(e)) => Err(e), + Err(TimeoutCancellableError::Timeout) => { + let latest_state = tenant.current_state(); + if latest_state == TenantState::Active { + Ok(tenant) + } else { + Err(GetActiveTenantError::WaitForActiveTimeout { + latest_state: Some(latest_state), + wait_time: timeout, + }) + } + } + Err(TimeoutCancellableError::Cancelled) => Err(GetActiveTenantError::Cancelled), } } @@ -900,7 +1057,33 @@ pub(crate) async fn delete_tenant( remote_storage: Option, tenant_id: TenantId, ) -> Result<(), DeleteTenantError> { - DeleteTenantFlow::run(conf, remote_storage, &TENANTS, tenant_id).await + // We acquire a SlotGuard during this function to protect against concurrent + // changes while the ::prepare phase of DeleteTenantFlow executes, but then + // have to return the Tenant to the map while the background deletion runs. + // + // TODO: refactor deletion to happen outside the lifetime of a Tenant. + // Currently, deletion requires a reference to the tenants map in order to + // keep the Tenant in the map until deletion is complete, and then remove + // it at the end. + // + // See https://github.com/neondatabase/neon/issues/5080 + + let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustExist)?; + + // unwrap is safe because we used MustExist mode when acquiring + let tenant = match slot_guard.get_old_value().as_ref().unwrap() { + TenantSlot::Attached(tenant) => tenant.clone(), + _ => { + // Express "not attached" as equivalent to "not found" + return Err(DeleteTenantError::NotAttached); + } + }; + + let result = DeleteTenantFlow::run(conf, remote_storage, &TENANTS, tenant).await; + + // The Tenant goes back into the map in Stopping state, it will eventually be removed by DeleteTenantFLow + slot_guard.revert(); + result } #[derive(Debug, thiserror::Error)] @@ -917,18 +1100,20 @@ pub(crate) async fn delete_timeline( timeline_id: TimelineId, _ctx: &RequestContext, ) -> Result<(), DeleteTimelineError> { - let tenant = get_tenant(tenant_id, true).await?; + let tenant = get_tenant(tenant_id, true)?; DeleteTimelineFlow::run(&tenant, timeline_id, false).await?; Ok(()) } #[derive(Debug, thiserror::Error)] pub(crate) enum TenantStateError { - #[error("Tenant {0} not found")] - NotFound(TenantId), #[error("Tenant {0} is stopping")] IsStopping(TenantId), #[error(transparent)] + SlotError(#[from] TenantSlotError), + #[error(transparent)] + SlotUpsertError(#[from] TenantSlotUpsertError), + #[error(transparent)] Other(#[from] anyhow::Error), } @@ -967,7 +1152,7 @@ pub(crate) async fn detach_tenant( async fn detach_tenant0( conf: &'static PageServerConf, - tenants: &tokio::sync::RwLock, + tenants: &std::sync::RwLock, tenant_id: TenantId, detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, @@ -988,7 +1173,12 @@ async fn detach_tenant0( // Ignored tenants are not present in memory and will bail the removal from memory operation. // Before returning the error, check for ignored tenant removal case — we only need to clean its local files then. - if detach_ignored && matches!(removal_result, Err(TenantStateError::NotFound(_))) { + if detach_ignored + && matches!( + removal_result, + Err(TenantStateError::SlotError(TenantSlotError::NotFound(_))) + ) + { let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); if tenant_ignore_mark.exists() { info!("Detaching an ignored tenant"); @@ -1011,31 +1201,44 @@ pub(crate) async fn load_tenant( deletion_queue_client: DeletionQueueClient, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let tenant_path = conf.tenant_path(&tenant_id); - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); - if tenant_ignore_mark.exists() { - std::fs::remove_file(&tenant_ignore_mark) - .with_context(|| format!("Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading"))?; - } + let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + let tenant_path = conf.tenant_path(&tenant_id); - let resources = TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client - }; + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); + if tenant_ignore_mark.exists() { + std::fs::remove_file(&tenant_ignore_mark).with_context(|| { + format!( + "Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading" + ) + })?; + } - let mut location_conf = Tenant::load_tenant_config(conf, &tenant_id).map_err( TenantMapInsertError::Other)?; - location_conf.attach_in_generation(generation); - Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + let resources = TenantSharedResources { + broker_client, + remote_storage, + deletion_queue_client, + }; - let new_tenant = tenant_spawn(conf, tenant_id, &tenant_path, resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx) - .with_context(|| { - format!("Failed to schedule tenant processing in path {tenant_path:?}") - })?; + let mut location_conf = + Tenant::load_tenant_config(conf, &tenant_id).map_err(TenantMapInsertError::Other)?; + location_conf.attach_in_generation(generation); - Ok(new_tenant) - }).await?; + Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + + let new_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + ) + .with_context(|| format!("Failed to schedule tenant processing in path {tenant_path:?}"))?; + + slot_guard.upsert(TenantSlot::Attached(new_tenant))?; Ok(()) } @@ -1048,7 +1251,7 @@ pub(crate) async fn ignore_tenant( async fn ignore_tenant0( conf: &'static PageServerConf, - tenants: &tokio::sync::RwLock, + tenants: &std::sync::RwLock, tenant_id: TenantId, ) -> Result<(), TenantStateError> { remove_tenant_from_memory(tenants, tenant_id, async { @@ -1076,7 +1279,7 @@ pub(crate) enum TenantMapListError { /// Get list of tenants, for the mgmt API /// pub(crate) async fn list_tenants() -> Result, TenantMapListError> { - let tenants = TENANTS.read().await; + let tenants = TENANTS.read().unwrap(); let m = match &*tenants { TenantsMap::Initializing => return Err(TenantMapListError::Initializing), TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m, @@ -1085,6 +1288,7 @@ pub(crate) async fn list_tenants() -> Result, Tenan .filter_map(|(id, tenant)| match tenant { TenantSlot::Attached(tenant) => Some((*id, tenant.current_state())), TenantSlot::Secondary => None, + TenantSlot::InProgress(_) => None, }) .collect()) } @@ -1101,101 +1305,436 @@ pub(crate) async fn attach_tenant( resources: TenantSharedResources, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let location_conf = LocationConf::attached_single(tenant_conf, generation); - let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; - // TODO: tenant directory remains on disk if we bail out from here on. - // See https://github.com/neondatabase/neon/issues/4233 + let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + let location_conf = LocationConf::attached_single(tenant_conf, generation); + let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; + // TODO: tenant directory remains on disk if we bail out from here on. + // See https://github.com/neondatabase/neon/issues/4233 - let attached_tenant = tenant_spawn(conf, tenant_id, &tenant_dir, - resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx)?; - // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. - // See https://github.com/neondatabase/neon/issues/4233 + let attached_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_dir, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + )?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 - let attached_tenant_id = attached_tenant.tenant_id(); - anyhow::ensure!( - tenant_id == attached_tenant_id, + let attached_tenant_id = attached_tenant.tenant_id(); + if tenant_id != attached_tenant_id { + return Err(TenantMapInsertError::Other(anyhow::anyhow!( "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {attached_tenant_id})", - ); - Ok(attached_tenant) - }) - .await?; + ))); + } + + slot_guard.upsert(TenantSlot::Attached(attached_tenant))?; Ok(()) } #[derive(Debug, thiserror::Error)] pub(crate) enum TenantMapInsertError { - #[error("tenant map is still initializing")] - StillInitializing, - #[error("tenant map is shutting down")] - ShuttingDown, - #[error("tenant {0} already exists, state: {1:?}")] - TenantAlreadyExists(TenantId, TenantState), - #[error("tenant {0} already exists in secondary state")] - TenantExistsSecondary(TenantId), + #[error(transparent)] + SlotError(#[from] TenantSlotError), + #[error(transparent)] + SlotUpsertError(#[from] TenantSlotUpsertError), #[error(transparent)] Other(#[from] anyhow::Error), } -/// Give the given closure access to the tenants map entry for the given `tenant_id`, iff that -/// entry is vacant. The closure is responsible for creating the tenant object and inserting -/// it into the tenants map through the vacnt entry that it receives as argument. +/// Superset of TenantMapError: issues that can occur when acquiring a slot +/// for a particular tenant ID. +#[derive(Debug, thiserror::Error)] +pub enum TenantSlotError { + /// When acquiring a slot with the expectation that the tenant already exists. + #[error("Tenant {0} not found")] + NotFound(TenantId), + + /// When acquiring a slot with the expectation that the tenant does not already exist. + #[error("tenant {0} already exists, state: {1:?}")] + AlreadyExists(TenantId, TenantState), + + #[error("tenant {0} already exists in but is not attached")] + Conflict(TenantId), + + // Tried to read a slot that is currently being mutated by another administrative + // operation. + #[error("tenant has a state change in progress, try again later")] + InProgress, + + #[error(transparent)] + MapState(#[from] TenantMapError), +} + +/// Superset of TenantMapError: issues that can occur when using a SlotGuard +/// to insert a new value. +#[derive(Debug, thiserror::Error)] +pub enum TenantSlotUpsertError { + /// An error where the slot is in an unexpected state, indicating a code bug + #[error("Internal error updating Tenant")] + InternalError(Cow<'static, str>), + + #[error(transparent)] + MapState(#[from] TenantMapError), +} + +#[derive(Debug)] +enum TenantSlotDropError { + /// It is only legal to drop a TenantSlot if its contents are fully shut down + NotShutdown, +} + +/// Errors that can happen any time we are walking the tenant map to try and acquire +/// the TenantSlot for a particular tenant. +#[derive(Debug, thiserror::Error)] +pub enum TenantMapError { + // Tried to read while initializing + #[error("tenant map is still initializing")] + StillInitializing, + + // Tried to read while shutting down + #[error("tenant map is shutting down")] + ShuttingDown, +} + +/// Guards a particular tenant_id's content in the TenantsMap. While this +/// structure exists, the TenantsMap will contain a [`TenantSlot::InProgress`] +/// for this tenant, which acts as a marker for any operations targeting +/// this tenant to retry later, or wait for the InProgress state to end. /// -/// NB: the closure should return quickly because the current implementation of tenants map -/// serializes access through an `RwLock`. -async fn tenant_map_insert( +/// This structure enforces the important invariant that we do not have overlapping +/// tasks that will try use local storage for a the same tenant ID: we enforce that +/// the previous contents of a slot have been shut down before the slot can be +/// left empty or used for something else +/// +/// Holders of a SlotGuard should explicitly dispose of it, using either `upsert` +/// to provide a new value, or `revert` to put the slot back into its initial +/// state. If the SlotGuard is dropped without calling either of these, then +/// we will leave the slot empty if our `old_value` is already shut down, else +/// we will replace the slot with `old_value` (equivalent to doing a revert). +/// +/// The `old_value` may be dropped before the SlotGuard is dropped, by calling +/// `drop_old_value`. It is an error to call this without shutting down +/// the conents of `old_value`. +pub struct SlotGuard { tenant_id: TenantId, - insert_fn: F, -) -> Result, TenantMapInsertError> -where - F: FnOnce() -> R, - R: std::future::Future>>, -{ - let mut guard = TENANTS.write().await; - let m = match &mut *guard { - TenantsMap::Initializing => return Err(TenantMapInsertError::StillInitializing), - TenantsMap::ShuttingDown(_) => return Err(TenantMapInsertError::ShuttingDown), - TenantsMap::Open(m) => m, - }; - match m.entry(tenant_id) { - hash_map::Entry::Occupied(e) => match e.get() { - TenantSlot::Attached(t) => Err(TenantMapInsertError::TenantAlreadyExists( - tenant_id, - t.current_state(), - )), - TenantSlot::Secondary => Err(TenantMapInsertError::TenantExistsSecondary(tenant_id)), - }, - hash_map::Entry::Vacant(v) => match insert_fn().await { - Ok(tenant) => { - v.insert(TenantSlot::Attached(tenant.clone())); - Ok(tenant) + old_value: Option, + upserted: bool, + + /// [`TenantSlot::InProgress`] carries the corresponding Barrier: it will + /// release any waiters as soon as this SlotGuard is dropped. + _completion: utils::completion::Completion, +} + +unsafe impl Send for SlotGuard {} +unsafe impl Sync for SlotGuard {} + +impl SlotGuard { + fn new( + tenant_id: TenantId, + old_value: Option, + completion: utils::completion::Completion, + ) -> Self { + Self { + tenant_id, + old_value, + upserted: false, + _completion: completion, + } + } + + /// Take any value that was present in the slot before we acquired ownership + /// of it: in state transitions, this will be the old state. + fn get_old_value(&mut self) -> &Option { + &self.old_value + } + + /// Emplace a new value in the slot. This consumes the guard, and after + /// returning, the slot is no longer protected from concurrent changes. + fn upsert(mut self, new_value: TenantSlot) -> Result<(), TenantSlotUpsertError> { + if !self.old_value_is_shutdown() { + // This is a bug: callers should never try to drop an old value without + // shutting it down + return Err(TenantSlotUpsertError::InternalError( + "Old TenantSlot value not shut down".into(), + )); + } + + let replaced = { + let mut locked = TENANTS.write().unwrap(); + + if let TenantSlot::InProgress(_) = new_value { + // It is never expected to try and upsert InProgress via this path: it should + // only be written via the tenant_map_acquire_slot path. If we hit this it's a bug. + return Err(TenantSlotUpsertError::InternalError( + "Attempt to upsert an InProgress state".into(), + )); } - Err(e) => Err(TenantMapInsertError::Other(e)), - }, + + let m = match &mut *locked { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing.into()), + TenantsMap::ShuttingDown(_) => { + return Err(TenantMapError::ShuttingDown.into()); + } + TenantsMap::Open(m) => m, + }; + + let replaced = m.insert(self.tenant_id, new_value); + self.upserted = true; + + METRICS.tenant_slots.set(m.len() as u64); + + replaced + }; + + // Sanity check: on an upsert we should always be replacing an InProgress marker + match replaced { + Some(TenantSlot::InProgress(_)) => { + // Expected case: we find our InProgress in the map: nothing should have + // replaced it because the code that acquires slots will not grant another + // one for the same TenantId. + Ok(()) + } + None => { + METRICS.unexpected_errors.inc(); + error!( + tenant_id = %self.tenant_id, + "Missing InProgress marker during tenant upsert, this is a bug." + ); + Err(TenantSlotUpsertError::InternalError( + "Missing InProgress marker during tenant upsert".into(), + )) + } + Some(slot) => { + METRICS.unexpected_errors.inc(); + error!(tenant_id=%self.tenant_id, "Unexpected contents of TenantSlot during upsert, this is a bug. Contents: {:?}", slot); + Err(TenantSlotUpsertError::InternalError( + "Unexpected contents of TenantSlot".into(), + )) + } + } + } + + /// Replace the InProgress slot with whatever was in the guard when we started + fn revert(mut self) { + if let Some(value) = self.old_value.take() { + match self.upsert(value) { + Err(TenantSlotUpsertError::InternalError(_)) => { + // We already logged the error, nothing else we can do. + } + Err(TenantSlotUpsertError::MapState(_)) => { + // If the map is shutting down, we need not replace anything + } + Ok(()) => {} + } + } + } + + /// We may never drop our old value until it is cleanly shut down: otherwise we might leave + /// rogue background tasks that would write to the local tenant directory that this guard + /// is responsible for protecting + fn old_value_is_shutdown(&self) -> bool { + match self.old_value.as_ref() { + Some(TenantSlot::Attached(tenant)) => { + // TODO: PR #5711 will add a gate that enables properly checking that + // shutdown completed. + matches!( + tenant.current_state(), + TenantState::Stopping { .. } | TenantState::Broken { .. } + ) + } + Some(TenantSlot::Secondary) => { + // TODO: when adding secondary mode tenants, this will check for shutdown + // in the same way that we do for `Tenant` above + true + } + Some(TenantSlot::InProgress(_)) => { + // A SlotGuard cannot be constructed for a slot that was already InProgress + unreachable!() + } + None => true, + } + } + + /// The guard holder is done with the old value of the slot: they are obliged to already + /// shut it down before we reach this point. + fn drop_old_value(&mut self) -> Result<(), TenantSlotDropError> { + if !self.old_value_is_shutdown() { + Err(TenantSlotDropError::NotShutdown) + } else { + self.old_value.take(); + Ok(()) + } } } -async fn tenant_map_upsert_slot<'a, F, R>( - tenant_id: TenantId, - upsert_fn: F, -) -> Result<(), TenantMapInsertError> -where - F: FnOnce(Option) -> R, - R: std::future::Future>, -{ - let mut guard = TENANTS.write().await; - let m = match &mut *guard { - TenantsMap::Initializing => return Err(TenantMapInsertError::StillInitializing), - TenantsMap::ShuttingDown(_) => return Err(TenantMapInsertError::ShuttingDown), +impl Drop for SlotGuard { + fn drop(&mut self) { + if self.upserted { + return; + } + // Our old value is already shutdown, or it never existed: it is safe + // for us to fully release the TenantSlot back into an empty state + + let mut locked = TENANTS.write().unwrap(); + + let m = match &mut *locked { + TenantsMap::Initializing => { + // There is no map, this should never happen. + return; + } + TenantsMap::ShuttingDown(_) => { + // When we transition to shutdown, InProgress elements are removed + // from the map, so we do not need to clean up our Inprogress marker. + // See [`shutdown_all_tenants0`] + return; + } + TenantsMap::Open(m) => m, + }; + + use std::collections::hash_map::Entry; + match m.entry(self.tenant_id) { + Entry::Occupied(mut entry) => { + if !matches!(entry.get(), TenantSlot::InProgress(_)) { + METRICS.unexpected_errors.inc(); + error!(tenant_id=%self.tenant_id, "Unexpected contents of TenantSlot during drop, this is a bug. Contents: {:?}", entry.get()); + } + + if self.old_value_is_shutdown() { + entry.remove(); + } else { + entry.insert(self.old_value.take().unwrap()); + } + } + Entry::Vacant(_) => { + METRICS.unexpected_errors.inc(); + error!( + tenant_id = %self.tenant_id, + "Missing InProgress marker during SlotGuard drop, this is a bug." + ); + } + } + + METRICS.tenant_slots.set(m.len() as u64); + } +} + +enum TenantSlotPeekMode { + /// In Read mode, peek will be permitted to see the slots even if the pageserver is shutting down + Read, + /// In Write mode, trying to peek at a slot while the pageserver is shutting down is an error + Write, +} + +fn tenant_map_peek_slot<'a>( + tenants: &'a std::sync::RwLockReadGuard<'a, TenantsMap>, + tenant_id: &TenantId, + mode: TenantSlotPeekMode, +) -> Result, TenantMapError> { + let m = match tenants.deref() { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing), + TenantsMap::ShuttingDown(m) => match mode { + TenantSlotPeekMode::Read => m, + TenantSlotPeekMode::Write => { + return Err(TenantMapError::ShuttingDown); + } + }, TenantsMap::Open(m) => m, }; - match upsert_fn(m.remove(&tenant_id)).await { - Ok(upsert_val) => { - m.insert(tenant_id, upsert_val); - Ok(()) + Ok(m.get(tenant_id)) +} + +enum TenantSlotAcquireMode { + /// Acquire the slot irrespective of current state, or whether it already exists + Any, + /// Return an error if trying to acquire a slot and it doesn't already exist + MustExist, + /// Return an error if trying to acquire a slot and it already exists + MustNotExist, +} + +fn tenant_map_acquire_slot( + tenant_id: &TenantId, + mode: TenantSlotAcquireMode, +) -> Result { + tenant_map_acquire_slot_impl(tenant_id, &TENANTS, mode) +} + +fn tenant_map_acquire_slot_impl( + tenant_id: &TenantId, + tenants: &std::sync::RwLock, + mode: TenantSlotAcquireMode, +) -> Result { + use TenantSlotAcquireMode::*; + METRICS.tenant_slot_writes.inc(); + + let mut locked = tenants.write().unwrap(); + let span = tracing::info_span!("acquire_slot", %tenant_id); + let _guard = span.enter(); + + let m = match &mut *locked { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing.into()), + TenantsMap::ShuttingDown(_) => return Err(TenantMapError::ShuttingDown.into()), + TenantsMap::Open(m) => m, + }; + + use std::collections::hash_map::Entry; + let entry = m.entry(*tenant_id); + match entry { + Entry::Vacant(v) => match mode { + MustExist => { + tracing::debug!("Vacant && MustExist: return NotFound"); + Err(TenantSlotError::NotFound(*tenant_id)) + } + _ => { + let (completion, barrier) = utils::completion::channel(); + v.insert(TenantSlot::InProgress(barrier)); + tracing::debug!("Vacant, inserted InProgress"); + Ok(SlotGuard::new(*tenant_id, None, completion)) + } + }, + Entry::Occupied(mut o) => { + // Apply mode-driven checks + match (o.get(), mode) { + (TenantSlot::InProgress(_), _) => { + tracing::debug!("Occupied, failing for InProgress"); + Err(TenantSlotError::InProgress) + } + (slot, MustNotExist) => match slot { + TenantSlot::Attached(tenant) => { + tracing::debug!("Attached && MustNotExist, return AlreadyExists"); + Err(TenantSlotError::AlreadyExists( + *tenant_id, + tenant.current_state(), + )) + } + _ => { + // FIXME: the AlreadyExists error assumes that we have a Tenant + // to get the state from + tracing::debug!("Occupied & MustNotExist, return AlreadyExists"); + Err(TenantSlotError::AlreadyExists( + *tenant_id, + TenantState::Broken { + reason: "Present but not attached".to_string(), + backtrace: "".to_string(), + }, + )) + } + }, + _ => { + // Happy case: the slot was not in any state that violated our mode + let (completion, barrier) = utils::completion::channel(); + let old_value = o.insert(TenantSlot::InProgress(barrier)); + tracing::debug!("Occupied, replaced with InProgress"); + Ok(SlotGuard::new(*tenant_id, Some(old_value), completion)) + } + } } - Err(e) => Err(TenantMapInsertError::Other(e)), } } @@ -1204,7 +1743,7 @@ where /// If the cleanup fails, tenant will stay in memory in [`TenantState::Broken`] state, and another removal /// operation would be needed to remove it. async fn remove_tenant_from_memory( - tenants: &tokio::sync::RwLock, + tenants: &std::sync::RwLock, tenant_id: TenantId, tenant_cleanup: F, ) -> Result @@ -1213,20 +1752,14 @@ where { use utils::completion; - // It's important to keep the tenant in memory after the final cleanup, to avoid cleanup races. - // The exclusive lock here ensures we don't miss the tenant state updates before trying another removal. - // tenant-wde cleanup operations may take some time (removing the entire tenant directory), we want to - // avoid holding the lock for the entire process. - let tenant = { - match tenants - .write() - .await - .get_slot(&tenant_id) - .ok_or(TenantStateError::NotFound(tenant_id))? - { - TenantSlot::Attached(t) => Some(t.clone()), - TenantSlot::Secondary => None, - } + let mut slot_guard = + tenant_map_acquire_slot_impl(&tenant_id, tenants, TenantSlotAcquireMode::MustExist)?; + + // The SlotGuard allows us to manipulate the Tenant object without fear of some + // concurrent API request doing something else for the same tenant ID. + let attached_tenant = match slot_guard.get_old_value() { + Some(TenantSlot::Attached(t)) => Some(t), + _ => None, }; // allow pageserver shutdown to await for our completion @@ -1234,7 +1767,7 @@ where // If the tenant was attached, shut it down gracefully. For secondary // locations this part is not necessary - match tenant { + match &attached_tenant { Some(attached_tenant) => { // whenever we remove a tenant from memory, we don't want to flush and wait for upload let freeze_and_flush = false; @@ -1246,6 +1779,7 @@ where Err(_other) => { // if pageserver shutdown or other detach/ignore is already ongoing, we don't want to // wait for it but return an error right away because these are distinct requests. + slot_guard.revert(); return Err(TenantStateError::IsStopping(tenant_id)); } } @@ -1260,23 +1794,21 @@ where .with_context(|| format!("Failed to run cleanup for tenant {tenant_id}")) { Ok(hook_value) => { - let mut tenants_accessor = tenants.write().await; - if tenants_accessor.remove(&tenant_id).is_none() { - warn!("Tenant {tenant_id} got removed from memory before operation finished"); - } + // Success: drop the old TenantSlot::Attached. + slot_guard + .drop_old_value() + .expect("We just called shutdown"); + Ok(hook_value) } Err(e) => { - let tenants_accessor = tenants.read().await; - match tenants_accessor.get(&tenant_id) { - Some(tenant) => { - tenant.set_broken(e.to_string()).await; - } - None => { - warn!("Tenant {tenant_id} got removed from memory"); - return Err(TenantStateError::NotFound(tenant_id)); - } + // If we had a Tenant, set it to Broken and put it back in the TenantsMap + if let Some(attached_tenant) = attached_tenant { + attached_tenant.set_broken(e.to_string()).await; } + // Leave the broken tenant in the map + slot_guard.revert(); + Err(TenantStateError::Other(e)) } } @@ -1293,7 +1825,7 @@ pub(crate) async fn immediate_gc( gc_req: TimelineGcRequest, ctx: &RequestContext, ) -> Result>, ApiError> { - let guard = TENANTS.read().await; + let guard = TENANTS.read().unwrap(); let tenant = guard .get(&tenant_id) .map(Arc::clone) @@ -1346,14 +1878,12 @@ mod tests { use super::{super::harness::TenantHarness, TenantsMap}; - #[tokio::test(start_paused = true)] - async fn shutdown_joins_remove_tenant_from_memory() { - // the test is a bit ugly with the lockstep together with spawned tasks. the aim is to make - // sure `shutdown_all_tenants0` per-tenant processing joins in any active - // remove_tenant_from_memory calls, which is enforced by making the operation last until - // we've ran `shutdown_all_tenants0` for a long time. + #[tokio::test] + async fn shutdown_awaits_in_progress_tenant() { + // Test that if an InProgress tenant is in the map during shutdown, the shutdown will gracefully + // wait for it to complete before proceeding. - let (t, _ctx) = TenantHarness::create("shutdown_joins_detach") + let (t, _ctx) = TenantHarness::create("shutdown_awaits_in_progress_tenant") .unwrap() .load() .await; @@ -1366,13 +1896,14 @@ mod tests { let _e = info_span!("testing", tenant_id = %id).entered(); let tenants = HashMap::from([(id, TenantSlot::Attached(t.clone()))]); - let tenants = Arc::new(tokio::sync::RwLock::new(TenantsMap::Open(tenants))); + let tenants = Arc::new(std::sync::RwLock::new(TenantsMap::Open(tenants))); + + // Invoke remove_tenant_from_memory with a cleanup hook that blocks until we manually + // permit it to proceed: that will stick the tenant in InProgress let (until_cleanup_completed, can_complete_cleanup) = utils::completion::channel(); let (until_cleanup_started, cleanup_started) = utils::completion::channel(); - - // start a "detaching operation", which will take a while, until can_complete_cleanup - let cleanup_task = { + let mut remove_tenant_from_memory_task = { let jh = tokio::spawn({ let tenants = tenants.clone(); async move { @@ -1391,12 +1922,6 @@ mod tests { jh }; - let mut cleanup_progress = std::pin::pin!(t - .shutdown(utils::completion::Barrier::default(), false) - .await - .unwrap_err() - .wait()); - let mut shutdown_task = { let (until_shutdown_started, shutdown_started) = utils::completion::channel(); @@ -1409,37 +1934,17 @@ mod tests { shutdown_task }; - // if the joining in is removed from shutdown_all_tenants0, the shutdown_task should always - // get to complete within timeout and fail the test. it is expected to continue awaiting - // until completion or SIGKILL during normal shutdown. - // - // the timeout is long to cover anything that shutdown_task could be doing, but it is - // handled instantly because we use tokio's time pausing in this test. 100s is much more than - // what we get from systemd on shutdown (10s). - let long_time = std::time::Duration::from_secs(100); + let long_time = std::time::Duration::from_secs(15); tokio::select! { - _ = &mut shutdown_task => unreachable!("shutdown must continue, until_cleanup_completed is not dropped"), - _ = &mut cleanup_progress => unreachable!("cleanup progress must continue, until_cleanup_completed is not dropped"), + _ = &mut shutdown_task => unreachable!("shutdown should block on remove_tenant_from_memory completing"), + _ = &mut remove_tenant_from_memory_task => unreachable!("remove_tenant_from_memory_task should not complete until explicitly unblocked"), _ = tokio::time::sleep(long_time) => {}, } - // allow the remove_tenant_from_memory and thus eventually the shutdown to continue drop(until_cleanup_completed); - let (je, ()) = tokio::join!(shutdown_task, cleanup_progress); - je.expect("Tenant::shutdown shutdown not have panicked"); - cleanup_task - .await - .expect("no panicking") - .expect("remove_tenant_from_memory failed"); - - futures::future::poll_immediate( - t.shutdown(utils::completion::Barrier::default(), false) - .await - .unwrap_err() - .wait(), - ) - .await - .expect("the stopping progress must still be complete"); + // Now that we allow it to proceed, shutdown should complete immediately + remove_tenant_from_memory_task.await.unwrap().unwrap(); + shutdown_task.await.unwrap(); } } diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 52c53a5c3b..e3aad22e40 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -341,20 +341,7 @@ impl Timeline { // Make one of the tenant's timelines draw the short straw and run the calculation. // The others wait until the calculation is done so that they take into account the // imitated accesses that the winner made. - // - // It is critical we are responsive to cancellation here. Otherwise, we deadlock with - // tenant deletion (holds TENANTS in read mode) any other task that attempts to - // acquire TENANTS in write mode before we here call get_tenant. - // See https://github.com/neondatabase/neon/issues/5284. - let res = tokio::select! { - _ = cancel.cancelled() => { - return ControlFlow::Break(()); - } - res = crate::tenant::mgr::get_tenant(self.tenant_id, true) => { - res - } - }; - let tenant = match res { + let tenant = match crate::tenant::mgr::get_tenant(self.tenant_id, true) { Ok(t) => t, Err(_) => { return ControlFlow::Break(()); diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 740c05dcea..d00bf8c4d8 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -626,6 +626,8 @@ class NeonEnvBuilder: sk.stop(immediate=True) for pageserver in self.env.pageservers: + pageserver.assert_no_metric_errors() + pageserver.stop(immediate=True) if self.env.attachment_service is not None: @@ -1784,6 +1786,21 @@ class NeonPageserver(PgProtocol): assert not errors + def assert_no_metric_errors(self): + """ + Certain metrics should _always_ be zero: they track conditions that indicate a bug. + """ + if not self.running: + log.info(f"Skipping metrics check on pageserver {self.id}, it is not running") + return + + for metric in [ + "pageserver_tenant_manager_unexpected_errors_total", + "pageserver_deletion_queue_unexpected_errors_total", + ]: + value = self.http_client().get_metric_value(metric) + assert value == 0, f"Nonzero {metric} == {value}" + def log_contains(self, pattern: str) -> Optional[str]: """Check that the pageserver log contains a line that matches the given regex""" logfile = open(os.path.join(self.workdir, "pageserver.log"), "r") diff --git a/test_runner/regress/test_neon_cli.py b/test_runner/regress/test_neon_cli.py index 1b3984583a..de18ea0e6b 100644 --- a/test_runner/regress/test_neon_cli.py +++ b/test_runner/regress/test_neon_cli.py @@ -134,6 +134,9 @@ def test_cli_start_stop(neon_env_builder: NeonEnvBuilder): env.neon_cli.pageserver_stop(env.pageserver.id) env.neon_cli.safekeeper_stop() + # Keep NeonEnv state up to date, it usually owns starting/stopping services + env.pageserver.running = False + # Default start res = env.neon_cli.raw_cli(["start"]) res.check_returncode() @@ -155,6 +158,10 @@ def test_cli_start_stop_multi(neon_env_builder: NeonEnvBuilder): env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID) env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 1) + # Keep NeonEnv state up to date, it usually owns starting/stopping services + env.pageservers[0].running = False + env.pageservers[1].running = False + # Addressing a nonexistent ID throws with pytest.raises(RuntimeError): env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 100) diff --git a/test_runner/regress/test_pageserver_restart.py b/test_runner/regress/test_pageserver_restart.py index 4c51155236..fa1b131537 100644 --- a/test_runner/regress/test_pageserver_restart.py +++ b/test_runner/regress/test_pageserver_restart.py @@ -20,6 +20,8 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool) endpoint = env.endpoints.create_start("main") pageserver_http = env.pageserver.http_client() + assert pageserver_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + pg_conn = endpoint.connect() cur = pg_conn.cursor() @@ -52,6 +54,9 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool) env.pageserver.stop() env.pageserver.start() + # We reloaded our tenant + assert pageserver_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + cur.execute("SELECT count(*) FROM foo") assert cur.fetchone() == (100000,) diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index ae77197088..6e35890922 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -63,6 +63,9 @@ def test_tenant_delete_smoke( conf=MANY_SMALL_LAYERS_TENANT_CONFIG, ) + # Default tenant and the one we created + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 2 + # create two timelines one being the parent of another parent = None for timeline in ["first", "second"]: @@ -88,7 +91,9 @@ def test_tenant_delete_smoke( iterations = poll_for_remote_storage_iterations(remote_storage_kind) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 2 tenant_delete_wait_completed(ps_http, tenant_id, iterations) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 1 tenant_path = env.pageserver.tenant_dir(tenant_id) assert not tenant_path.exists() @@ -104,6 +109,9 @@ def test_tenant_delete_smoke( ), ) + # Deletion updates the tenant count: the one default tenant remains + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + class Check(enum.Enum): RETRY_WITHOUT_RESTART = enum.auto() diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 11e8a80e1d..03e78dda2b 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -26,6 +26,16 @@ from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until from prometheus_client.samples import Sample +# In tests that overlap endpoint activity with tenant attach/detach, there are +# a variety of warnings that the page service may emit when it cannot acquire +# an active tenant to serve a request +PERMIT_PAGE_SERVICE_ERRORS = [ + ".*page_service.*Tenant .* not found", + ".*page_service.*Tenant .* is not active", + ".*page_service.*cancelled", + ".*page_service.*will not become active.*", +] + def do_gc_target( pageserver_http: PageserverHttpClient, tenant_id: TenantId, timeline_id: TimelineId @@ -60,12 +70,7 @@ def test_tenant_reattach( # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: with endpoint.cursor() as cur: @@ -235,10 +240,7 @@ def test_tenant_reattach_while_busy( # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) endpoint = env.endpoints.create_start("main", tenant_id=tenant_id) @@ -259,7 +261,7 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() - env.pageserver.allowed_errors.append(".*NotFound: Tenant .*") + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # first check for non existing tenant tenant_id = TenantId.generate() @@ -271,19 +273,9 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): assert excinfo.value.status_code == 404 - # the error will be printed to the log too - env.pageserver.allowed_errors.append(".*NotFound: tenant *") - # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) - # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -345,12 +337,7 @@ def test_tenant_detach_ignored_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -401,12 +388,7 @@ def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -453,12 +435,7 @@ def test_detach_while_attaching( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # Create table, and insert some rows. Make it big enough that it doesn't fit in # shared_buffers, otherwise the SELECT after restart will just return answer @@ -593,12 +570,7 @@ def test_ignored_tenant_download_missing_layers(neon_env_builder: NeonEnvBuilder tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) data_id = 1 data_secret = "very secret secret" @@ -649,12 +621,7 @@ def test_load_attach_negatives(neon_env_builder: NeonEnvBuilder): tenant_id = env.initial_tenant - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) env.pageserver.allowed_errors.append(".*tenant .*? already exists, state:.*") with pytest.raises( @@ -693,12 +660,7 @@ def test_ignore_while_attaching( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) data_id = 1 data_secret = "very secret secret" From b09a8517059a74499c1fdfe37f9c2842a7cbb229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 6 Nov 2023 16:16:55 +0100 Subject: [PATCH 10/63] Make azure blob storage not do extra metadata requests (#5777) Load the metadata from the returned `GetBlobResponse` and avoid downloading it via a separate request. As it turns out, the SDK does return the metadata: https://github.com/Azure/azure-sdk-for-rust/issues/1439 . This PR will reduce the number of requests to Azure caused by downloads. Fixes #5571 --- libs/remote_storage/src/azure_blob.rs | 43 +++++++-------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index f0521d27c5..ae08e9b171 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -1,21 +1,18 @@ //! Azure Blob Storage wrapper +use std::collections::HashMap; use std::env; use std::num::NonZeroU32; use std::sync::Arc; -use std::{borrow::Cow, collections::HashMap, io::Cursor}; +use std::{borrow::Cow, io::Cursor}; use super::REMOTE_STORAGE_PREFIX_SEPARATOR; use anyhow::Result; use azure_core::request_options::{MaxResults, Metadata, Range}; -use azure_core::Header; use azure_identity::DefaultAzureCredential; use azure_storage::StorageCredentials; use azure_storage_blobs::prelude::ClientBuilder; -use azure_storage_blobs::{ - blob::operations::GetBlobBuilder, - prelude::{BlobClient, ContainerClient}, -}; +use azure_storage_blobs::{blob::operations::GetBlobBuilder, prelude::ContainerClient}; use futures_util::StreamExt; use http_types::StatusCode; use tokio::io::AsyncRead; @@ -112,16 +109,19 @@ impl AzureBlobStorage { async fn download_for_builder( &self, - metadata: StorageMetadata, builder: GetBlobBuilder, ) -> Result { let mut response = builder.into_stream(); + let mut metadata = HashMap::new(); // TODO give proper streaming response instead of buffering into RAM // https://github.com/neondatabase/neon/issues/5563 let mut buf = Vec::new(); while let Some(part) = response.next().await { let part = part.map_err(to_download_error)?; + if let Some(blob_meta) = part.blob.metadata { + metadata.extend(blob_meta.iter().map(|(k, v)| (k.to_owned(), v.to_owned()))); + } let data = part .data .collect() @@ -131,28 +131,9 @@ impl AzureBlobStorage { } Ok(Download { download_stream: Box::pin(Cursor::new(buf)), - metadata: Some(metadata), + metadata: Some(StorageMetadata(metadata)), }) } - // TODO get rid of this function once we have metadata included in the response - // https://github.com/Azure/azure-sdk-for-rust/issues/1439 - async fn get_metadata( - &self, - blob_client: &BlobClient, - ) -> Result { - let builder = blob_client.get_metadata(); - - let response = builder.into_future().await.map_err(to_download_error)?; - let mut map = HashMap::new(); - - for md in response.metadata.iter() { - map.insert( - md.name().as_str().to_string(), - md.value().as_str().to_string(), - ); - } - Ok(StorageMetadata(map)) - } async fn permit(&self, kind: RequestKind) -> tokio::sync::SemaphorePermit<'_> { self.concurrency_limiter @@ -269,11 +250,9 @@ impl RemoteStorage for AzureBlobStorage { let _permit = self.permit(RequestKind::Get).await; let blob_client = self.client.blob_client(self.relative_path_to_name(from)); - let metadata = self.get_metadata(&blob_client).await?; - let builder = blob_client.get(); - self.download_for_builder(metadata, builder).await + self.download_for_builder(builder).await } async fn download_byte_range( @@ -285,8 +264,6 @@ impl RemoteStorage for AzureBlobStorage { let _permit = self.permit(RequestKind::Get).await; let blob_client = self.client.blob_client(self.relative_path_to_name(from)); - let metadata = self.get_metadata(&blob_client).await?; - let mut builder = blob_client.get(); if let Some(end_exclusive) = end_exclusive { @@ -301,7 +278,7 @@ impl RemoteStorage for AzureBlobStorage { builder = builder.range(Range::new(start_inclusive, end_exclusive)); } - self.download_for_builder(metadata, builder).await + self.download_for_builder(builder).await } async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> { From ad5b02e17518445d390d6dab523d4efd13db2c69 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 6 Nov 2023 17:44:44 +0000 Subject: [PATCH 11/63] proxy: remove unsafe (#5805) ## Problem `unsafe {}` ## Summary of changes `CStr` has a method to parse the bytes up to a null byte, so we don't have to do it ourselves. --- proxy/src/parse.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/proxy/src/parse.rs b/proxy/src/parse.rs index cbd48d91e9..0d03574901 100644 --- a/proxy/src/parse.rs +++ b/proxy/src/parse.rs @@ -3,10 +3,9 @@ use std::ffi::CStr; pub fn split_cstr(bytes: &[u8]) -> Option<(&CStr, &[u8])> { - let pos = bytes.iter().position(|&x| x == 0)?; - let (cstr, other) = bytes.split_at(pos + 1); - // SAFETY: we've already checked that there's a terminator - Some((unsafe { CStr::from_bytes_with_nul_unchecked(cstr) }, other)) + let cstr = CStr::from_bytes_until_nul(bytes).ok()?; + let (_, other) = bytes.split_at(cstr.to_bytes_with_nul().len()); + Some((cstr, other)) } /// See . From bea8efac242e32ad0850153f5199e8d15b5fd91c Mon Sep 17 00:00:00 2001 From: Richy Wang Date: Tue, 7 Nov 2023 16:13:01 +0800 Subject: [PATCH 12/63] Fix comments in 'receive_wal.rs'. (#5807) ## Problem Some comments in 'receive_wal.rs' is not suitable. It may copy from 'send_wal.rs' and leave it unchanged. ## Summary of changes This commit fixes two comments in the code: Changed "/// Unregister walsender." to "/// Unregister walreceiver." Changed "///Scope guard to access slot in WalSenders registry" to "///Scope guard to access slot in WalReceivers registry." --- safekeeper/src/receive_wal.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index 934dda10b2..9ce9b049ba 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -111,7 +111,7 @@ impl WalReceivers { .count() } - /// Unregister walsender. + /// Unregister walreceiver. fn unregister(self: &Arc, id: WalReceiverId) { let mut shared = self.mutex.lock(); shared.slots[id] = None; @@ -138,8 +138,8 @@ pub enum WalReceiverStatus { Streaming, } -/// Scope guard to access slot in WalSenders registry and unregister from it in -/// Drop. +/// Scope guard to access slot in WalReceivers registry and unregister from +/// it in Drop. pub struct WalReceiverGuard { id: WalReceiverId, walreceivers: Arc, From c00651ff9bf7941aa617f6185ee9f25f74555d8a Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 7 Nov 2023 09:06:53 +0000 Subject: [PATCH 13/63] pageserver: start refactoring into TenantManager (#5797) ## Problem See: https://github.com/neondatabase/neon/issues/5796 ## Summary of changes Completing the refactor is quite verbose and can be done in stages: each interface that is currently called directly from a top-level mgr.rs function can be moved into TenantManager once the relevant subsystems have access to it. Landing the initial change to create of TenantManager is useful because it enables new code to use it without having to be altered later, and sets us up to incrementally fix the existing code to use an explicit Arc instead of relying on the static TENANTS. --- pageserver/src/bin/pageserver.rs | 4 +- pageserver/src/http/routes.rs | 29 ++-- pageserver/src/tenant/mgr.rs | 258 ++++++++++++++++--------------- 3 files changed, 153 insertions(+), 138 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index df2db25ec6..93ae43a14b 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -410,7 +410,7 @@ fn start_pageserver( // Scan the local 'tenants/' directory and start loading the tenants let deletion_queue_client = deletion_queue.new_client(); - BACKGROUND_RUNTIME.block_on(mgr::init_tenant_mgr( + let tenant_manager = BACKGROUND_RUNTIME.block_on(mgr::init_tenant_mgr( conf, TenantSharedResources { broker_client: broker_client.clone(), @@ -420,6 +420,7 @@ fn start_pageserver( order, shutdown_pageserver.clone(), ))?; + let tenant_manager = Arc::new(tenant_manager); BACKGROUND_RUNTIME.spawn({ let init_done_rx = init_done_rx; @@ -548,6 +549,7 @@ fn start_pageserver( let router_state = Arc::new( http::routes::State::new( conf, + tenant_manager, http_auth.clone(), remote_storage.clone(), broker_client.clone(), diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 820b8eae46..a0643bc580 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -35,8 +35,8 @@ use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; use crate::tenant::config::{LocationConf, TenantConfOpt}; use crate::tenant::mgr::{ - GetTenantError, SetNewTenantConfigError, TenantMapError, TenantMapInsertError, TenantSlotError, - TenantSlotUpsertError, TenantStateError, + GetTenantError, SetNewTenantConfigError, TenantManager, TenantMapError, TenantMapInsertError, + TenantSlotError, TenantSlotUpsertError, TenantStateError, }; use crate::tenant::size::ModelInputs; use crate::tenant::storage_layer::LayerAccessStatsReset; @@ -63,6 +63,7 @@ use super::models::ConfigureFailpointsRequest; pub struct State { conf: &'static PageServerConf, + tenant_manager: Arc, auth: Option>, allowlist_routes: Vec, remote_storage: Option, @@ -74,6 +75,7 @@ pub struct State { impl State { pub fn new( conf: &'static PageServerConf, + tenant_manager: Arc, auth: Option>, remote_storage: Option, broker_client: storage_broker::BrokerClientChannel, @@ -86,6 +88,7 @@ impl State { .collect::>(); Ok(Self { conf, + tenant_manager, auth, allowlist_routes, remote_storage, @@ -1140,20 +1143,14 @@ async fn put_tenant_location_config_handler( let location_conf = LocationConf::try_from(&request_data.config).map_err(ApiError::BadRequest)?; - mgr::upsert_location( - state.conf, - tenant_id, - location_conf, - state.broker_client.clone(), - state.remote_storage.clone(), - state.deletion_queue_client.clone(), - &ctx, - ) - .await - // TODO: badrequest assumes the caller was asking for something unreasonable, but in - // principle we might have hit something like concurrent API calls to the same tenant, - // which is not a 400 but a 409. - .map_err(ApiError::BadRequest)?; + state + .tenant_manager + .upsert_location(tenant_id, location_conf, &ctx) + .await + // TODO: badrequest assumes the caller was asking for something unreasonable, but in + // principle we might have hit something like concurrent API calls to the same tenant, + // which is not a 400 but a 409. + .map_err(ApiError::BadRequest)?; json_response(StatusCode::OK, ()) } diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 576bcea2ce..664181e40d 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -200,6 +200,22 @@ async fn unsafe_create_dir_all(path: &Utf8PathBuf) -> std::io::Result<()> { Ok(()) } +/// The TenantManager is responsible for storing and mutating the collection of all tenants +/// that this pageserver process has state for. Every Tenant and SecondaryTenant instance +/// lives inside the TenantManager. +/// +/// The most important role of the TenantManager is to prevent conflicts: e.g. trying to attach +/// the same tenant twice concurrently, or trying to configure the same tenant into secondary +/// and attached modes concurrently. +pub struct TenantManager { + conf: &'static PageServerConf, + // TODO: currently this is a &'static pointing to TENANTs. When we finish refactoring + // out of that static variable, the TenantManager can own this. + // See https://github.com/neondatabase/neon/issues/5796 + tenants: &'static std::sync::RwLock, + resources: TenantSharedResources, +} + fn emergency_generations( tenant_confs: &HashMap>, ) -> HashMap { @@ -366,7 +382,7 @@ pub async fn init_tenant_mgr( resources: TenantSharedResources, init_order: InitializationOrder, cancel: CancellationToken, -) -> anyhow::Result<()> { +) -> anyhow::Result { let mut tenants = HashMap::new(); let ctx = RequestContext::todo_child(TaskKind::Startup, DownloadBehavior::Warn); @@ -468,7 +484,12 @@ pub async fn init_tenant_mgr( assert!(matches!(&*tenants_map, &TenantsMap::Initializing)); METRICS.tenant_slots.set(tenants.len() as u64); *tenants_map = TenantsMap::Open(tenants); - Ok(()) + + Ok(TenantManager { + conf, + tenants: &TENANTS, + resources, + }) } /// Wrapper for Tenant::spawn that checks invariants before running, and inserts @@ -742,139 +763,134 @@ pub(crate) async fn set_new_tenant_config( Ok(()) } -#[instrument(skip_all, fields(%tenant_id))] -pub(crate) async fn upsert_location( - conf: &'static PageServerConf, - tenant_id: TenantId, - new_location_config: LocationConf, - broker_client: storage_broker::BrokerClientChannel, - remote_storage: Option, - deletion_queue_client: DeletionQueueClient, - ctx: &RequestContext, -) -> Result<(), anyhow::Error> { - info!("configuring tenant location {tenant_id} to state {new_location_config:?}"); +impl TenantManager { + #[instrument(skip_all, fields(%tenant_id))] + pub(crate) async fn upsert_location( + &self, + tenant_id: TenantId, + new_location_config: LocationConf, + ctx: &RequestContext, + ) -> Result<(), anyhow::Error> { + info!("configuring tenant location {tenant_id} to state {new_location_config:?}"); - // Special case fast-path for updates to Tenant: if our upsert is only updating configuration, - // then we do not need to set the slot to InProgress, we can just call into the - // existng tenant. - { - let locked = TENANTS.read().unwrap(); - let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Write)?; - match (&new_location_config.mode, peek_slot) { - (LocationMode::Attached(attach_conf), Some(TenantSlot::Attached(tenant))) => { - if attach_conf.generation == tenant.generation { - // A transition from Attached to Attached in the same generation, we may - // take our fast path and just provide the updated configuration - // to the tenant. - tenant.set_new_location_config(AttachedTenantConf::try_from( - new_location_config, - )?); + // Special case fast-path for updates to Tenant: if our upsert is only updating configuration, + // then we do not need to set the slot to InProgress, we can just call into the + // existng tenant. + { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Write)?; + match (&new_location_config.mode, peek_slot) { + (LocationMode::Attached(attach_conf), Some(TenantSlot::Attached(tenant))) => { + if attach_conf.generation == tenant.generation { + // A transition from Attached to Attached in the same generation, we may + // take our fast path and just provide the updated configuration + // to the tenant. + tenant.set_new_location_config(AttachedTenantConf::try_from( + new_location_config, + )?); - // Persist the new config in the background, to avoid holding up any - // locks while we do so. - // TODO + // Persist the new config in the background, to avoid holding up any + // locks while we do so. + // TODO - return Ok(()); - } else { - // Different generations, fall through to general case + return Ok(()); + } else { + // Different generations, fall through to general case + } + } + _ => { + // Not an Attached->Attached transition, fall through to general case } } - _ => { - // Not an Attached->Attached transition, fall through to general case - } } - } - // General case for upserts to TenantsMap, excluding the case above: we will substitute an - // InProgress value to the slot while we make whatever changes are required. The state for - // the tenant is inaccessible to the outside world while we are doing this, but that is sensible: - // the state is ill-defined while we're in transition. Transitions are async, but fast: we do - // not do significant I/O, and shutdowns should be prompt via cancellation tokens. - let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::Any)?; + // General case for upserts to TenantsMap, excluding the case above: we will substitute an + // InProgress value to the slot while we make whatever changes are required. The state for + // the tenant is inaccessible to the outside world while we are doing this, but that is sensible: + // the state is ill-defined while we're in transition. Transitions are async, but fast: we do + // not do significant I/O, and shutdowns should be prompt via cancellation tokens. + let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::Any)?; - if let Some(TenantSlot::Attached(tenant)) = slot_guard.get_old_value() { - // The case where we keep a Tenant alive was covered above in the special case - // for Attached->Attached transitions in the same generation. By this point, - // if we see an attached tenant we know it will be discarded and should be - // shut down. - let (_guard, progress) = utils::completion::channel(); + if let Some(TenantSlot::Attached(tenant)) = slot_guard.get_old_value() { + // The case where we keep a Tenant alive was covered above in the special case + // for Attached->Attached transitions in the same generation. By this point, + // if we see an attached tenant we know it will be discarded and should be + // shut down. + let (_guard, progress) = utils::completion::channel(); - match tenant.get_attach_mode() { - AttachmentMode::Single | AttachmentMode::Multi => { - // Before we leave our state as the presumed holder of the latest generation, - // flush any outstanding deletions to reduce the risk of leaking objects. - deletion_queue_client.flush_advisory() + match tenant.get_attach_mode() { + AttachmentMode::Single | AttachmentMode::Multi => { + // Before we leave our state as the presumed holder of the latest generation, + // flush any outstanding deletions to reduce the risk of leaking objects. + self.resources.deletion_queue_client.flush_advisory() + } + AttachmentMode::Stale => { + // If we're stale there's not point trying to flush deletions + } + }; + + info!("Shutting down attached tenant"); + match tenant.shutdown(progress, false).await { + Ok(()) => {} + Err(barrier) => { + info!("Shutdown already in progress, waiting for it to complete"); + barrier.wait().await; + } } - AttachmentMode::Stale => { - // If we're stale there's not point trying to flush deletions + slot_guard.drop_old_value().expect("We just shut it down"); + } + + let tenant_path = self.conf.tenant_path(&tenant_id); + + let new_slot = match &new_location_config.mode { + LocationMode::Secondary(_) => { + let tenant_path = self.conf.tenant_path(&tenant_id); + // Directory doesn't need to be fsync'd because if we crash it can + // safely be recreated next time this tenant location is configured. + unsafe_create_dir_all(&tenant_path) + .await + .with_context(|| format!("Creating {tenant_path}"))?; + + Tenant::persist_tenant_config(self.conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; + + TenantSlot::Secondary + } + LocationMode::Attached(_attach_config) => { + let timelines_path = self.conf.timelines_path(&tenant_id); + + // Directory doesn't need to be fsync'd because we do not depend on + // it to exist after crashes: it may be recreated when tenant is + // re-attached, see https://github.com/neondatabase/neon/issues/5550 + unsafe_create_dir_all(&timelines_path) + .await + .with_context(|| format!("Creating {timelines_path}"))?; + + Tenant::persist_tenant_config(self.conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; + + let tenant = tenant_spawn( + self.conf, + tenant_id, + &tenant_path, + self.resources.clone(), + AttachedTenantConf::try_from(new_location_config)?, + None, + self.tenants, + SpawnMode::Normal, + ctx, + )?; + + TenantSlot::Attached(tenant) } }; - info!("Shutting down attached tenant"); - match tenant.shutdown(progress, false).await { - Ok(()) => {} - Err(barrier) => { - info!("Shutdown already in progress, waiting for it to complete"); - barrier.wait().await; - } - } - slot_guard.drop_old_value().expect("We just shut it down"); + slot_guard.upsert(new_slot)?; + + Ok(()) } - - let tenant_path = conf.tenant_path(&tenant_id); - - let new_slot = match &new_location_config.mode { - LocationMode::Secondary(_) => { - let tenant_path = conf.tenant_path(&tenant_id); - // Directory doesn't need to be fsync'd because if we crash it can - // safely be recreated next time this tenant location is configured. - unsafe_create_dir_all(&tenant_path) - .await - .with_context(|| format!("Creating {tenant_path}"))?; - - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; - - TenantSlot::Secondary - } - LocationMode::Attached(_attach_config) => { - let timelines_path = conf.timelines_path(&tenant_id); - - // Directory doesn't need to be fsync'd because we do not depend on - // it to exist after crashes: it may be recreated when tenant is - // re-attached, see https://github.com/neondatabase/neon/issues/5550 - unsafe_create_dir_all(&timelines_path) - .await - .with_context(|| format!("Creating {timelines_path}"))?; - - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; - - let tenant = tenant_spawn( - conf, - tenant_id, - &tenant_path, - TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client, - }, - AttachedTenantConf::try_from(new_location_config)?, - None, - &TENANTS, - SpawnMode::Normal, - ctx, - )?; - - TenantSlot::Attached(tenant) - } - }; - - slot_guard.upsert(new_slot)?; - - Ok(()) } #[derive(Debug, thiserror::Error)] From a394f49e0dba004884dd1f7c7bc7e5888a3e5fb8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 7 Nov 2023 09:35:45 +0000 Subject: [PATCH 14/63] pageserver: avoid converting an error to anyhow::Error (#5803) This was preventing it getting cleanly converted to a CalculateLogicalSizeError::Cancelled, resulting in "Logical size calculation failed" errors in logs. --- pageserver/src/pgdatadir_mapping.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 88974588d4..aa4d155bcc 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -589,11 +589,7 @@ impl Timeline { let mut total_size: u64 = 0; for (spcnode, dbnode) in dbdir.dbdirs.keys() { - for rel in self - .list_rels(*spcnode, *dbnode, lsn, ctx) - .await - .context("list rels")? - { + for rel in self.list_rels(*spcnode, *dbnode, lsn, ctx).await? { if cancel.is_cancelled() { return Err(CalculateLogicalSizeError::Cancelled); } From 4be6bc7251ebe857fada6c94d8e7dbe31da2f839 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 7 Nov 2023 12:26:25 +0200 Subject: [PATCH 15/63] refactor: remove unnecessary unsafe (#5802) unsafe impls for `Send` and `Sync` should not be added by default. in the case of `SlotGuard` removing them does not cause any issues, as the compiler automatically derives those. This PR adds requirement to document the unsafety (see [clippy::undocumented_unsafe_blocks]) and opportunistically adds `#![deny(unsafe_code)]` to most places where we don't have unsafe code right now. TRPL on Send and Sync: https://doc.rust-lang.org/book/ch16-04-extensible-concurrency-sync-and-send.html [clippy::undocumented_unsafe_blocks]: https://rust-lang.github.io/rust-clippy/master/#/undocumented_unsafe_blocks --- compute_tools/src/lib.rs | 4 ++-- control_plane/src/background_process.rs | 2 +- control_plane/src/lib.rs | 15 +++++++------- libs/compute_api/src/lib.rs | 2 ++ libs/consumption_metrics/src/lib.rs | 4 ++-- libs/metrics/src/lib.rs | 1 + libs/pageserver_api/src/lib.rs | 2 ++ libs/postgres_backend/src/lib.rs | 2 ++ libs/postgres_connection/src/lib.rs | 2 ++ libs/postgres_ffi/src/lib.rs | 2 ++ libs/pq_proto/src/lib.rs | 1 + libs/remote_storage/src/lib.rs | 2 ++ libs/safekeeper_api/src/lib.rs | 2 ++ libs/tenant_size_model/src/lib.rs | 2 ++ libs/tracing-utils/src/lib.rs | 2 ++ libs/utils/src/id.rs | 2 ++ libs/utils/src/lib.rs | 1 + libs/utils/src/shutdown.rs | 3 ++- libs/vm_monitor/src/lib.rs | 2 ++ pageserver/src/lib.rs | 2 ++ pageserver/src/tenant/mgr.rs | 3 --- pageserver/src/walredo.rs | 26 ++++++++++++------------- proxy/src/lib.rs | 2 ++ s3_scrubber/src/lib.rs | 2 ++ safekeeper/src/lib.rs | 1 + 25 files changed, 59 insertions(+), 30 deletions(-) diff --git a/compute_tools/src/lib.rs b/compute_tools/src/lib.rs index 1cd9602089..4e01ffd954 100644 --- a/compute_tools/src/lib.rs +++ b/compute_tools/src/lib.rs @@ -1,7 +1,7 @@ -//! //! Various tools and helpers to handle cluster / compute node (Postgres) //! configuration. -//! +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] pub mod checker; pub mod config; pub mod configurator; diff --git a/control_plane/src/background_process.rs b/control_plane/src/background_process.rs index c0016ece17..26fc08fc8f 100644 --- a/control_plane/src/background_process.rs +++ b/control_plane/src/background_process.rs @@ -262,7 +262,7 @@ where P: Into, { let path: Utf8PathBuf = path.into(); - // SAFETY + // SAFETY: // pre_exec is marked unsafe because it runs between fork and exec. // Why is that dangerous in various ways? // Long answer: https://github.com/rust-lang/rust/issues/39575 diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 7592880402..bb79d36bfc 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,11 +1,10 @@ -// -// Local control plane. -// -// Can start, configure and stop postgres instances running as a local processes. -// -// Intended to be used in integration tests and in CLI tools for -// local installations. -// +//! Local control plane. +//! +//! Can start, configure and stop postgres instances running as a local processes. +//! +//! Intended to be used in integration tests and in CLI tools for +//! local installations. +#![deny(clippy::undocumented_unsafe_blocks)] pub mod attachment_service; mod background_process; diff --git a/libs/compute_api/src/lib.rs b/libs/compute_api/src/lib.rs index b660799ec0..210a52d089 100644 --- a/libs/compute_api/src/lib.rs +++ b/libs/compute_api/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] pub mod requests; pub mod responses; pub mod spec; diff --git a/libs/consumption_metrics/src/lib.rs b/libs/consumption_metrics/src/lib.rs index 9e89327e84..810196aff6 100644 --- a/libs/consumption_metrics/src/lib.rs +++ b/libs/consumption_metrics/src/lib.rs @@ -1,6 +1,6 @@ -//! //! Shared code for consumption metics collection -//! +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use chrono::{DateTime, Utc}; use rand::Rng; use serde::{Deserialize, Serialize}; diff --git a/libs/metrics/src/lib.rs b/libs/metrics/src/lib.rs index 0b3b24c18e..ed375a152f 100644 --- a/libs/metrics/src/lib.rs +++ b/libs/metrics/src/lib.rs @@ -2,6 +2,7 @@ //! make sure that we use the same dep version everywhere. //! Otherwise, we might not see all metrics registered via //! a default registry. +#![deny(clippy::undocumented_unsafe_blocks)] use once_cell::sync::Lazy; use prometheus::core::{AtomicU64, Collector, GenericGauge, GenericGaugeVec}; pub use prometheus::opts; diff --git a/libs/pageserver_api/src/lib.rs b/libs/pageserver_api/src/lib.rs index d844021785..e49f7d00c1 100644 --- a/libs/pageserver_api/src/lib.rs +++ b/libs/pageserver_api/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use const_format::formatcp; /// Public API types diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 455fe7a481..1e25c6ed6a 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -2,6 +2,8 @@ //! To use, create PostgresBackend and run() it, passing the Handler //! implementation determining how to process the queries. Currently its API //! is rather narrow, but we can extend it once required. +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use anyhow::Context; use bytes::Bytes; use futures::pin_mut; diff --git a/libs/postgres_connection/src/lib.rs b/libs/postgres_connection/src/lib.rs index 35344a9168..35cb1a2691 100644 --- a/libs/postgres_connection/src/lib.rs +++ b/libs/postgres_connection/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use anyhow::{bail, Context}; use itertools::Itertools; use std::borrow::Cow; diff --git a/libs/postgres_ffi/src/lib.rs b/libs/postgres_ffi/src/lib.rs index d0009d0dc9..d10ebfe277 100644 --- a/libs/postgres_ffi/src/lib.rs +++ b/libs/postgres_ffi/src/lib.rs @@ -8,6 +8,7 @@ // modules included with the postgres_ffi macro depend on the types of the specific version's // types, and trigger a too eager lint. #![allow(clippy::duplicate_mod)] +#![deny(clippy::undocumented_unsafe_blocks)] use bytes::Bytes; use utils::bin_ser::SerializeError; @@ -20,6 +21,7 @@ macro_rules! postgres_ffi { pub mod bindings { // bindgen generates bindings for a lot of stuff we don't need #![allow(dead_code)] + #![allow(clippy::undocumented_unsafe_blocks)] use serde::{Deserialize, Serialize}; include!(concat!( diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs index 94bc7f60f8..41fc206cd7 100644 --- a/libs/pq_proto/src/lib.rs +++ b/libs/pq_proto/src/lib.rs @@ -1,6 +1,7 @@ //! Postgres protocol messages serialization-deserialization. See //! //! on message formats. +#![deny(clippy::undocumented_unsafe_blocks)] pub mod framed; diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 9b5b340db0..7054610d9e 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -6,6 +6,8 @@ //! * [`s3_bucket`] uses AWS S3 bucket as an external storage //! * [`azure_blob`] allows to use Azure Blob storage as an external storage //! +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] mod azure_blob; mod local_fs; diff --git a/libs/safekeeper_api/src/lib.rs b/libs/safekeeper_api/src/lib.rs index 0a391478da..63c2c51188 100644 --- a/libs/safekeeper_api/src/lib.rs +++ b/libs/safekeeper_api/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use const_format::formatcp; /// Public API types diff --git a/libs/tenant_size_model/src/lib.rs b/libs/tenant_size_model/src/lib.rs index c151e3b42c..a3e12cf0e3 100644 --- a/libs/tenant_size_model/src/lib.rs +++ b/libs/tenant_size_model/src/lib.rs @@ -1,4 +1,6 @@ //! Synthetic size calculation +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] mod calculation; pub mod svg; diff --git a/libs/tracing-utils/src/lib.rs b/libs/tracing-utils/src/lib.rs index de0e2ad799..9cf2495771 100644 --- a/libs/tracing-utils/src/lib.rs +++ b/libs/tracing-utils/src/lib.rs @@ -32,6 +32,8 @@ //! .init(); //! } //! ``` +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use opentelemetry::sdk::Resource; use opentelemetry::KeyValue; diff --git a/libs/utils/src/id.rs b/libs/utils/src/id.rs index eef6a358e6..57dcc27719 100644 --- a/libs/utils/src/id.rs +++ b/libs/utils/src/id.rs @@ -120,6 +120,8 @@ impl Id { chunk[0] = HEX[((b >> 4) & 0xf) as usize]; chunk[1] = HEX[(b & 0xf) as usize]; } + + // SAFETY: vec constructed out of `HEX`, it can only be ascii unsafe { String::from_utf8_unchecked(buf) } } } diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index ff5d774285..bb6c848bf4 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -1,5 +1,6 @@ //! `utils` is intended to be a place to put code that is shared //! between other crates in this repository. +#![deny(clippy::undocumented_unsafe_blocks)] pub mod backoff; diff --git a/libs/utils/src/shutdown.rs b/libs/utils/src/shutdown.rs index 7eba905997..cb5a44d664 100644 --- a/libs/utils/src/shutdown.rs +++ b/libs/utils/src/shutdown.rs @@ -1,6 +1,7 @@ /// Immediately terminate the calling process without calling /// atexit callbacks, C runtime destructors etc. We mainly use /// this to protect coverage data from concurrent writes. -pub fn exit_now(code: u8) { +pub fn exit_now(code: u8) -> ! { + // SAFETY: exiting is safe, the ffi is not safe unsafe { nix::libc::_exit(code as _) }; } diff --git a/libs/vm_monitor/src/lib.rs b/libs/vm_monitor/src/lib.rs index a844f78bd6..89ca91fdd7 100644 --- a/libs/vm_monitor/src/lib.rs +++ b/libs/vm_monitor/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] #![cfg(target_os = "linux")] use anyhow::Context; diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 04c3ae9746..3f74694ef2 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(clippy::undocumented_unsafe_blocks)] + mod auth; pub mod basebackup; pub mod config; diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 664181e40d..818872c093 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -1446,9 +1446,6 @@ pub struct SlotGuard { _completion: utils::completion::Completion, } -unsafe impl Send for SlotGuard {} -unsafe impl Sync for SlotGuard {} - impl SlotGuard { fn new( tenant_id: TenantId, diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index e1d699fcba..38f2c64420 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -596,21 +596,21 @@ trait CloseFileDescriptors: CommandExt { impl CloseFileDescriptors for C { fn close_fds(&mut self) -> &mut Command { + // SAFETY: Code executed inside pre_exec should have async-signal-safety, + // which means it should be safe to execute inside a signal handler. + // The precise meaning depends on platform. See `man signal-safety` + // for the linux definition. + // + // The set_fds_cloexec_threadsafe function is documented to be + // async-signal-safe. + // + // Aside from this function, the rest of the code is re-entrant and + // doesn't make any syscalls. We're just passing constants. + // + // NOTE: It's easy to indirectly cause a malloc or lock a mutex, + // which is not async-signal-safe. Be careful. unsafe { self.pre_exec(move || { - // SAFETY: Code executed inside pre_exec should have async-signal-safety, - // which means it should be safe to execute inside a signal handler. - // The precise meaning depends on platform. See `man signal-safety` - // for the linux definition. - // - // The set_fds_cloexec_threadsafe function is documented to be - // async-signal-safe. - // - // Aside from this function, the rest of the code is re-entrant and - // doesn't make any syscalls. We're just passing constants. - // - // NOTE: It's easy to indirectly cause a malloc or lock a mutex, - // which is not async-signal-safe. Be careful. close_fds::set_fds_cloexec_threadsafe(3, &[]); Ok(()) }) diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index 803b278482..4a95e473f6 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(clippy::undocumented_unsafe_blocks)] + use std::convert::Infallible; use anyhow::{bail, Context}; diff --git a/s3_scrubber/src/lib.rs b/s3_scrubber/src/lib.rs index d892438633..777276a4d1 100644 --- a/s3_scrubber/src/lib.rs +++ b/s3_scrubber/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] pub mod checks; pub mod cloud_admin_api; pub mod garbage; diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index aa70cee591..344b0b14e4 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -1,3 +1,4 @@ +#![deny(clippy::undocumented_unsafe_blocks)] use camino::Utf8PathBuf; use once_cell::sync::Lazy; use remote_storage::RemoteStorageConfig; From 0ac4cf67a61534dcfbaeaea1b35e5dbac50f2bb8 Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Tue, 7 Nov 2023 13:38:02 +0200 Subject: [PATCH 16/63] Use self.tenants instead of TENANTS (#5811) --- pageserver/src/tenant/mgr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 818872c093..27f9d50c54 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -777,7 +777,7 @@ impl TenantManager { // then we do not need to set the slot to InProgress, we can just call into the // existng tenant. { - let locked = TENANTS.read().unwrap(); + let locked = self.tenants.read().unwrap(); let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Write)?; match (&new_location_config.mode, peek_slot) { (LocationMode::Attached(attach_conf), Some(TenantSlot::Attached(tenant))) => { From 0141c957887aa9fe0340459d644d92cb9e6b0627 Mon Sep 17 00:00:00 2001 From: Fernando Luz Date: Tue, 7 Nov 2023 12:13:05 +0000 Subject: [PATCH 17/63] build: Add warning when missing postgres submodule during the build (#5614) I forked the project and in my local repo, I wasn't able to compile the project and in my search, I found the solution in neon forum. After a PR discussion, I made a change in the makefile to alert the missing `git submodules update` step. --------- Signed-off-by: Fernando Luz Co-authored-by: Joonas Koivunen --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 64bbc1677c..89acbe564a 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,10 @@ neon: postgres-headers walproposer-lib # $(POSTGRES_INSTALL_DIR)/build/%/config.status: +@echo "Configuring Postgres $* build" + @test -s $(ROOT_PROJECT_DIR)/vendor/postgres-$*/configure || { \ + echo "\nPostgres submodule not found in $(ROOT_PROJECT_DIR)/vendor/postgres-$*/, execute "; \ + echo "'git submodule update --init --recursive --depth 2 --progress .' in project root.\n"; \ + exit 1; } mkdir -p $(POSTGRES_INSTALL_DIR)/build/$* (cd $(POSTGRES_INSTALL_DIR)/build/$* && \ env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-$*/configure \ From 4cd47b7d4bfcb0483f0b717d977a8cb6064dd979 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 7 Nov 2023 13:45:59 +0000 Subject: [PATCH 18/63] Dockerfile: Set BUILD_TAG for storage services (#5812) ## Problem https://github.com/neondatabase/neon/pull/5576 added `build-tag` reporting to `libmetrics_build_info`, but it's not reported because we didn't set the corresponding env variable in the build process. ## Summary of changes - Add `BUILD_TAG` env var while building services --- .github/workflows/build_and_test.yml | 1 + Dockerfile | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e0f8afc954..babf767fcf 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -723,6 +723,7 @@ jobs: --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} + --build-arg BUILD_TAG=${{ needs.tag.outputs.build-tag }} --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} --destination neondatabase/neon:${{needs.tag.outputs.build-tag}} diff --git a/Dockerfile b/Dockerfile index eb4c4bba25..60de9cfa3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN set -e \ FROM $REPOSITORY/$IMAGE:$TAG AS build WORKDIR /home/nonroot ARG GIT_VERSION=local +ARG BUILD_TAG # Enable https://github.com/paritytech/cachepot to cache Rust crates' compilation results in Docker builds. # Set up cachepot to use an AWS S3 bucket for cache results, to reuse it between `docker build` invocations. @@ -78,9 +79,9 @@ COPY --from=build --chown=neon:neon /home/nonroot/target/release/pg_sni_router COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/pagectl /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper /usr/local/bin -COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_broker /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_broker /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy /usr/local/bin -COPY --from=build --chown=neon:neon /home/nonroot/target/release/neon_local /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/neon_local /usr/local/bin COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/ COPY --from=pg-build /home/nonroot/pg_install/v15 /usr/local/v15/ From 1d68f52b57e91fe19b5a1695d75ab92775be7fd8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 7 Nov 2023 14:25:51 +0000 Subject: [PATCH 19/63] pageserver: move deletion failpoint inside backoff (#5814) ## Problem When enabled, this failpoint would busy-spin in a loop that emits log messages. ## Summary of changes Move the failpoint inside a backoff::exponential block: it will still spam the log, but at much lower rate. --------- Co-authored-by: Joonas Koivunen --- pageserver/src/deletion_queue/deleter.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pageserver/src/deletion_queue/deleter.rs b/pageserver/src/deletion_queue/deleter.rs index 81cb016d49..57421b1547 100644 --- a/pageserver/src/deletion_queue/deleter.rs +++ b/pageserver/src/deletion_queue/deleter.rs @@ -55,21 +55,24 @@ impl Deleter { /// Wrap the remote `delete_objects` with a failpoint async fn remote_delete(&self) -> Result<(), anyhow::Error> { - fail::fail_point!("deletion-queue-before-execute", |_| { - info!("Skipping execution, failpoint set"); - metrics::DELETION_QUEUE - .remote_errors - .with_label_values(&["failpoint"]) - .inc(); - Err(anyhow::anyhow!("failpoint hit")) - }); - // A backoff::retry is used here for two reasons: // - To provide a backoff rather than busy-polling the API on errors // - To absorb transient 429/503 conditions without hitting our error // logging path for issues deleting objects. backoff::retry( - || async { self.remote_storage.delete_objects(&self.accumulator).await }, + || async { + fail::fail_point!("deletion-queue-before-execute", |_| { + info!("Skipping execution, failpoint set"); + + metrics::DELETION_QUEUE + .remote_errors + .with_label_values(&["failpoint"]) + .inc(); + Err(anyhow::anyhow!("failpoint: deletion-queue-before-execute")) + }); + + self.remote_storage.delete_objects(&self.accumulator).await + }, |_| false, 3, 10, From e310533ed353e8e92c8b845f403d79eb991f8a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 7 Nov 2023 15:43:29 +0100 Subject: [PATCH 20/63] Support JWT key reload in pageserver (#5594) ## Problem For quickly rotating JWT secrets, we want to be able to reload the JWT public key file in the pageserver, and also support multiple JWT keys. See #4897. ## Summary of changes * Allow directories for the `auth_validation_public_key_path` config param instead of just files. for the safekeepers, all of their config options also support multiple JWT keys. * For the pageservers, make the JWT public keys easily globally swappable by using the `arc-swap` crate. * Add an endpoint to the pageserver, triggered by a POST to `/v1/reload_auth_validation_keys`, that reloads the JWT public keys from the pre-configured path (for security reasons, you cannot upload any keys yourself). Fixes #4897 --------- Co-authored-by: Heikki Linnakangas Co-authored-by: Joonas Koivunen --- Cargo.lock | 7 ++ Cargo.toml | 1 + libs/utils/Cargo.toml | 1 + libs/utils/src/auth.rs | 76 +++++++++++-- libs/utils/src/http/endpoint.rs | 4 +- pageserver/src/bin/pageserver.rs | 18 ++-- pageserver/src/config.rs | 2 +- pageserver/src/http/openapi_spec.yml | 25 +++++ pageserver/src/http/routes.rs | 38 ++++++- pageserver/src/page_service.rs | 10 +- safekeeper/src/bin/safekeeper.rs | 9 +- safekeeper/src/http/routes.rs | 9 +- safekeeper/src/lib.rs | 7 +- test_runner/fixtures/neon_fixtures.py | 27 ++++- test_runner/fixtures/pageserver/http.py | 4 + test_runner/regress/test_auth.py | 137 ++++++++++++++++++++---- 16 files changed, 313 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7e88688ce..10f13e21fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,6 +170,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "archery" version = "0.5.0" @@ -5951,6 +5957,7 @@ name = "utils" version = "0.1.0" dependencies = [ "anyhow", + "arc-swap", "async-trait", "bincode", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index d992db4858..6be831a2b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ license = "Apache-2.0" ## All dependency versions, used in the project [workspace.dependencies] anyhow = { version = "1.0", features = ["backtrace"] } +arc-swap = "1.6" async-compression = { version = "0.4.0", features = ["tokio", "gzip"] } azure_core = "0.16" azure_identity = "0.16" diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 3f4ef2abeb..ccf6f4f2d7 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +arc-swap.workspace = true sentry.workspace = true async-trait.workspace = true anyhow.workspace = true diff --git a/libs/utils/src/auth.rs b/libs/utils/src/auth.rs index 37299e4e7f..6a26f17115 100644 --- a/libs/utils/src/auth.rs +++ b/libs/utils/src/auth.rs @@ -1,7 +1,8 @@ // For details about authentication see docs/authentication.md +use arc_swap::ArcSwap; use serde; -use std::fs; +use std::{fs, sync::Arc}; use anyhow::Result; use camino::Utf8Path; @@ -44,31 +45,88 @@ impl Claims { } } +pub struct SwappableJwtAuth(ArcSwap); + +impl SwappableJwtAuth { + pub fn new(jwt_auth: JwtAuth) -> Self { + SwappableJwtAuth(ArcSwap::new(Arc::new(jwt_auth))) + } + pub fn swap(&self, jwt_auth: JwtAuth) { + self.0.swap(Arc::new(jwt_auth)); + } + pub fn decode(&self, token: &str) -> Result> { + self.0.load().decode(token) + } +} + +impl std::fmt::Debug for SwappableJwtAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Swappable({:?})", self.0.load()) + } +} + pub struct JwtAuth { - decoding_key: DecodingKey, + decoding_keys: Vec, validation: Validation, } impl JwtAuth { - pub fn new(decoding_key: DecodingKey) -> Self { + pub fn new(decoding_keys: Vec) -> Self { let mut validation = Validation::default(); validation.algorithms = vec![STORAGE_TOKEN_ALGORITHM]; // The default 'required_spec_claims' is 'exp'. But we don't want to require // expiration. validation.required_spec_claims = [].into(); Self { - decoding_key, + decoding_keys, validation, } } pub fn from_key_path(key_path: &Utf8Path) -> Result { - let public_key = fs::read(key_path)?; - Ok(Self::new(DecodingKey::from_ed_pem(&public_key)?)) + let metadata = key_path.metadata()?; + let decoding_keys = if metadata.is_dir() { + let mut keys = Vec::new(); + for entry in fs::read_dir(key_path)? { + let path = entry?.path(); + if !path.is_file() { + // Ignore directories (don't recurse) + continue; + } + let public_key = fs::read(path)?; + keys.push(DecodingKey::from_ed_pem(&public_key)?); + } + keys + } else if metadata.is_file() { + let public_key = fs::read(key_path)?; + vec![DecodingKey::from_ed_pem(&public_key)?] + } else { + anyhow::bail!("path is neither a directory or a file") + }; + if decoding_keys.is_empty() { + anyhow::bail!("Configured for JWT auth with zero decoding keys. All JWT gated requests would be rejected."); + } + Ok(Self::new(decoding_keys)) } + /// Attempt to decode the token with the internal decoding keys. + /// + /// The function tries the stored decoding keys in succession, + /// and returns the first yielding a successful result. + /// If there is no working decoding key, it returns the last error. pub fn decode(&self, token: &str) -> Result> { - Ok(decode(token, &self.decoding_key, &self.validation)?) + let mut res = None; + for decoding_key in &self.decoding_keys { + res = Some(decode(token, decoding_key, &self.validation)); + if let Some(Ok(res)) = res { + return Ok(res); + } + } + if let Some(res) = res { + res.map_err(anyhow::Error::new) + } else { + anyhow::bail!("no JWT decoding keys configured") + } } } @@ -129,7 +187,7 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH let encoded_eddsa = "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzY29wZSI6InRlbmFudCIsInRlbmFudF9pZCI6IjNkMWY3NTk1YjQ2ODIzMDMwNGUwYjczY2VjYmNiMDgxIiwiaXNzIjoibmVvbi5jb250cm9scGxhbmUiLCJleHAiOjE3MDkyMDA4NzksImlhdCI6MTY3ODQ0MjQ3OX0.U3eA8j-uU-JnhzeO3EDHRuXLwkAUFCPxtGHEgw6p7Ccc3YRbFs2tmCdbD9PZEXP-XsxSeBQi1FY0YPcT3NXADw"; // Check it can be validated with the public key - let auth = JwtAuth::new(DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?); + let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?]); let claims_from_token = auth.decode(encoded_eddsa)?.claims; assert_eq!(claims_from_token, expected_claims); @@ -146,7 +204,7 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH let encoded = encode_from_key_file(&claims, TEST_PRIV_KEY_ED25519)?; // decode it back - let auth = JwtAuth::new(DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?); + let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?]); let decoded = auth.decode(&encoded)?; assert_eq!(decoded.claims, claims); diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index a9ee2425a4..a8a635b6fd 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -1,4 +1,4 @@ -use crate::auth::{Claims, JwtAuth}; +use crate::auth::{Claims, SwappableJwtAuth}; use crate::http::error::{api_error_handler, route_error_handler, ApiError}; use anyhow::Context; use hyper::header::{HeaderName, AUTHORIZATION}; @@ -389,7 +389,7 @@ fn parse_token(header_value: &str) -> Result<&str, ApiError> { } pub fn auth_middleware( - provide_auth: fn(&Request) -> Option<&JwtAuth>, + provide_auth: fn(&Request) -> Option<&SwappableJwtAuth>, ) -> Middleware { Middleware::pre(move |req| async move { if let Some(auth) = provide_auth(&req) { diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 93ae43a14b..5b0c140d00 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -34,8 +34,11 @@ use postgres_backend::AuthType; use utils::logging::TracingErrorLayerEnablement; use utils::signals::ShutdownSignals; use utils::{ - auth::JwtAuth, logging, project_build_tag, project_git_version, sentry_init::init_sentry, - signals::Signal, tcp_listener, + auth::{JwtAuth, SwappableJwtAuth}, + logging, project_build_tag, project_git_version, + sentry_init::init_sentry, + signals::Signal, + tcp_listener, }; project_git_version!(GIT_VERSION); @@ -321,13 +324,12 @@ fn start_pageserver( let http_auth; let pg_auth; if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT { - // unwrap is ok because check is performed when creating config, so path is set and file exists + // unwrap is ok because check is performed when creating config, so path is set and exists let key_path = conf.auth_validation_public_key_path.as_ref().unwrap(); - info!( - "Loading public key for verifying JWT tokens from {:#?}", - key_path - ); - let auth: Arc = Arc::new(JwtAuth::from_key_path(key_path)?); + info!("Loading public key(s) for verifying JWT tokens from {key_path:?}"); + + let jwt_auth = JwtAuth::from_key_path(key_path)?; + let auth: Arc = Arc::new(SwappableJwtAuth::new(jwt_auth)); http_auth = match &conf.http_auth_type { AuthType::Trust => None, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index a58c08e29b..05ed9c0cee 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -161,7 +161,7 @@ pub struct PageServerConf { pub http_auth_type: AuthType, /// authentication method for libpq connections from compute pub pg_auth_type: AuthType, - /// Path to a file containing public key for verifying JWT tokens. + /// Path to a file or directory containing public key(s) for verifying JWT tokens. /// Used for both mgmt and compute auth, if enabled. pub auth_validation_public_key_path: Option, diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index d31e4a1554..9bff0fd668 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -52,6 +52,31 @@ paths: schema: type: object + /v1/reload_auth_validation_keys: + post: + description: Reloads the JWT public keys from their pre-configured location on disk. + responses: + "200": + description: The reload completed successfully. + "401": + description: Unauthorized Error + content: + application/json: + schema: + $ref: "#/components/schemas/UnauthorizedError" + "403": + description: Forbidden Error + content: + application/json: + schema: + $ref: "#/components/schemas/ForbiddenError" + "500": + description: Generic operation error (also hits if no keys were found) + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /v1/tenant/{tenant_id}: parameters: - name: tenant_id diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index a0643bc580..b215b3016e 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -20,6 +20,7 @@ use remote_storage::GenericRemoteStorage; use tenant_size_model::{SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; use tracing::*; +use utils::auth::JwtAuth; use utils::http::endpoint::request_span; use utils::http::json::json_request_or_empty_body; use utils::http::request::{get_request_param, must_get_query_param, parse_query_param}; @@ -45,7 +46,7 @@ use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError, TenantSha use crate::{config::PageServerConf, tenant::mgr}; use crate::{disk_usage_eviction_task, tenant}; use utils::{ - auth::JwtAuth, + auth::SwappableJwtAuth, generation::Generation, http::{ endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with}, @@ -64,7 +65,7 @@ use super::models::ConfigureFailpointsRequest; pub struct State { conf: &'static PageServerConf, tenant_manager: Arc, - auth: Option>, + auth: Option>, allowlist_routes: Vec, remote_storage: Option, broker_client: storage_broker::BrokerClientChannel, @@ -76,7 +77,7 @@ impl State { pub fn new( conf: &'static PageServerConf, tenant_manager: Arc, - auth: Option>, + auth: Option>, remote_storage: Option, broker_client: storage_broker::BrokerClientChannel, disk_usage_eviction_state: Arc, @@ -392,6 +393,32 @@ async fn status_handler( json_response(StatusCode::OK, StatusResponse { id: config.id }) } +async fn reload_auth_validation_keys_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + check_permission(&request, None)?; + let config = get_config(&request); + let state = get_state(&request); + let Some(shared_auth) = &state.auth else { + return json_response(StatusCode::BAD_REQUEST, ()); + }; + // unwrap is ok because check is performed when creating config, so path is set and exists + let key_path = config.auth_validation_public_key_path.as_ref().unwrap(); + info!("Reloading public key(s) for verifying JWT tokens from {key_path:?}"); + + match JwtAuth::from_key_path(key_path) { + Ok(new_auth) => { + shared_auth.swap(new_auth); + json_response(StatusCode::OK, ()) + } + Err(e) => { + warn!("Error reloading public keys from {key_path:?}: {e:}"); + json_response(StatusCode::INTERNAL_SERVER_ERROR, ()) + } + } +} + async fn timeline_create_handler( mut request: Request, _cancel: CancellationToken, @@ -1692,7 +1719,7 @@ where pub fn make_router( state: Arc, launch_ts: &'static LaunchTimestamp, - auth: Option>, + auth: Option>, ) -> anyhow::Result> { let spec = include_bytes!("openapi_spec.yml"); let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc"); @@ -1721,6 +1748,9 @@ pub fn make_router( .put("/v1/failpoints", |r| { testing_api_handler("manage failpoints", r, failpoints_handler) }) + .post("/v1/reload_auth_validation_keys", |r| { + api_handler(r, reload_auth_validation_keys_handler) + }) .get("/v1/tenant", |r| api_handler(r, tenant_list_handler)) .post("/v1/tenant", |r| api_handler(r, tenant_create_handler)) .get("/v1/tenant/:tenant_id", |r| api_handler(r, tenant_status)) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 6086d0b063..be9782847b 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -39,7 +39,7 @@ use tracing::field; use tracing::*; use utils::id::ConnectionId; use utils::{ - auth::{Claims, JwtAuth, Scope}, + auth::{Claims, Scope, SwappableJwtAuth}, id::{TenantId, TimelineId}, lsn::Lsn, simple_rcu::RcuReadGuard, @@ -121,7 +121,7 @@ async fn read_tar_eof(mut reader: (impl AsyncRead + Unpin)) -> anyhow::Result<() pub async fn libpq_listener_main( conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, listener: TcpListener, auth_type: AuthType, listener_ctx: RequestContext, @@ -189,7 +189,7 @@ pub async fn libpq_listener_main( async fn page_service_conn_main( conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, socket: tokio::net::TcpStream, auth_type: AuthType, connection_ctx: RequestContext, @@ -252,7 +252,7 @@ async fn page_service_conn_main( struct PageServerHandler { _conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, claims: Option, /// The context created for the lifetime of the connection @@ -266,7 +266,7 @@ impl PageServerHandler { pub fn new( conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, connection_ctx: RequestContext, ) -> Self { PageServerHandler { diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index d476cf33ae..2e54380471 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -38,7 +38,7 @@ use safekeeper::{http, WAL_REMOVER_RUNTIME}; use safekeeper::{remove_wal, WAL_BACKUP_RUNTIME}; use safekeeper::{wal_backup, HTTP_RUNTIME}; use storage_broker::DEFAULT_ENDPOINT; -use utils::auth::{JwtAuth, Scope}; +use utils::auth::{JwtAuth, Scope, SwappableJwtAuth}; use utils::{ id::NodeId, logging::{self, LogFormat}, @@ -251,10 +251,9 @@ async fn main() -> anyhow::Result<()> { None } Some(path) => { - info!("loading http auth JWT key from {path}"); - Some(Arc::new( - JwtAuth::from_key_path(path).context("failed to load the auth key")?, - )) + info!("loading http auth JWT key(s) from {path}"); + let jwt_auth = JwtAuth::from_key_path(path).context("failed to load the auth key")?; + Some(Arc::new(SwappableJwtAuth::new(jwt_auth))) } }; diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 06b903719e..c48b5330b3 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -30,7 +30,7 @@ use crate::timelines_global_map::TimelineDeleteForceResult; use crate::GlobalTimelines; use crate::SafeKeeperConf; use utils::{ - auth::JwtAuth, + auth::SwappableJwtAuth, http::{ endpoint::{self, auth_middleware, check_permission_with}, error::ApiError, @@ -428,8 +428,11 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder if ALLOWLIST_ROUTES.contains(request.uri()) { None } else { - // Option> is always provided as data below, hence unwrap(). - request.data::>>().unwrap().as_deref() + // Option> is always provided as data below, hence unwrap(). + request + .data::>>() + .unwrap() + .as_deref() } })) } diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 344b0b14e4..3a086f1f54 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -7,7 +7,10 @@ use tokio::runtime::Runtime; use std::time::Duration; use storage_broker::Uri; -use utils::id::{NodeId, TenantId, TenantTimelineId}; +use utils::{ + auth::SwappableJwtAuth, + id::{NodeId, TenantId, TenantTimelineId}, +}; mod auth; pub mod broker; @@ -70,7 +73,7 @@ pub struct SafeKeeperConf { pub wal_backup_enabled: bool, pub pg_auth: Option>, pub pg_tenant_only_auth: Option>, - pub http_auth: Option>, + pub http_auth: Option>, pub current_thread_runtime: bool, } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index d00bf8c4d8..4057f620a0 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -361,7 +361,6 @@ class PgProtocol: @dataclass class AuthKeys: - pub: str priv: str def generate_token(self, *, scope: str, **token_data: str) -> str: @@ -877,9 +876,31 @@ class NeonEnv: @cached_property def auth_keys(self) -> AuthKeys: - pub = (Path(self.repo_dir) / "auth_public_key.pem").read_text() priv = (Path(self.repo_dir) / "auth_private_key.pem").read_text() - return AuthKeys(pub=pub, priv=priv) + return AuthKeys(priv=priv) + + def regenerate_keys_at(self, privkey_path: Path, pubkey_path: Path): + # compare generate_auth_keys() in local_env.rs + subprocess.run( + ["openssl", "genpkey", "-algorithm", "ed25519", "-out", privkey_path], + cwd=self.repo_dir, + check=True, + ) + + subprocess.run( + [ + "openssl", + "pkey", + "-in", + privkey_path, + "-pubout", + "-out", + pubkey_path, + ], + cwd=self.repo_dir, + check=True, + ) + del self.auth_keys def generate_endpoint_id(self) -> str: """ diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index d2c3715c52..aff7959aa7 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -189,6 +189,10 @@ class PageserverHttpClient(requests.Session): assert res_json is None return res_json + def reload_auth_validation_keys(self): + res = self.post(f"http://localhost:{self.port}/v1/reload_auth_validation_keys") + self.verbose_error(res) + def tenant_list(self) -> List[Dict[Any, Any]]: res = self.get(f"http://localhost:{self.port}/v1/tenant") self.verbose_error(res) diff --git a/test_runner/regress/test_auth.py b/test_runner/regress/test_auth.py index 76b75c1caf..f785dc0c8e 100644 --- a/test_runner/regress/test_auth.py +++ b/test_runner/regress/test_auth.py @@ -1,12 +1,35 @@ +import os from contextlib import closing +from pathlib import Path import psycopg2 import pytest -from fixtures.neon_fixtures import NeonEnvBuilder, PgProtocol -from fixtures.pageserver.http import PageserverApiException +from fixtures.neon_fixtures import ( + NeonEnv, + NeonEnvBuilder, + PgProtocol, +) +from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.types import TenantId, TimelineId +def assert_client_authorized(env: NeonEnv, http_client: PageserverHttpClient): + http_client.timeline_create( + pg_version=env.pg_version, + tenant_id=env.initial_tenant, + new_timeline_id=TimelineId.generate(), + ancestor_timeline_id=env.initial_timeline, + ) + + +def assert_client_not_authorized(env: NeonEnv, http_client: PageserverHttpClient): + with pytest.raises( + PageserverApiException, + match="Unauthorized: malformed jwt token", + ): + assert_client_authorized(env, http_client) + + def test_pageserver_auth(neon_env_builder: NeonEnvBuilder): neon_env_builder.auth_enabled = True env = neon_env_builder.init_start() @@ -27,30 +50,16 @@ def test_pageserver_auth(neon_env_builder: NeonEnvBuilder): ps.safe_psql("set FOO", password=pageserver_token) # tenant can create branches - tenant_http_client.timeline_create( - pg_version=env.pg_version, - tenant_id=env.initial_tenant, - new_timeline_id=TimelineId.generate(), - ancestor_timeline_id=env.initial_timeline, - ) + assert_client_authorized(env, tenant_http_client) + # console can create branches for tenant - pageserver_http_client.timeline_create( - pg_version=env.pg_version, - tenant_id=env.initial_tenant, - new_timeline_id=TimelineId.generate(), - ancestor_timeline_id=env.initial_timeline, - ) + assert_client_authorized(env, pageserver_http_client) # fail to create branch using token with different tenant_id with pytest.raises( PageserverApiException, match="Forbidden: Tenant id mismatch. Permission denied" ): - invalid_tenant_http_client.timeline_create( - pg_version=env.pg_version, - tenant_id=env.initial_tenant, - new_timeline_id=TimelineId.generate(), - ancestor_timeline_id=env.initial_timeline, - ) + assert_client_authorized(env, invalid_tenant_http_client) # create tenant using management token pageserver_http_client.tenant_create(TenantId.generate()) @@ -82,6 +91,94 @@ def test_compute_auth_to_pageserver(neon_env_builder: NeonEnvBuilder): assert cur.fetchone() == (5000050000,) +def test_pageserver_multiple_keys(neon_env_builder: NeonEnvBuilder): + neon_env_builder.auth_enabled = True + env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*Unauthorized: malformed jwt token.*") + + pageserver_token_old = env.auth_keys.generate_pageserver_token() + pageserver_http_client_old = env.pageserver.http_client(pageserver_token_old) + + pageserver_http_client_old.reload_auth_validation_keys() + + # This test is to ensure that the pageserver supports multiple keys. + # The neon_local tool generates one key pair at a hardcoded path by default. + # As a preparation for our test, move the public key of the key pair into a + # directory at the same location as the hardcoded path by: + # 1. moving the the file at `configured_pub_key_path` to a temporary location + # 2. creating a new directory at `configured_pub_key_path` + # 3. moving the file from the temporary location into the newly created directory + configured_pub_key_path = Path(env.repo_dir) / "auth_public_key.pem" + os.rename(configured_pub_key_path, Path(env.repo_dir) / "auth_public_key.pem.file") + os.mkdir(configured_pub_key_path) + os.rename( + Path(env.repo_dir) / "auth_public_key.pem.file", + configured_pub_key_path / "auth_public_key_old.pem", + ) + + # Add a new key pair + # This invalidates env.auth_keys and makes them be regenerated + env.regenerate_keys_at( + Path("auth_private_key.pem"), Path("auth_public_key.pem/auth_public_key_new.pem") + ) + + # Reload the keys on the pageserver side + pageserver_http_client_old.reload_auth_validation_keys() + + # We can continue doing things using the old token + assert_client_authorized(env, pageserver_http_client_old) + + pageserver_token_new = env.auth_keys.generate_pageserver_token() + pageserver_http_client_new = env.pageserver.http_client(pageserver_token_new) + + # The new token also works + assert_client_authorized(env, pageserver_http_client_new) + + # Remove the old token and reload + os.remove(Path(env.repo_dir) / "auth_public_key.pem" / "auth_public_key_old.pem") + pageserver_http_client_old.reload_auth_validation_keys() + + # Reloading fails now with the old token, but the new token still works + assert_client_not_authorized(env, pageserver_http_client_old) + assert_client_authorized(env, pageserver_http_client_new) + + +def test_pageserver_key_reload(neon_env_builder: NeonEnvBuilder): + neon_env_builder.auth_enabled = True + env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*Unauthorized: malformed jwt token.*") + + pageserver_token_old = env.auth_keys.generate_pageserver_token() + pageserver_http_client_old = env.pageserver.http_client(pageserver_token_old) + + pageserver_http_client_old.reload_auth_validation_keys() + + # Regenerate the keys + env.regenerate_keys_at(Path("auth_private_key.pem"), Path("auth_public_key.pem")) + + # Reload the keys on the pageserver side + pageserver_http_client_old.reload_auth_validation_keys() + + # Next attempt fails as we use the old auth token + with pytest.raises( + PageserverApiException, + match="Unauthorized: malformed jwt token", + ): + pageserver_http_client_old.reload_auth_validation_keys() + + # same goes for attempts trying to create a timeline + assert_client_not_authorized(env, pageserver_http_client_old) + + pageserver_token_new = env.auth_keys.generate_pageserver_token() + pageserver_http_client_new = env.pageserver.http_client(pageserver_token_new) + + # timeline creation works with the new token + assert_client_authorized(env, pageserver_http_client_new) + + # reloading also works with the new token + pageserver_http_client_new.reload_auth_validation_keys() + + @pytest.mark.parametrize("auth_enabled", [False, True]) def test_auth_failures(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): neon_env_builder.auth_enabled = auth_enabled From fc47af156f4d712e04722a3319a24e4c136c1f0b Mon Sep 17 00:00:00 2001 From: Andrew Rudenko Date: Tue, 7 Nov 2023 16:49:26 +0100 Subject: [PATCH 21/63] Passing neon options to the console (#5781) The idea is to pass neon_* prefixed options to control plane. It can be used by cplane to dynamically create timelines and computes. Such options also should be excluded from passing to compute. Another issue is how connection caching is working now, because compute's instance now depends not only on hostname but probably on such options too I included them to cache key. --- proxy/src/auth/credentials.rs | 39 +++++++++++++++++++++++++-- proxy/src/compute.rs | 9 ++++++- proxy/src/console/provider.rs | 1 + proxy/src/console/provider/neon.rs | 3 ++- proxy/src/proxy.rs | 31 ++++++++++++++++++++- proxy/src/proxy/tests.rs | 1 + proxy/src/serverless/conn_pool.rs | 21 ++++++++------- proxy/src/serverless/sql_over_http.rs | 12 +++++++++ 8 files changed, 103 insertions(+), 14 deletions(-) diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index 7dc304d7ac..d7a8edca79 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -1,6 +1,8 @@ //! User credentials used in authentication. -use crate::{auth::password_hack::parse_endpoint_param, error::UserFacingError}; +use crate::{ + auth::password_hack::parse_endpoint_param, error::UserFacingError, proxy::neon_options, +}; use itertools::Itertools; use pq_proto::StartupMessageParams; use std::collections::HashSet; @@ -38,6 +40,8 @@ pub struct ClientCredentials<'a> { pub user: &'a str, // TODO: this is a severe misnomer! We should think of a new name ASAP. pub project: Option, + + pub cache_key: String, } impl ClientCredentials<'_> { @@ -53,6 +57,7 @@ impl<'a> ClientCredentials<'a> { ClientCredentials { user: "", project: None, + cache_key: "".to_string(), } } @@ -120,7 +125,17 @@ impl<'a> ClientCredentials<'a> { info!(user, project = project.as_deref(), "credentials"); - Ok(Self { user, project }) + let cache_key = format!( + "{}{}", + project.as_deref().unwrap_or(""), + neon_options(params).unwrap_or("".to_string()) + ); + + Ok(Self { + user, + project, + cache_key, + }) } } @@ -176,6 +191,7 @@ mod tests { let creds = ClientCredentials::parse(&options, sni, common_names)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project.as_deref(), Some("foo")); + assert_eq!(creds.cache_key, "foo"); Ok(()) } @@ -303,4 +319,23 @@ mod tests { _ => panic!("bad error: {err:?}"), } } + + #[test] + fn parse_neon_options() -> anyhow::Result<()> { + let options = StartupMessageParams::new([ + ("user", "john_doe"), + ("options", "neon_lsn:0/2 neon_endpoint_type:read_write"), + ]); + + let sni = Some("project.localhost"); + let common_names = Some(["localhost".into()].into()); + let creds = ClientCredentials::parse(&options, sni, common_names)?; + assert_eq!(creds.project.as_deref(), Some("project")); + assert_eq!( + creds.cache_key, + "projectneon_endpoint_type:read_write neon_lsn:0/2" + ); + + Ok(()) + } } diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index e96b79ed92..53eb0e3a76 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -3,6 +3,7 @@ use crate::{ cancellation::CancelClosure, console::errors::WakeComputeError, error::{io_error, UserFacingError}, + proxy::is_neon_param, }; use futures::{FutureExt, TryFutureExt}; use itertools::Itertools; @@ -278,7 +279,7 @@ fn filtered_options(params: &StartupMessageParams) -> Option { #[allow(unstable_name_collisions)] let options: String = params .options_raw()? - .filter(|opt| parse_endpoint_param(opt).is_none()) + .filter(|opt| parse_endpoint_param(opt).is_none() && !is_neon_param(opt)) .intersperse(" ") // TODO: use impl from std once it's stabilized .collect(); @@ -313,5 +314,11 @@ mod tests { let params = StartupMessageParams::new([("options", "project = foo")]); assert_eq!(filtered_options(¶ms).as_deref(), Some("project = foo")); + + let params = StartupMessageParams::new([( + "options", + "project = foo neon_endpoint_type:read_write neon_lsn:0/2", + )]); + assert_eq!(filtered_options(¶ms).as_deref(), Some("project = foo")); } } diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index 32c3467092..c7cfc88c75 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -178,6 +178,7 @@ pub struct ConsoleReqExtra<'a> { pub session_id: uuid::Uuid, /// Name of client application, if set. pub application_name: Option<&'a str>, + pub options: Option<&'a str>, } /// Auth secret which is managed by the cloud. diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 927fea0a13..6229840c46 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -99,6 +99,7 @@ impl Api { .query(&[ ("application_name", extra.application_name), ("project", Some(project)), + ("options", extra.options), ]) .build()?; @@ -151,7 +152,7 @@ impl super::Api for Api { extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials, ) -> Result { - let key = creds.project().expect("impossible"); + let key: &str = &creds.cache_key; // Every time we do a wakeup http request, the compute node will stay up // for some time (highly depends on the console's scale-to-zero policy); diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 884aae1651..54c3503c93 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -15,10 +15,12 @@ use crate::{ use anyhow::{bail, Context}; use async_trait::async_trait; use futures::TryFutureExt; +use itertools::Itertools; use metrics::{exponential_buckets, register_int_counter_vec, IntCounterVec}; -use once_cell::sync::Lazy; +use once_cell::sync::{Lazy, OnceCell}; use pq_proto::{BeMessage as Be, FeStartupPacket, StartupMessageParams}; use prometheus::{register_histogram_vec, HistogramVec}; +use regex::Regex; use std::{error::Error, io, ops::ControlFlow, sync::Arc, time::Instant}; use tokio::{ io::{AsyncRead, AsyncWrite, AsyncWriteExt}, @@ -881,9 +883,12 @@ impl Client<'_, S> { allow_self_signed_compute, } = self; + let console_options = neon_options(params); + let extra = console::ConsoleReqExtra { session_id, // aka this connection's id application_name: params.get("application_name"), + options: console_options.as_deref(), }; let mut latency_timer = LatencyTimer::new(mode.protocol_label()); @@ -945,3 +950,27 @@ impl Client<'_, S> { proxy_pass(stream, node.stream, &aux).await } } + +pub fn neon_options(params: &StartupMessageParams) -> Option { + #[allow(unstable_name_collisions)] + let options: String = params + .options_raw()? + .filter(|opt| is_neon_param(opt)) + .sorted() // we sort it to use as cache key + .intersperse(" ") // TODO: use impl from std once it's stabilized + .collect(); + + // Don't even bother with empty options. + if options.is_empty() { + return None; + } + + Some(options) +} + +pub fn is_neon_param(bytes: &str) -> bool { + static RE: OnceCell = OnceCell::new(); + RE.get_or_init(|| Regex::new(r"^neon_\w+:").unwrap()); + + RE.get().unwrap().is_match(bytes) +} diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index 142c32fb84..3ae4df46ef 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -440,6 +440,7 @@ fn helper_create_connect_info( let extra = console::ConsoleReqExtra { session_id: uuid::Uuid::new_v4(), application_name: Some("TEST"), + options: None, }; let creds = auth::BackendType::Test(mechanism); (cache, extra, creds) diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index c5bfc32568..d09554a922 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -22,7 +22,10 @@ use tokio_postgres::{AsyncMessage, ReadyForQueryStatus}; use crate::{ auth, console, - proxy::{LatencyTimer, NUM_DB_CONNECTIONS_CLOSED_COUNTER, NUM_DB_CONNECTIONS_OPENED_COUNTER}, + proxy::{ + neon_options, LatencyTimer, NUM_DB_CONNECTIONS_CLOSED_COUNTER, + NUM_DB_CONNECTIONS_OPENED_COUNTER, + }, usage_metrics::{Ids, MetricCounter, USAGE_METRICS}, }; use crate::{compute, config}; @@ -41,6 +44,7 @@ pub struct ConnInfo { pub dbname: String, pub hostname: String, pub password: String, + pub options: Option, } impl ConnInfo { @@ -401,26 +405,25 @@ async fn connect_to_compute( let tls = config.tls_config.as_ref(); let common_names = tls.and_then(|tls| tls.common_names.clone()); - let credential_params = StartupMessageParams::new([ + let params = StartupMessageParams::new([ ("user", &conn_info.username), ("database", &conn_info.dbname), ("application_name", APP_NAME), + ("options", conn_info.options.as_deref().unwrap_or("")), ]); let creds = config .auth_backend .as_ref() - .map(|_| { - auth::ClientCredentials::parse( - &credential_params, - Some(&conn_info.hostname), - common_names, - ) - }) + .map(|_| auth::ClientCredentials::parse(¶ms, Some(&conn_info.hostname), common_names)) .transpose()?; + + let console_options = neon_options(¶ms); + let extra = console::ConsoleReqExtra { session_id: uuid::Uuid::new_v4(), application_name: Some(APP_NAME), + options: console_options.as_deref(), }; let node_info = creds diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 8f7cf7fbaf..16736ac00d 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -174,11 +174,23 @@ fn get_conn_info( } } + let pairs = connection_url.query_pairs(); + + let mut options = Option::None; + + for (key, value) in pairs { + if key == "options" { + options = Some(value.to_string()); + break; + } + } + Ok(ConnInfo { username: username.to_owned(), dbname: dbname.to_owned(), hostname: hostname.to_owned(), password: password.to_owned(), + options, }) } From 11d9d801b50f9e9b9085d7dd700dccb7cda553b6 Mon Sep 17 00:00:00 2001 From: duguorong009 <80258679+duguorong009@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:57:26 -0500 Subject: [PATCH 22/63] pageserver: improve the shutdown log error (#5792) ## Problem - Close #5784 ## Summary of changes - Update the `GetActiveTenantError` -> `QueryError` conversion process in `pageserver/src/page_service.rs` - Update the pytest logging exceptions in `./test_runner/regress/test_tenant_detach.py` --- pageserver/src/page_service.rs | 4 ++++ .../regress/test_pageserver_restarts_under_workload.py | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index be9782847b..5193b3c5ff 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -14,6 +14,7 @@ use async_compression::tokio::write::GzipEncoder; use bytes::Buf; use bytes::Bytes; use futures::Stream; +use pageserver_api::models::TenantState; use pageserver_api::models::{ PagestreamBeMessage, PagestreamDbSizeRequest, PagestreamDbSizeResponse, PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse, @@ -1330,6 +1331,9 @@ impl From for QueryError { GetActiveTenantError::WaitForActiveTimeout { .. } => QueryError::Disconnected( ConnectionError::Io(io::Error::new(io::ErrorKind::TimedOut, e.to_string())), ), + GetActiveTenantError::WillNotBecomeActive(TenantState::Stopping { .. }) => { + QueryError::Shutdown + } e => QueryError::Other(anyhow::anyhow!(e)), } } diff --git a/test_runner/regress/test_pageserver_restarts_under_workload.py b/test_runner/regress/test_pageserver_restarts_under_workload.py index d07b8dbe6b..65569f3bac 100644 --- a/test_runner/regress/test_pageserver_restarts_under_workload.py +++ b/test_runner/regress/test_pageserver_restarts_under_workload.py @@ -17,10 +17,6 @@ def test_pageserver_restarts_under_worload(neon_simple_env: NeonEnv, pg_bin: PgB n_restarts = 10 scale = 10 - # Pageserver currently logs requests on non-active tenants at error level - # https://github.com/neondatabase/neon/issues/5784 - env.pageserver.allowed_errors.append(".* will not become active. Current state: Stopping.*") - def run_pgbench(connstr: str): log.info(f"Start a pgbench workload on pg {connstr}") pg_bin.run_capture(["pgbench", "-i", f"-s{scale}", connstr]) From acef742a6e471e214dfe6e88e8e94a77b6fcd2db Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Tue, 7 Nov 2023 09:41:20 -0800 Subject: [PATCH 23/63] vm-monitor: Remove dependency on workspace_hack (#5752) neondatabase/autoscaling builds libs/vm-monitor during CI because it's a necessary component of autoscaling. workspace_hack includes a lot of crates that are not necessary for vm-monitor, which artificially inflates the build time on the autoscaling side, so hopefully removing the dependency should speed things up. Co-authored-by: Joonas Koivunen --- .config/hakari.toml | 6 ++++++ Cargo.lock | 1 - libs/vm_monitor/Cargo.toml | 3 +-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.config/hakari.toml b/.config/hakari.toml index 15b939e86f..9913ecc9c0 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -22,5 +22,11 @@ platforms = [ # "x86_64-pc-windows-msvc", ] +[final-excludes] +# vm_monitor benefits from the same Cargo.lock as the rest of our artifacts, but +# it is built primarly in separate repo neondatabase/autoscaling and thus is excluded +# from depending on workspace-hack because most of the dependencies are not used. +workspace-members = ["vm_monitor"] + # Write out exact versions rather than a semver range. (Defaults to false.) # exact-versions = true diff --git a/Cargo.lock b/Cargo.lock index 10f13e21fd..5e7fce3e8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6055,7 +6055,6 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", - "workspace_hack", ] [[package]] diff --git a/libs/vm_monitor/Cargo.toml b/libs/vm_monitor/Cargo.toml index 26b976830a..46e9f880a1 100644 --- a/libs/vm_monitor/Cargo.toml +++ b/libs/vm_monitor/Cargo.toml @@ -19,13 +19,12 @@ inotify.workspace = true serde.workspace = true serde_json.workspace = true sysinfo.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread"] } tokio-postgres.workspace = true tokio-stream.workspace = true tokio-util.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -workspace_hack = { version = "0.1", path = "../../workspace_hack" } [target.'cfg(target_os = "linux")'.dependencies] cgroups-rs = "0.3.3" From b989ad1922a46dbb6bc21d71b805246c22519139 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 8 Nov 2023 11:26:56 +0000 Subject: [PATCH 24/63] extend test_change_pageserver for failure case, rework changing pageserver (#5693) Reproducer for https://github.com/neondatabase/neon/issues/5692 The test change in this PR intentionally fails, to demonstrate the issue. --------- Co-authored-by: Sasha Krassovsky --- compute_tools/src/compute.rs | 10 +- pgxn/neon/libpagestore.c | 142 ++++++++++++++++-- test_runner/regress/test_change_pageserver.py | 77 ++++++++-- 3 files changed, 200 insertions(+), 29 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 9fd88e5818..373f05ab2f 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -710,8 +710,12 @@ impl ComputeNode { // `pg_ctl` for start / stop, so this just seems much easier to do as we already // have opened connection to Postgres and superuser access. #[instrument(skip_all)] - fn pg_reload_conf(&self, client: &mut Client) -> Result<()> { - client.simple_query("SELECT pg_reload_conf()")?; + fn pg_reload_conf(&self) -> Result<()> { + let pgctl_bin = Path::new(&self.pgbin).parent().unwrap().join("pg_ctl"); + Command::new(pgctl_bin) + .args(["reload", "-D", &self.pgdata]) + .output() + .expect("cannot run pg_ctl process"); Ok(()) } @@ -724,9 +728,9 @@ impl ComputeNode { // Write new config let pgdata_path = Path::new(&self.pgdata); config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), &spec, None)?; + self.pg_reload_conf()?; let mut client = Client::connect(self.connstr.as_str(), NoTls)?; - self.pg_reload_conf(&mut client)?; // Proceed with post-startup configuration. Note, that order of operations is important. // Disable DDL forwarding because control plane already knows about these roles/databases. diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index 0a944a6bf0..cc09fb849d 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -19,7 +19,10 @@ #include "access/xlog.h" #include "access/xlogutils.h" #include "storage/buf_internals.h" +#include "storage/lwlock.h" +#include "storage/ipc.h" #include "c.h" +#include "postmaster/interrupt.h" #include "libpq-fe.h" #include "libpq/pqformat.h" @@ -61,23 +64,63 @@ int flush_every_n_requests = 8; int n_reconnect_attempts = 0; int max_reconnect_attempts = 60; +#define MAX_PAGESERVER_CONNSTRING_SIZE 256 + +typedef struct +{ + LWLockId lock; + pg_atomic_uint64 update_counter; + char pageserver_connstring[MAX_PAGESERVER_CONNSTRING_SIZE]; +} PagestoreShmemState; + +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void walproposer_shmem_request(void); +#endif +static shmem_startup_hook_type prev_shmem_startup_hook; +static PagestoreShmemState *pagestore_shared; +static uint64 pagestore_local_counter = 0; +static char local_pageserver_connstring[MAX_PAGESERVER_CONNSTRING_SIZE]; + bool (*old_redo_read_buffer_filter) (XLogReaderState *record, uint8 block_id) = NULL; static bool pageserver_flush(void); static void pageserver_disconnect(void); - -static pqsigfunc prev_signal_handler; +static bool +CheckPageserverConnstring(char **newval, void **extra, GucSource source) +{ + return strlen(*newval) < MAX_PAGESERVER_CONNSTRING_SIZE; +} static void -pageserver_sighup_handler(SIGNAL_ARGS) +AssignPageserverConnstring(const char *newval, void *extra) { - if (prev_signal_handler) - { - prev_signal_handler(postgres_signal_arg); - } - neon_log(LOG, "Received SIGHUP, disconnecting pageserver. New pageserver connstring is %s", page_server_connstring); - pageserver_disconnect(); + if(!pagestore_shared) + return; + LWLockAcquire(pagestore_shared->lock, LW_EXCLUSIVE); + strlcpy(pagestore_shared->pageserver_connstring, newval, MAX_PAGESERVER_CONNSTRING_SIZE); + pg_atomic_fetch_add_u64(&pagestore_shared->update_counter, 1); + LWLockRelease(pagestore_shared->lock); +} + +static bool +CheckConnstringUpdated() +{ + if(!pagestore_shared) + return false; + return pagestore_local_counter < pg_atomic_read_u64(&pagestore_shared->update_counter); +} + +static void +ReloadConnstring() +{ + if(!pagestore_shared) + return; + LWLockAcquire(pagestore_shared->lock, LW_SHARED); + strlcpy(local_pageserver_connstring, pagestore_shared->pageserver_connstring, sizeof(local_pageserver_connstring)); + pagestore_local_counter = pg_atomic_read_u64(&pagestore_shared->update_counter); + LWLockRelease(pagestore_shared->lock); } static bool @@ -91,6 +134,11 @@ pageserver_connect(int elevel) Assert(!connected); + if(CheckConnstringUpdated()) + { + ReloadConnstring(); + } + /* * Connect using the connection string we got from the * neon.pageserver_connstring GUC. If the NEON_AUTH_TOKEN environment @@ -110,7 +158,7 @@ pageserver_connect(int elevel) n++; } keywords[n] = "dbname"; - values[n] = page_server_connstring; + values[n] = local_pageserver_connstring; n++; keywords[n] = NULL; values[n] = NULL; @@ -254,6 +302,12 @@ pageserver_send(NeonRequest * request) { StringInfoData req_buff; + if(CheckConnstringUpdated()) + { + pageserver_disconnect(); + ReloadConnstring(); + } + /* If the connection was lost for some reason, reconnect */ if (connected && PQstatus(pageserver_conn) == CONNECTION_BAD) { @@ -274,6 +328,7 @@ pageserver_send(NeonRequest * request) { while (!pageserver_connect(n_reconnect_attempts < max_reconnect_attempts ? LOG : ERROR)) { + HandleMainLoopInterrupts(); n_reconnect_attempts += 1; pg_usleep(RECONNECT_INTERVAL_USEC); } @@ -391,7 +446,8 @@ pageserver_flush(void) return true; } -page_server_api api = { +page_server_api api = +{ .send = pageserver_send, .flush = pageserver_flush, .receive = pageserver_receive @@ -405,12 +461,72 @@ check_neon_id(char **newval, void **extra, GucSource source) return **newval == '\0' || HexDecodeString(id, *newval, 16); } +static Size +PagestoreShmemSize(void) +{ + return sizeof(PagestoreShmemState); +} + +static bool +PagestoreShmemInit(void) +{ + bool found; + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + pagestore_shared = ShmemInitStruct("libpagestore shared state", + PagestoreShmemSize(), + &found); + if(!found) + { + pagestore_shared->lock = &(GetNamedLWLockTranche("neon_libpagestore")->lock); + pg_atomic_init_u64(&pagestore_shared->update_counter, 0); + AssignPageserverConnstring(page_server_connstring, NULL); + } + LWLockRelease(AddinShmemInitLock); + return found; +} + +static void +pagestore_shmem_startup_hook(void) +{ + if(prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + PagestoreShmemInit(); +} + +static void +pagestore_shmem_request(void) +{ +#if PG_VERSION_NUM >= 150000 + if(prev_shmem_request_hook) + prev_shmem_request_hook(); +#endif + + RequestAddinShmemSpace(PagestoreShmemSize()); + RequestNamedLWLockTranche("neon_libpagestore", 1); +} + +static void +pagestore_prepare_shmem(void) +{ +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = pagestore_shmem_request; +#else + pagestore_shmem_request(); +#endif + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = pagestore_shmem_startup_hook; +} + /* * Module initialization function */ void pg_init_libpagestore(void) { + pagestore_prepare_shmem(); + DefineCustomStringVariable("neon.pageserver_connstring", "connection string to the page server", NULL, @@ -418,7 +534,7 @@ pg_init_libpagestore(void) "", PGC_SIGHUP, 0, /* no flags required */ - NULL, NULL, NULL); + CheckPageserverConnstring, AssignPageserverConnstring, NULL); DefineCustomStringVariable("neon.timeline_id", "Neon timeline_id the server is running on", @@ -499,7 +615,5 @@ pg_init_libpagestore(void) redo_read_buffer_filter = neon_redo_read_buffer_filter; } - prev_signal_handler = pqsignal(SIGHUP, pageserver_sighup_handler); - lfc_init(); } diff --git a/test_runner/regress/test_change_pageserver.py b/test_runner/regress/test_change_pageserver.py index 31092b70b9..410bf03c2b 100644 --- a/test_runner/regress/test_change_pageserver.py +++ b/test_runner/regress/test_change_pageserver.py @@ -1,9 +1,13 @@ +import asyncio + from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder from fixtures.remote_storage import RemoteStorageKind def test_change_pageserver(neon_env_builder: NeonEnvBuilder): + num_connections = 3 + neon_env_builder.num_pageservers = 2 neon_env_builder.enable_pageserver_remote_storage( remote_storage_kind=RemoteStorageKind.MOCK_S3, @@ -16,15 +20,24 @@ def test_change_pageserver(neon_env_builder: NeonEnvBuilder): alt_pageserver_id = env.pageservers[1].id env.pageservers[1].tenant_attach(env.initial_tenant) - pg_conn = endpoint.connect() - cur = pg_conn.cursor() + pg_conns = [endpoint.connect() for i in range(num_connections)] + curs = [pg_conn.cursor() for pg_conn in pg_conns] + + def execute(statement: str): + for cur in curs: + cur.execute(statement) + + def fetchone(): + results = [cur.fetchone() for cur in curs] + assert all(result == results[0] for result in results) + return results[0] # Create table, and insert some rows. Make it big enough that it doesn't fit in # shared_buffers, otherwise the SELECT after restart will just return answer # from shared_buffers without hitting the page server, which defeats the point # of this test. - cur.execute("CREATE TABLE foo (t text)") - cur.execute( + curs[0].execute("CREATE TABLE foo (t text)") + curs[0].execute( """ INSERT INTO foo SELECT 'long string to consume some space' || g @@ -33,25 +46,25 @@ def test_change_pageserver(neon_env_builder: NeonEnvBuilder): ) # Verify that the table is larger than shared_buffers - cur.execute( + curs[0].execute( """ select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_size from pg_settings where name = 'shared_buffers' """ ) - row = cur.fetchone() + row = curs[0].fetchone() assert row is not None log.info(f"shared_buffers is {row[0]}, table size {row[1]}") assert int(row[0]) < int(row[1]) - cur.execute("SELECT count(*) FROM foo") - assert cur.fetchone() == (100000,) + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) endpoint.reconfigure(pageserver_id=alt_pageserver_id) # Verify that the neon.pageserver_connstring GUC is set to the correct thing - cur.execute("SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'") - connstring = cur.fetchone() + execute("SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'") + connstring = fetchone() assert connstring is not None expected_connstring = f"postgresql://no_user:@localhost:{env.pageservers[1].service_port.pg}" assert expected_connstring == expected_connstring @@ -60,5 +73,45 @@ def test_change_pageserver(neon_env_builder: NeonEnvBuilder): 0 ].stop() # Stop the old pageserver just to make sure we're reading from the new one - cur.execute("SELECT count(*) FROM foo") - assert cur.fetchone() == (100000,) + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) + + # Try failing back, and this time we will stop the current pageserver before reconfiguring + # the endpoint. Whereas the previous reconfiguration was like a healthy migration, this + # is more like what happens in an unexpected pageserver failure. + env.pageservers[0].start() + env.pageservers[1].stop() + + endpoint.reconfigure(pageserver_id=env.pageservers[0].id) + + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) + + env.pageservers[0].stop() + env.pageservers[1].start() + + # Test a (former) bug where a child process spins without updating its connection string + # by executing a query separately. This query will hang until we issue the reconfigure. + async def reconfigure_async(): + await asyncio.sleep( + 1 + ) # Sleep for 1 second just to make sure we actually started our count(*) query + endpoint.reconfigure(pageserver_id=env.pageservers[1].id) + + def execute_count(): + execute("SELECT count(*) FROM FOO") + + async def execute_and_reconfigure(): + task_exec = asyncio.to_thread(execute_count) + task_reconfig = asyncio.create_task(reconfigure_async()) + await asyncio.gather( + task_exec, + task_reconfig, + ) + + asyncio.run(execute_and_reconfigure()) + assert fetchone() == (100000,) + + # One final check that nothing hangs + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) From a8a39cd4647f2b4123ca6a4d47ce8042bd3e1ce5 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 8 Nov 2023 12:41:48 +0000 Subject: [PATCH 25/63] test: de-flake test_deletion_queue_recovery (#5822) ## Problem This test could fail if timing is unlucky, and the deletions in the test land in two deletion lists instead of one. ## Summary of changes We await _some_ validations instead of _all_ validations, because our execution failpoint will prevent validation proceeding for any but the first DeletionList. Usually the workload just generates one, but if it generates two due to timing, then we must not expect that the second one will be validated. --- .../regress/test_pageserver_generations.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 3e5021ae06..c3f4ad476f 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -366,11 +366,17 @@ def test_deletion_queue_recovery( assert get_deletion_queue_dropped_lsn_updates(ps_http) == 0 if validate_before == ValidateBefore.VALIDATE: + # At this point, one or more DeletionLists have been written. We have set a failpoint + # to prevent them successfully executing, but we want to see them get validated. + # + # We await _some_ validations instead of _all_ validations, because our execution failpoint + # will prevent validation proceeding for any but the first DeletionList. Usually the workload + # just generates one, but if it generates two due to timing, then we must not expect that the + # second one will be validated. + def assert_some_validations(): + assert get_deletion_queue_validated(ps_http) > 0 - def assert_validation_complete(): - assert get_deletion_queue_submitted(ps_http) == get_deletion_queue_validated(ps_http) - - wait_until(20, 1, assert_validation_complete) + wait_until(20, 1, assert_some_validations) # The validatated keys statistic advances before the header is written, so we # also wait to see the header hit the disk: this seems paranoid but the race @@ -380,6 +386,11 @@ def test_deletion_queue_recovery( wait_until(20, 1, assert_header_written) + # If we will lose attachment, then our expectation on restart is that only the ones + # we already validated will execute. Act like only those were present in the queue. + if keep_attachment == KeepAttachment.LOSE: + before_restart_depth = get_deletion_queue_validated(ps_http) + log.info(f"Restarting pageserver with {before_restart_depth} deletions enqueued") env.pageserver.stop(immediate=True) @@ -402,11 +413,13 @@ def test_deletion_queue_recovery( ps_http.deletion_queue_flush(execute=True) wait_until(10, 1, lambda: assert_deletion_queue(ps_http, lambda n: n == 0)) - if keep_attachment == KeepAttachment.KEEP or validate_before == ValidateBefore.VALIDATE: + if keep_attachment == KeepAttachment.KEEP: # - If we kept the attachment, then our pre-restart deletions should execute # because on re-attach they were from the immediately preceding generation - # - If we validated before restart, then the deletions should execute because the - # deletion queue header records a validated deletion list sequence number. + assert get_deletion_queue_executed(ps_http) == before_restart_depth + elif validate_before == ValidateBefore.VALIDATE: + # - If we validated before restart, then we should execute however many keys were + # validated before restart. assert get_deletion_queue_executed(ps_http) == before_restart_depth else: env.pageserver.allowed_errors.extend([".*Dropping stale deletions.*"]) From 40441f8ada8d4daf7ec9839224090931035f2589 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 8 Nov 2023 13:00:11 +0000 Subject: [PATCH 26/63] pageserver: use `Gate` for stronger safety check in `SlotGuard` (#5793) ## Problem #5711 and #5367 raced -- the `SlotGuard` type needs `Gate` to properly enforce its invariant that we may not drop an `Arc` from a slot. ## Summary of changes Replace the TODO with the intended check of Gate. --- libs/utils/src/sync/gate.rs | 7 +++++++ pageserver/src/tenant/mgr.rs | 9 +-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/utils/src/sync/gate.rs b/libs/utils/src/sync/gate.rs index 1391d238e6..9aad0af22d 100644 --- a/libs/utils/src/sync/gate.rs +++ b/libs/utils/src/sync/gate.rs @@ -85,6 +85,13 @@ impl Gate { warn_if_stuck(self.do_close(), &self.name, Duration::from_millis(1000)).await } + /// Check if [`Self::close()`] has finished waiting for all [`Self::enter()`] users to finish. This + /// is usually analoguous for "Did shutdown finish?" for types that include a Gate, whereas checking + /// the CancellationToken on such types is analogous to "Did shutdown start?" + pub fn close_complete(&self) -> bool { + self.sem.is_closed() + } + async fn do_close(&self) { tracing::debug!(gate = self.name, "Closing Gate..."); match self.sem.acquire_many(Self::MAX_UNITS).await { diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 27f9d50c54..07d1618272 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -1552,14 +1552,7 @@ impl SlotGuard { /// is responsible for protecting fn old_value_is_shutdown(&self) -> bool { match self.old_value.as_ref() { - Some(TenantSlot::Attached(tenant)) => { - // TODO: PR #5711 will add a gate that enables properly checking that - // shutdown completed. - matches!( - tenant.current_state(), - TenantState::Stopping { .. } | TenantState::Broken { .. } - ) - } + Some(TenantSlot::Attached(tenant)) => tenant.gate.close_complete(), Some(TenantSlot::Secondary) => { // TODO: when adding secondary mode tenants, this will check for shutdown // in the same way that we do for `Tenant` above From e9b227a11e4d1f5df09cca9d72e32c95f02750e3 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 8 Nov 2023 17:54:33 +0100 Subject: [PATCH 27/63] cleanup unused RemoteStorage fields (#5830) Found this while working on #5771 --- compute_tools/src/extension_server.rs | 4 +-- libs/remote_storage/src/lib.rs | 36 ++------------------ libs/remote_storage/tests/test_real_azure.rs | 4 +-- libs/remote_storage/tests/test_real_s3.rs | 4 +-- pageserver/src/config.rs | 8 ----- pageserver/src/deletion_queue.rs | 8 ----- pageserver/src/tenant.rs | 4 --- 7 files changed, 5 insertions(+), 63 deletions(-) diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 3d7ed8c360..2278509c1f 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -78,7 +78,7 @@ use regex::Regex; use remote_storage::*; use serde_json; use std::io::Read; -use std::num::{NonZeroU32, NonZeroUsize}; +use std::num::NonZeroUsize; use std::path::Path; use std::str; use tar::Archive; @@ -281,8 +281,6 @@ pub fn init_remote_storage(remote_ext_config: &str) -> anyhow::Result @@ -443,10 +431,6 @@ pub struct StorageMetadata(HashMap); /// External backup storage configuration, enough for creating a client for that storage. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RemoteStorageConfig { - /// Max allowed number of concurrent sync operations between the API user and the remote storage. - pub max_concurrent_syncs: NonZeroUsize, - /// Max allowed errors before the sync task is considered failed and evicted. - pub max_sync_errors: NonZeroU32, /// The storage connection configuration. pub storage: RemoteStorageKind, } @@ -542,18 +526,6 @@ impl RemoteStorageConfig { let use_azure = container_name.is_some() && container_region.is_some(); - let max_concurrent_syncs = NonZeroUsize::new( - parse_optional_integer("max_concurrent_syncs", toml)? - .unwrap_or(DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS), - ) - .context("Failed to parse 'max_concurrent_syncs' as a positive integer")?; - - let max_sync_errors = NonZeroU32::new( - parse_optional_integer("max_sync_errors", toml)? - .unwrap_or(DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS), - ) - .context("Failed to parse 'max_sync_errors' as a positive integer")?; - let default_concurrency_limit = if use_azure { DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT } else { @@ -635,11 +607,7 @@ impl RemoteStorageConfig { } }; - Ok(Some(RemoteStorageConfig { - max_concurrent_syncs, - max_sync_errors, - storage, - })) + Ok(Some(RemoteStorageConfig { storage })) } } diff --git a/libs/remote_storage/tests/test_real_azure.rs b/libs/remote_storage/tests/test_real_azure.rs index 0338270aaf..59b92b48fb 100644 --- a/libs/remote_storage/tests/test_real_azure.rs +++ b/libs/remote_storage/tests/test_real_azure.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; use std::env; -use std::num::{NonZeroU32, NonZeroUsize}; +use std::num::NonZeroUsize; use std::ops::ControlFlow; use std::path::PathBuf; use std::sync::Arc; @@ -469,8 +469,6 @@ fn create_azure_client( let random = rand::thread_rng().gen::(); let remote_storage_config = RemoteStorageConfig { - max_concurrent_syncs: NonZeroUsize::new(100).unwrap(), - max_sync_errors: NonZeroU32::new(5).unwrap(), storage: RemoteStorageKind::AzureContainer(AzureConfig { container_name: remote_storage_azure_container, container_region: remote_storage_azure_region, diff --git a/libs/remote_storage/tests/test_real_s3.rs b/libs/remote_storage/tests/test_real_s3.rs index 7e2aa9f6d7..e2360b7533 100644 --- a/libs/remote_storage/tests/test_real_s3.rs +++ b/libs/remote_storage/tests/test_real_s3.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; use std::env; -use std::num::{NonZeroU32, NonZeroUsize}; +use std::num::NonZeroUsize; use std::ops::ControlFlow; use std::path::PathBuf; use std::sync::Arc; @@ -396,8 +396,6 @@ fn create_s3_client( let random = rand::thread_rng().gen::(); let remote_storage_config = RemoteStorageConfig { - max_concurrent_syncs: NonZeroUsize::new(100).unwrap(), - max_sync_errors: NonZeroU32::new(5).unwrap(), storage: RemoteStorageKind::AwsS3(S3Config { bucket_name: remote_storage_s3_bucket, bucket_region: remote_storage_s3_region, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 05ed9c0cee..87d9cc522e 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -1314,12 +1314,6 @@ broker_endpoint = '{broker_endpoint}' assert_eq!( parsed_remote_storage_config, RemoteStorageConfig { - max_concurrent_syncs: NonZeroUsize::new( - remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS - ) - .unwrap(), - max_sync_errors: NonZeroU32::new(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS) - .unwrap(), storage: RemoteStorageKind::LocalFs(local_storage_path.clone()), }, "Remote storage config should correctly parse the local FS config and fill other storage defaults" @@ -1380,8 +1374,6 @@ broker_endpoint = '{broker_endpoint}' assert_eq!( parsed_remote_storage_config, RemoteStorageConfig { - max_concurrent_syncs, - max_sync_errors, storage: RemoteStorageKind::AwsS3(S3Config { bucket_name: bucket_name.clone(), bucket_region: bucket_region.clone(), diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index b6f889c682..6a732d1029 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -893,14 +893,6 @@ mod test { std::fs::create_dir_all(remote_fs_dir)?; let remote_fs_dir = harness.conf.workdir.join("remote_fs").canonicalize_utf8()?; let storage_config = RemoteStorageConfig { - max_concurrent_syncs: std::num::NonZeroUsize::new( - remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS, - ) - .unwrap(), - max_sync_errors: std::num::NonZeroU32::new( - remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS, - ) - .unwrap(), storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), }; let storage = GenericRemoteStorage::from_config(&storage_config).unwrap(); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index a738633d5e..dc07ea7346 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3528,10 +3528,6 @@ pub(crate) mod harness { let remote_fs_dir = conf.workdir.join("localfs"); std::fs::create_dir_all(&remote_fs_dir).unwrap(); let config = RemoteStorageConfig { - // TODO: why not remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS, - max_concurrent_syncs: std::num::NonZeroUsize::new(2_000_000).unwrap(), - // TODO: why not remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS, - max_sync_errors: std::num::NonZeroU32::new(3_000_000).unwrap(), storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), }; let remote_storage = GenericRemoteStorage::from_config(&config).unwrap(); From ea118a238a31187248b8ca44bb77849e63bd3fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Wed, 8 Nov 2023 17:56:53 +0100 Subject: [PATCH 28/63] JWT logging improvements (#5823) * lower level on auth success from info to debug (fixes #5820) * don't log stacktraces on auth errors (as requested on slack). we do this by introducing an `AuthError` type instead of using `anyhow` and `bail`. * return errors that have been censored for improved security. --- libs/postgres_backend/src/lib.rs | 12 ++++++-- libs/utils/src/auth.rs | 52 ++++++++++++++++++++------------ libs/utils/src/http/endpoint.rs | 17 ++++++----- libs/utils/src/http/error.rs | 5 ++- pageserver/src/auth.rs | 19 ++++++------ pageserver/src/http/routes.rs | 2 ++ pageserver/src/page_service.rs | 17 ++++++----- safekeeper/src/auth.rs | 17 ++++++----- safekeeper/src/handler.rs | 27 +++++++++-------- test_runner/regress/test_auth.py | 12 ++++---- 10 files changed, 105 insertions(+), 75 deletions(-) diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 1e25c6ed6a..92a0ec7c73 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -17,7 +17,7 @@ use std::{fmt, io}; use std::{future::Future, str::FromStr}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_rustls::TlsAcceptor; -use tracing::{debug, error, info, trace}; +use tracing::{debug, error, info, trace, warn}; use pq_proto::framed::{ConnectionError, Framed, FramedReader, FramedWriter}; use pq_proto::{ @@ -35,6 +35,9 @@ pub enum QueryError { /// We were instructed to shutdown while processing the query #[error("Shutting down")] Shutdown, + /// Authentication failure + #[error("Unauthorized: {0}")] + Unauthorized(std::borrow::Cow<'static, str>), /// Some other error #[error(transparent)] Other(#[from] anyhow::Error), @@ -51,6 +54,7 @@ impl QueryError { match self { Self::Disconnected(_) => b"08006", // connection failure Self::Shutdown => SQLSTATE_ADMIN_SHUTDOWN, + Self::Unauthorized(_) => SQLSTATE_INTERNAL_ERROR, Self::Other(_) => SQLSTATE_INTERNAL_ERROR, // internal error } } @@ -610,7 +614,7 @@ impl PostgresBackend { if let Err(e) = handler.check_auth_jwt(self, jwt_response) { self.write_message_noflush(&BeMessage::ErrorResponse( - &e.to_string(), + &short_error(&e), Some(e.pg_error_code()), ))?; return Err(e); @@ -966,6 +970,7 @@ pub fn short_error(e: &QueryError) -> String { match e { QueryError::Disconnected(connection_error) => connection_error.to_string(), QueryError::Shutdown => "shutdown".to_string(), + QueryError::Unauthorized(_e) => "JWT authentication error".to_string(), QueryError::Other(e) => format!("{e:#}"), } } @@ -985,6 +990,9 @@ fn log_query_error(query: &str, e: &QueryError) { QueryError::Shutdown => { info!("query handler for '{query}' cancelled during tenant shutdown") } + QueryError::Unauthorized(e) => { + warn!("query handler for '{query}' failed with authentication error: {e}"); + } QueryError::Other(e) => { error!("query handler for '{query}' failed: {e:?}"); } diff --git a/libs/utils/src/auth.rs b/libs/utils/src/auth.rs index 6a26f17115..66b1f6e866 100644 --- a/libs/utils/src/auth.rs +++ b/libs/utils/src/auth.rs @@ -2,7 +2,7 @@ use arc_swap::ArcSwap; use serde; -use std::{fs, sync::Arc}; +use std::{borrow::Cow, fmt::Display, fs, sync::Arc}; use anyhow::Result; use camino::Utf8Path; @@ -11,7 +11,7 @@ use jsonwebtoken::{ }; use serde::{Deserialize, Serialize}; -use crate::id::TenantId; +use crate::{http::error::ApiError, id::TenantId}; /// Algorithm to use. We require EdDSA. const STORAGE_TOKEN_ALGORITHM: Algorithm = Algorithm::EdDSA; @@ -54,7 +54,7 @@ impl SwappableJwtAuth { pub fn swap(&self, jwt_auth: JwtAuth) { self.0.swap(Arc::new(jwt_auth)); } - pub fn decode(&self, token: &str) -> Result> { + pub fn decode(&self, token: &str) -> std::result::Result, AuthError> { self.0.load().decode(token) } } @@ -65,6 +65,24 @@ impl std::fmt::Debug for SwappableJwtAuth { } } +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct AuthError(pub Cow<'static, str>); + +impl Display for AuthError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for ApiError { + fn from(_value: AuthError) -> Self { + // Don't pass on the value of the AuthError as a precautionary measure. + // Being intentionally vague in public error communication hurts debugability + // but it is more secure. + ApiError::Forbidden("JWT authentication error".to_string()) + } +} + pub struct JwtAuth { decoding_keys: Vec, validation: Validation, @@ -114,7 +132,7 @@ impl JwtAuth { /// The function tries the stored decoding keys in succession, /// and returns the first yielding a successful result. /// If there is no working decoding key, it returns the last error. - pub fn decode(&self, token: &str) -> Result> { + pub fn decode(&self, token: &str) -> std::result::Result, AuthError> { let mut res = None; for decoding_key in &self.decoding_keys { res = Some(decode(token, decoding_key, &self.validation)); @@ -123,9 +141,9 @@ impl JwtAuth { } } if let Some(res) = res { - res.map_err(anyhow::Error::new) + res.map_err(|e| AuthError(Cow::Owned(e.to_string()))) } else { - anyhow::bail!("no JWT decoding keys configured") + Err(AuthError(Cow::Borrowed("no JWT decoding keys configured"))) } } } @@ -166,9 +184,9 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH "#; #[test] - fn test_decode() -> Result<(), anyhow::Error> { + fn test_decode() { let expected_claims = Claims { - tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081")?), + tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081").unwrap()), scope: Scope::Tenant, }; @@ -187,28 +205,24 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH let encoded_eddsa = "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzY29wZSI6InRlbmFudCIsInRlbmFudF9pZCI6IjNkMWY3NTk1YjQ2ODIzMDMwNGUwYjczY2VjYmNiMDgxIiwiaXNzIjoibmVvbi5jb250cm9scGxhbmUiLCJleHAiOjE3MDkyMDA4NzksImlhdCI6MTY3ODQ0MjQ3OX0.U3eA8j-uU-JnhzeO3EDHRuXLwkAUFCPxtGHEgw6p7Ccc3YRbFs2tmCdbD9PZEXP-XsxSeBQi1FY0YPcT3NXADw"; // Check it can be validated with the public key - let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?]); - let claims_from_token = auth.decode(encoded_eddsa)?.claims; + let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519).unwrap()]); + let claims_from_token = auth.decode(encoded_eddsa).unwrap().claims; assert_eq!(claims_from_token, expected_claims); - - Ok(()) } #[test] - fn test_encode() -> Result<(), anyhow::Error> { + fn test_encode() { let claims = Claims { - tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081")?), + tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081").unwrap()), scope: Scope::Tenant, }; - let encoded = encode_from_key_file(&claims, TEST_PRIV_KEY_ED25519)?; + let encoded = encode_from_key_file(&claims, TEST_PRIV_KEY_ED25519).unwrap(); // decode it back - let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?]); - let decoded = auth.decode(&encoded)?; + let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519).unwrap()]); + let decoded = auth.decode(&encoded).unwrap(); assert_eq!(decoded.claims, claims); - - Ok(()) } } diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index a8a635b6fd..550ab10700 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -1,4 +1,4 @@ -use crate::auth::{Claims, SwappableJwtAuth}; +use crate::auth::{AuthError, Claims, SwappableJwtAuth}; use crate::http::error::{api_error_handler, route_error_handler, ApiError}; use anyhow::Context; use hyper::header::{HeaderName, AUTHORIZATION}; @@ -400,9 +400,11 @@ pub fn auth_middleware( })?; let token = parse_token(header_value)?; - let data = auth - .decode(token) - .map_err(|_| ApiError::Unauthorized("malformed jwt token".to_string()))?; + let data = auth.decode(token).map_err(|err| { + warn!("Authentication error: {err}"); + // Rely on From for ApiError impl + err + })?; req.set_context(data.claims); } None => { @@ -450,12 +452,11 @@ where pub fn check_permission_with( req: &Request, - check_permission: impl Fn(&Claims) -> Result<(), anyhow::Error>, + check_permission: impl Fn(&Claims) -> Result<(), AuthError>, ) -> Result<(), ApiError> { match req.context::() { - Some(claims) => { - Ok(check_permission(&claims).map_err(|err| ApiError::Forbidden(err.to_string()))?) - } + Some(claims) => Ok(check_permission(&claims) + .map_err(|_err| ApiError::Forbidden("JWT authentication error".to_string()))?), None => Ok(()), // claims is None because auth is disabled } } diff --git a/libs/utils/src/http/error.rs b/libs/utils/src/http/error.rs index 7233d3a662..ac68b04888 100644 --- a/libs/utils/src/http/error.rs +++ b/libs/utils/src/http/error.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::error::Error as StdError; use thiserror::Error; -use tracing::{error, info}; +use tracing::{error, info, warn}; #[derive(Debug, Error)] pub enum ApiError { @@ -118,6 +118,9 @@ pub fn api_error_handler(api_error: ApiError) -> Response { // Print a stack trace for Internal Server errors match api_error { + ApiError::Forbidden(_) | ApiError::Unauthorized(_) => { + warn!("Error processing HTTP request: {api_error:#}") + } ApiError::ResourceUnavailable(_) => info!("Error processing HTTP request: {api_error:#}"), ApiError::NotFound(_) => info!("Error processing HTTP request: {api_error:#}"), ApiError::InternalServerError(_) => error!("Error processing HTTP request: {api_error:?}"), diff --git a/pageserver/src/auth.rs b/pageserver/src/auth.rs index 268117cae2..2cb661863d 100644 --- a/pageserver/src/auth.rs +++ b/pageserver/src/auth.rs @@ -1,22 +1,21 @@ -use anyhow::{bail, Result}; -use utils::auth::{Claims, Scope}; +use utils::auth::{AuthError, Claims, Scope}; use utils::id::TenantId; -pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<()> { +pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<(), AuthError> { match (&claims.scope, tenant_id) { - (Scope::Tenant, None) => { - bail!("Attempt to access management api with tenant scope. Permission denied") - } + (Scope::Tenant, None) => Err(AuthError( + "Attempt to access management api with tenant scope. Permission denied".into(), + )), (Scope::Tenant, Some(tenant_id)) => { if claims.tenant_id.unwrap() != tenant_id { - bail!("Tenant id mismatch. Permission denied") + return Err(AuthError("Tenant id mismatch. Permission denied".into())); } Ok(()) } (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope - (Scope::SafekeeperData, _) => { - bail!("SafekeeperData scope makes no sense for Pageserver") - } + (Scope::SafekeeperData, _) => Err(AuthError( + "SafekeeperData scope makes no sense for Pageserver".into(), + )), } } diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index b215b3016e..7a8f37f923 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1671,6 +1671,8 @@ where ); match handle.await { + // TODO: never actually return Err from here, always Ok(...) so that we can log + // spanned errors. Call api_error_handler instead and return appropriate Body. Ok(result) => result, Err(e) => { // The handler task panicked. We have a global panic handler that logs the diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 5193b3c5ff..5487a9d7c1 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -898,7 +898,7 @@ impl PageServerHandler { // when accessing management api supply None as an argument // when using to authorize tenant pass corresponding tenant id - fn check_permission(&self, tenant_id: Option) -> anyhow::Result<()> { + fn check_permission(&self, tenant_id: Option) -> Result<(), QueryError> { if self.auth.is_none() { // auth is set to Trust, nothing to check so just return ok return Ok(()); @@ -910,7 +910,7 @@ impl PageServerHandler { .claims .as_ref() .expect("claims presence already checked"); - check_permission(claims, tenant_id) + check_permission(claims, tenant_id).map_err(|e| QueryError::Unauthorized(e.0)) } /// Shorthand for getting a reference to a Timeline of an Active tenant. @@ -949,16 +949,17 @@ where .auth .as_ref() .unwrap() - .decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?)?; + .decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?) + .map_err(|e| QueryError::Unauthorized(e.0))?; if matches!(data.claims.scope, Scope::Tenant) && data.claims.tenant_id.is_none() { - return Err(QueryError::Other(anyhow::anyhow!( - "jwt token scope is Tenant, but tenant id is missing" - ))); + return Err(QueryError::Unauthorized( + "jwt token scope is Tenant, but tenant id is missing".into(), + )); } - info!( - "jwt auth succeeded for scope: {:#?} by tenant id: {:?}", + debug!( + "jwt scope check succeeded for scope: {:#?} by tenant id: {:?}", data.claims.scope, data.claims.tenant_id, ); diff --git a/safekeeper/src/auth.rs b/safekeeper/src/auth.rs index 2684d82365..bf4905aaa7 100644 --- a/safekeeper/src/auth.rs +++ b/safekeeper/src/auth.rs @@ -1,19 +1,20 @@ -use anyhow::{bail, Result}; -use utils::auth::{Claims, Scope}; +use utils::auth::{AuthError, Claims, Scope}; use utils::id::TenantId; -pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<()> { +pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<(), AuthError> { match (&claims.scope, tenant_id) { - (Scope::Tenant, None) => { - bail!("Attempt to access management api with tenant scope. Permission denied") - } + (Scope::Tenant, None) => Err(AuthError( + "Attempt to access management api with tenant scope. Permission denied".into(), + )), (Scope::Tenant, Some(tenant_id)) => { if claims.tenant_id.unwrap() != tenant_id { - bail!("Tenant id mismatch. Permission denied") + return Err(AuthError("Tenant id mismatch. Permission denied".into())); } Ok(()) } - (Scope::PageServerApi, _) => bail!("PageServerApi scope makes no sense for Safekeeper"), + (Scope::PageServerApi, _) => Err(AuthError( + "PageServerApi scope makes no sense for Safekeeper".into(), + )), (Scope::SafekeeperData, _) => Ok(()), } } diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index 6d0ed8650d..d5333abae6 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use std::str::{self}; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::{info, info_span, Instrument}; +use tracing::{debug, info, info_span, Instrument}; use crate::auth::check_permission; use crate::json_ctrl::{handle_json_ctrl, AppendLogicalMessage}; @@ -165,26 +165,27 @@ impl postgres_backend::Handler .auth .as_ref() .expect("auth_type is configured but .auth of handler is missing"); - let data = - auth.decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?)?; + let data = auth + .decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?) + .map_err(|e| QueryError::Unauthorized(e.0))?; // The handler might be configured to allow only tenant scope tokens. if matches!(allowed_auth_scope, Scope::Tenant) && !matches!(data.claims.scope, Scope::Tenant) { - return Err(QueryError::Other(anyhow::anyhow!( - "passed JWT token is for full access, but only tenant scope is allowed" - ))); + return Err(QueryError::Unauthorized( + "passed JWT token is for full access, but only tenant scope is allowed".into(), + )); } if matches!(data.claims.scope, Scope::Tenant) && data.claims.tenant_id.is_none() { - return Err(QueryError::Other(anyhow::anyhow!( - "jwt token scope is Tenant, but tenant id is missing" - ))); + return Err(QueryError::Unauthorized( + "jwt token scope is Tenant, but tenant id is missing".into(), + )); } - info!( - "jwt auth succeeded for scope: {:#?} by tenant id: {:?}", + debug!( + "jwt scope check succeeded for scope: {:#?} by tenant id: {:?}", data.claims.scope, data.claims.tenant_id, ); @@ -263,7 +264,7 @@ impl SafekeeperPostgresHandler { // when accessing management api supply None as an argument // when using to authorize tenant pass corresponding tenant id - fn check_permission(&self, tenant_id: Option) -> anyhow::Result<()> { + fn check_permission(&self, tenant_id: Option) -> Result<(), QueryError> { if self.auth.is_none() { // auth is set to Trust, nothing to check so just return ok return Ok(()); @@ -275,7 +276,7 @@ impl SafekeeperPostgresHandler { .claims .as_ref() .expect("claims presence already checked"); - check_permission(claims, tenant_id) + check_permission(claims, tenant_id).map_err(|e| QueryError::Unauthorized(e.0)) } async fn handle_timeline_status( diff --git a/test_runner/regress/test_auth.py b/test_runner/regress/test_auth.py index f785dc0c8e..f729bdee98 100644 --- a/test_runner/regress/test_auth.py +++ b/test_runner/regress/test_auth.py @@ -25,7 +25,7 @@ def assert_client_authorized(env: NeonEnv, http_client: PageserverHttpClient): def assert_client_not_authorized(env: NeonEnv, http_client: PageserverHttpClient): with pytest.raises( PageserverApiException, - match="Unauthorized: malformed jwt token", + match="Forbidden: JWT authentication error", ): assert_client_authorized(env, http_client) @@ -56,9 +56,7 @@ def test_pageserver_auth(neon_env_builder: NeonEnvBuilder): assert_client_authorized(env, pageserver_http_client) # fail to create branch using token with different tenant_id - with pytest.raises( - PageserverApiException, match="Forbidden: Tenant id mismatch. Permission denied" - ): + with pytest.raises(PageserverApiException, match="Forbidden: JWT authentication error"): assert_client_authorized(env, invalid_tenant_http_client) # create tenant using management token @@ -67,7 +65,7 @@ def test_pageserver_auth(neon_env_builder: NeonEnvBuilder): # fail to create tenant using tenant token with pytest.raises( PageserverApiException, - match="Forbidden: Attempt to access management api with tenant scope. Permission denied", + match="Forbidden: JWT authentication error", ): tenant_http_client.tenant_create(TenantId.generate()) @@ -94,6 +92,7 @@ def test_compute_auth_to_pageserver(neon_env_builder: NeonEnvBuilder): def test_pageserver_multiple_keys(neon_env_builder: NeonEnvBuilder): neon_env_builder.auth_enabled = True env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*Authentication error: InvalidSignature.*") env.pageserver.allowed_errors.append(".*Unauthorized: malformed jwt token.*") pageserver_token_old = env.auth_keys.generate_pageserver_token() @@ -146,6 +145,7 @@ def test_pageserver_multiple_keys(neon_env_builder: NeonEnvBuilder): def test_pageserver_key_reload(neon_env_builder: NeonEnvBuilder): neon_env_builder.auth_enabled = True env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*Authentication error: InvalidSignature.*") env.pageserver.allowed_errors.append(".*Unauthorized: malformed jwt token.*") pageserver_token_old = env.auth_keys.generate_pageserver_token() @@ -162,7 +162,7 @@ def test_pageserver_key_reload(neon_env_builder: NeonEnvBuilder): # Next attempt fails as we use the old auth token with pytest.raises( PageserverApiException, - match="Unauthorized: malformed jwt token", + match="Forbidden: JWT authentication error", ): pageserver_http_client_old.reload_auth_validation_keys() From 87389bc933e870cb10d0f88aabce80333d1792a6 Mon Sep 17 00:00:00 2001 From: Sasha Krassovsky Date: Wed, 8 Nov 2023 11:48:57 -0800 Subject: [PATCH 29/63] Add test simulating bad connection between pageserver and compute (#5728) ## Problem We have a funny 3-day timeout for connections between the compute and pageserver. We want to get rid of it, so to do that we need to make sure the compute is resilient to connection failures. Closes: https://github.com/neondatabase/neon/issues/5518 ## Summary of changes This test makes the pageserver randomly drop the connection if the failpoint is enabled, and ensures we can keep querying the pageserver. This PR also reduces the default timeout to 10 minutes from 3 days. --- libs/postgres_backend/src/lib.rs | 11 +++- pageserver/src/page_service.rs | 30 +++++++++-- test_runner/regress/test_bad_connection.py | 60 ++++++++++++++++++++++ 3 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 test_runner/regress/test_bad_connection.py diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 92a0ec7c73..1dae008a4f 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -38,6 +38,8 @@ pub enum QueryError { /// Authentication failure #[error("Unauthorized: {0}")] Unauthorized(std::borrow::Cow<'static, str>), + #[error("Simulated Connection Error")] + SimulatedConnectionError, /// Some other error #[error(transparent)] Other(#[from] anyhow::Error), @@ -52,7 +54,7 @@ impl From for QueryError { impl QueryError { pub fn pg_error_code(&self) -> &'static [u8; 5] { match self { - Self::Disconnected(_) => b"08006", // connection failure + Self::Disconnected(_) | Self::SimulatedConnectionError => b"08006", // connection failure Self::Shutdown => SQLSTATE_ADMIN_SHUTDOWN, Self::Unauthorized(_) => SQLSTATE_INTERNAL_ERROR, Self::Other(_) => SQLSTATE_INTERNAL_ERROR, // internal error @@ -736,6 +738,9 @@ impl PostgresBackend { if let Err(e) = handler.process_query(self, query_string).await { match e { QueryError::Shutdown => return Ok(ProcessMsgResult::Break), + QueryError::SimulatedConnectionError => { + return Err(QueryError::SimulatedConnectionError) + } e => { log_query_error(query_string, &e); let short_error = short_error(&e); @@ -971,6 +976,7 @@ pub fn short_error(e: &QueryError) -> String { QueryError::Disconnected(connection_error) => connection_error.to_string(), QueryError::Shutdown => "shutdown".to_string(), QueryError::Unauthorized(_e) => "JWT authentication error".to_string(), + QueryError::SimulatedConnectionError => "simulated connection error".to_string(), QueryError::Other(e) => format!("{e:#}"), } } @@ -987,6 +993,9 @@ fn log_query_error(query: &str, e: &QueryError) { QueryError::Disconnected(other_connection_error) => { error!("query handler for '{query}' failed with connection error: {other_connection_error:?}") } + QueryError::SimulatedConnectionError => { + error!("query handler for query '{query}' failed due to a simulated connection error") + } QueryError::Shutdown => { info!("query handler for '{query}' cancelled during tenant shutdown") } diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 5487a9d7c1..2201d6c86b 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -218,9 +218,27 @@ async fn page_service_conn_main( // no write timeout is used, because the kernel is assumed to error writes after some time. let mut socket = tokio_io_timeout::TimeoutReader::new(socket); - // timeout should be lower, but trying out multiple days for - // - socket.set_timeout(Some(std::time::Duration::from_secs(60 * 60 * 24 * 3))); + let default_timeout_ms = 10 * 60 * 1000; // 10 minutes by default + let socket_timeout_ms = (|| { + fail::fail_point!("simulated-bad-compute-connection", |avg_timeout_ms| { + // Exponential distribution for simulating + // poor network conditions, expect about avg_timeout_ms to be around 15 + // in tests + if let Some(avg_timeout_ms) = avg_timeout_ms { + let avg = avg_timeout_ms.parse::().unwrap() as f32; + let u = rand::random::(); + ((1.0 - u).ln() / (-avg)) as u64 + } else { + default_timeout_ms + } + }); + default_timeout_ms + })(); + + // A timeout here does not mean the client died, it can happen if it's just idle for + // a while: we will tear down this PageServerHandler and instantiate a new one if/when + // they reconnect. + socket.set_timeout(Some(std::time::Duration::from_millis(socket_timeout_ms))); let socket = std::pin::pin!(socket); // XXX: pgbackend.run() should take the connection_ctx, @@ -981,9 +999,13 @@ where pgb: &mut PostgresBackend, query_string: &str, ) -> Result<(), QueryError> { + fail::fail_point!("simulated-bad-compute-connection", |_| { + info!("Hit failpoint for bad connection"); + Err(QueryError::SimulatedConnectionError) + }); + let ctx = self.connection_ctx.attached_child(); debug!("process query {query_string:?}"); - if query_string.starts_with("pagestream ") { let (_, params_raw) = query_string.split_at("pagestream ".len()); let params = params_raw.split(' ').collect::>(); diff --git a/test_runner/regress/test_bad_connection.py b/test_runner/regress/test_bad_connection.py new file mode 100644 index 0000000000..ba0624c730 --- /dev/null +++ b/test_runner/regress/test_bad_connection.py @@ -0,0 +1,60 @@ +import random +import time + +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnvBuilder + + +def test_compute_pageserver_connection_stress(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*simulated connection error.*") + + pageserver_http = env.pageserver.http_client() + env.neon_cli.create_branch("test_compute_pageserver_connection_stress") + endpoint = env.endpoints.create_start("test_compute_pageserver_connection_stress") + + # Enable failpoint after starting everything else up so that loading initial + # basebackup doesn't fail + pageserver_http.configure_failpoints(("simulated-bad-compute-connection", "50%return(15)")) + + pg_conn = endpoint.connect() + cur = pg_conn.cursor() + + # Create table, and insert some rows. Make it big enough that it doesn't fit in + # shared_buffers, otherwise the SELECT after restart will just return answer + # from shared_buffers without hitting the page server, which defeats the point + # of this test. + cur.execute("CREATE TABLE foo (t text)") + cur.execute( + """ + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 100000) g + """ + ) + + # Verify that the table is larger than shared_buffers + cur.execute( + """ + select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_size + from pg_settings where name = 'shared_buffers' + """ + ) + row = cur.fetchone() + assert row is not None + log.info(f"shared_buffers is {row[0]}, table size {row[1]}") + assert int(row[0]) < int(row[1]) + + cur.execute("SELECT count(*) FROM foo") + assert cur.fetchone() == (100000,) + + end_time = time.time() + 30 + times_executed = 0 + while time.time() < end_time: + if random.random() < 0.5: + cur.execute("INSERT INTO foo VALUES ('stas'), ('heikki')") + else: + cur.execute("SELECT t FROM foo ORDER BY RANDOM() LIMIT 10") + cur.fetchall() + times_executed += 1 + log.info(f"Workload executed {times_executed} times") From 04957985918df76aa41328abd3826ff5ad6e5d31 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 9 Nov 2023 13:05:17 +0000 Subject: [PATCH 30/63] Fix walproposer build on aarch64 (#5827) There was a compilation error due to `std::ffi::c_char` being different type on different platforms. Clippy also complained due to a similar reason. --- libs/walproposer/src/api_bindings.rs | 2 ++ libs/walproposer/src/walproposer.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/walproposer/src/api_bindings.rs b/libs/walproposer/src/api_bindings.rs index b0ac2a879e..7f1bbc3b80 100644 --- a/libs/walproposer/src/api_bindings.rs +++ b/libs/walproposer/src/api_bindings.rs @@ -188,6 +188,7 @@ extern "C" fn recovery_download( } } +#[allow(clippy::unnecessary_cast)] extern "C" fn wal_read( sk: *mut Safekeeper, buf: *mut ::std::os::raw::c_char, @@ -421,6 +422,7 @@ impl std::fmt::Display for Level { } /// Take ownership of `Vec` from StringInfoData. +#[allow(clippy::unnecessary_cast)] pub(crate) fn take_vec_u8(pg: &mut StringInfoData) -> Option> { if pg.data.is_null() { return None; diff --git a/libs/walproposer/src/walproposer.rs b/libs/walproposer/src/walproposer.rs index 4be15344c5..0661d3a969 100644 --- a/libs/walproposer/src/walproposer.rs +++ b/libs/walproposer/src/walproposer.rs @@ -186,7 +186,7 @@ impl Wrapper { .unwrap() .into_bytes_with_nul(); assert!(safekeepers_list_vec.len() == safekeepers_list_vec.capacity()); - let safekeepers_list = safekeepers_list_vec.as_mut_ptr() as *mut i8; + let safekeepers_list = safekeepers_list_vec.as_mut_ptr() as *mut std::ffi::c_char; let callback_data = Box::into_raw(Box::new(api)) as *mut ::std::os::raw::c_void; From 9c30883c4baf629e9adead0457322189e6833001 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 9 Nov 2023 13:50:13 +0000 Subject: [PATCH 31/63] remote_storage: use S3 SDK's adaptive retry policy (#5813) ## Problem Currently, we aren't doing any explicit slowdown in response to 429 responses. Recently, as we hit remote storage a bit harder (pageserver does more ListObjectsv2 requests than it used to since #5580 ), we're seeing storms of 429 responses that may be the result of not just doing too may requests, but continuing to do those extra requests without backing off any more than our usual backoff::exponential. ## Summary of changes Switch from AWS's "Standard" retry policy to "Adaptive" -- docs describe this as experimental but it has been around for a long time. The main difference between Standard and Adaptive is that Adaptive rate-limits the client in response to feedback from the server, which is meant to avoid scenarios where the client would otherwise repeatedly hit throttling responses. --- Cargo.lock | 1 + Cargo.toml | 1 + libs/remote_storage/Cargo.toml | 1 + libs/remote_storage/src/s3_bucket.rs | 27 ++++++++++++++++++++++----- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e7fce3e8d..0a434c6ee6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4064,6 +4064,7 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-s3", + "aws-smithy-async", "aws-smithy-http", "aws-types", "azure_core", diff --git a/Cargo.toml b/Cargo.toml index 6be831a2b3..5b719d776b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ async-trait = "0.1" aws-config = { version = "0.56", default-features = false, features=["rustls"] } aws-sdk-s3 = "0.29" aws-smithy-http = "0.56" +aws-smithy-async = { version = "0.56", default-features = false, features=["rt-tokio"] } aws-credential-types = "0.56" aws-types = "0.56" axum = { version = "0.6.20", features = ["ws"] } diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index 65b5392389..d7bcce28cb 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true anyhow.workspace = true async-trait.workspace = true once_cell.workspace = true +aws-smithy-async.workspace = true aws-smithy-http.workspace = true aws-types.workspace = true aws-config.workspace = true diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 560a2c14e9..ab3fd3fe62 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -4,23 +4,27 @@ //! allowing multiple api users to independently work with the same S3 bucket, if //! their bucket prefixes are both specified and different. -use std::borrow::Cow; +use std::{borrow::Cow, sync::Arc}; use anyhow::Context; use aws_config::{ environment::credentials::EnvironmentVariableCredentialsProvider, - imds::credentials::ImdsCredentialsProvider, meta::credentials::CredentialsProviderChain, - provider_config::ProviderConfig, web_identity_token::WebIdentityTokenCredentialsProvider, + imds::credentials::ImdsCredentialsProvider, + meta::credentials::CredentialsProviderChain, + provider_config::ProviderConfig, + retry::{RetryConfigBuilder, RetryMode}, + web_identity_token::WebIdentityTokenCredentialsProvider, }; use aws_credential_types::cache::CredentialsCache; use aws_sdk_s3::{ - config::{Config, Region}, + config::{AsyncSleep, Config, Region, SharedAsyncSleep}, error::SdkError, operation::get_object::GetObjectError, primitives::ByteStream, types::{Delete, ObjectIdentifier}, Client, }; +use aws_smithy_async::rt::sleep::TokioSleep; use aws_smithy_http::body::SdkBody; use hyper::Body; use scopeguard::ScopeGuard; @@ -83,10 +87,23 @@ impl S3Bucket { .or_else("imds", ImdsCredentialsProvider::builder().build()) }; + // AWS SDK requires us to specify how the RetryConfig should sleep when it wants to back off + let sleep_impl: Arc = Arc::new(TokioSleep::new()); + + // We do our own retries (see [`backoff::retry`]). However, for the AWS SDK to enable rate limiting in response to throttling + // responses (e.g. 429 on too many ListObjectsv2 requests), we must provide a retry config. We set it to use at most one + // attempt, and enable 'Adaptive' mode, which causes rate limiting to be enabled. + let mut retry_config = RetryConfigBuilder::new(); + retry_config + .set_max_attempts(Some(1)) + .set_mode(Some(RetryMode::Adaptive)); + let mut config_builder = Config::builder() .region(region) .credentials_cache(CredentialsCache::lazy()) - .credentials_provider(credentials_provider); + .credentials_provider(credentials_provider) + .sleep_impl(SharedAsyncSleep::from(sleep_impl)) + .retry_config(retry_config.build()); if let Some(custom_endpoint) = aws_config.endpoint.clone() { config_builder = config_builder From 7cdde285a5126491a169b7eaef6dd9e222062019 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 9 Nov 2023 14:14:30 +0000 Subject: [PATCH 32/63] proxy: limit concurrent wake_compute requests per endpoint (#5799) ## Problem A user can perform many database connections at the same instant of time - these will all cache miss and materialise as requests to the control plane. #5705 ## Summary of changes I am using a `DashMap` (a sharded `RwLock`) of endpoints -> semaphores to apply a limiter. If the limiter is enabled (permits > 0), the semaphore will be retrieved per endpoint and a permit will be awaited before continuing to call the wake_compute endpoint. ### Important details This dashmap would grow uncontrollably without maintenance. It's not a cache so I don't think an LRU-based reclamation makes sense. Instead, I've made use of the sharding functionality of DashMap to lock a single shard and clear out unused semaphores periodically. I ran a test in release, using 128 tokio tasks among 12 threads each pushing 1000 entries into the map per second, clearing a shard every 2 seconds (64 second epoch with 32 shards). The endpoint names were sampled from a gamma distribution to make sure some overlap would occur, and each permit was held for 1ms. The histogram for time to clear each shard settled between 256-512us without any variance in my testing. Holding a lock for under a millisecond for 1 of the shards does not concern me as blocking --- Cargo.lock | 1 + Cargo.toml | 2 +- proxy/src/bin/proxy.rs | 18 +++- proxy/src/config.rs | 111 +++++++++++++++++++ proxy/src/console.rs | 5 + proxy/src/console/provider.rs | 166 ++++++++++++++++++++++++++++- proxy/src/console/provider/neon.rs | 29 ++++- proxy/src/proxy.rs | 1 + workspace_hack/Cargo.toml | 1 + 9 files changed, 326 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a434c6ee6..738771f88b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6483,6 +6483,7 @@ dependencies = [ "clap", "clap_builder", "crossbeam-utils", + "dashmap", "either", "fail", "futures", diff --git a/Cargo.toml b/Cargo.toml index 5b719d776b..e528489f1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ comfy-table = "6.1" const_format = "0.2" crc32c = "0.6" crossbeam-utils = "0.8.5" -dashmap = "5.5.0" +dashmap = { version = "5.5.0", features = ["raw-api"] } either = "1.8" enum-map = "2.4.2" enumset = "1.0.12" diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 28d0d95c5b..7d1b7eaaae 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -80,6 +80,9 @@ struct ProxyCliArgs { /// cache for `wake_compute` api method (use `size=0` to disable) #[clap(long, default_value = config::CacheOptions::DEFAULT_OPTIONS_NODE_INFO)] wake_compute_cache: String, + /// lock for `wake_compute` api method. example: "shards=32,permits=4,epoch=10m,timeout=1s". (use `permits=0` to disable). + #[clap(long, default_value = config::WakeComputeLockOptions::DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK)] + wake_compute_lock: String, /// Allow self-signed certificates for compute nodes (for testing) #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] allow_self_signed_compute: bool, @@ -220,10 +223,23 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { node_info: console::caches::NodeInfoCache::new("node_info_cache", size, ttl), })); + let config::WakeComputeLockOptions { + shards, + permits, + epoch, + timeout, + } = args.wake_compute_lock.parse()?; + info!(permits, shards, ?epoch, "Using NodeLocks (wake_compute)"); + let locks = Box::leak(Box::new( + console::locks::ApiLocks::new("wake_compute_lock", permits, shards, timeout) + .unwrap(), + )); + tokio::spawn(locks.garbage_collect_worker(epoch)); + let url = args.auth_endpoint.parse()?; let endpoint = http::Endpoint::new(url, http::new_client()); - let api = console::provider::neon::Api::new(endpoint, caches); + let api = console::provider::neon::Api::new(endpoint, caches, locks); auth::BackendType::Console(Cow::Owned(api), ()) } AuthBackend::Postgres => { diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 9607ecd153..bd00123905 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -264,6 +264,79 @@ impl FromStr for CacheOptions { } } +/// Helper for cmdline cache options parsing. +pub struct WakeComputeLockOptions { + /// The number of shards the lock map should have + pub shards: usize, + /// The number of allowed concurrent requests for each endpoitn + pub permits: usize, + /// Garbage collection epoch + pub epoch: Duration, + /// Lock timeout + pub timeout: Duration, +} + +impl WakeComputeLockOptions { + /// Default options for [`crate::console::provider::ApiLocks`]. + pub const DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK: &'static str = "permits=0"; + + // pub const DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK: &'static str = "shards=32,permits=4,epoch=10m,timeout=1s"; + + /// Parse lock options passed via cmdline. + /// Example: [`Self::DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK`]. + fn parse(options: &str) -> anyhow::Result { + let mut shards = None; + let mut permits = None; + let mut epoch = None; + let mut timeout = None; + + for option in options.split(',') { + let (key, value) = option + .split_once('=') + .with_context(|| format!("bad key-value pair: {option}"))?; + + match key { + "shards" => shards = Some(value.parse()?), + "permits" => permits = Some(value.parse()?), + "epoch" => epoch = Some(humantime::parse_duration(value)?), + "timeout" => timeout = Some(humantime::parse_duration(value)?), + unknown => bail!("unknown key: {unknown}"), + } + } + + // these dont matter if lock is disabled + if let Some(0) = permits { + timeout = Some(Duration::default()); + epoch = Some(Duration::default()); + shards = Some(2); + } + + let out = Self { + shards: shards.context("missing `shards`")?, + permits: permits.context("missing `permits`")?, + epoch: epoch.context("missing `epoch`")?, + timeout: timeout.context("missing `timeout`")?, + }; + + ensure!(out.shards > 1, "shard count must be > 1"); + ensure!( + out.shards.is_power_of_two(), + "shard count must be a power of two" + ); + + Ok(out) + } +} + +impl FromStr for WakeComputeLockOptions { + type Err = anyhow::Error; + + fn from_str(options: &str) -> Result { + let error = || format!("failed to parse cache lock options '{options}'"); + Self::parse(options).with_context(error) + } +} + #[cfg(test)] mod tests { use super::*; @@ -288,4 +361,42 @@ mod tests { Ok(()) } + + #[test] + fn test_parse_lock_options() -> anyhow::Result<()> { + let WakeComputeLockOptions { + epoch, + permits, + shards, + timeout, + } = "shards=32,permits=4,epoch=10m,timeout=1s".parse()?; + assert_eq!(epoch, Duration::from_secs(10 * 60)); + assert_eq!(timeout, Duration::from_secs(1)); + assert_eq!(shards, 32); + assert_eq!(permits, 4); + + let WakeComputeLockOptions { + epoch, + permits, + shards, + timeout, + } = "epoch=60s,shards=16,timeout=100ms,permits=8".parse()?; + assert_eq!(epoch, Duration::from_secs(60)); + assert_eq!(timeout, Duration::from_millis(100)); + assert_eq!(shards, 16); + assert_eq!(permits, 8); + + let WakeComputeLockOptions { + epoch, + permits, + shards, + timeout, + } = "permits=0".parse()?; + assert_eq!(epoch, Duration::ZERO); + assert_eq!(timeout, Duration::ZERO); + assert_eq!(shards, 2); + assert_eq!(permits, 0); + + Ok(()) + } } diff --git a/proxy/src/console.rs b/proxy/src/console.rs index 0e5eaaf845..6da627389e 100644 --- a/proxy/src/console.rs +++ b/proxy/src/console.rs @@ -13,5 +13,10 @@ pub mod caches { pub use super::provider::{ApiCaches, NodeInfoCache}; } +/// Various cache-related types. +pub mod locks { + pub use super::provider::ApiLocks; +} + /// Console's management API. pub mod mgmt; diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index c7cfc88c75..54bcd1f081 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -8,7 +8,13 @@ use crate::{ compute, scram, }; use async_trait::async_trait; -use std::sync::Arc; +use dashmap::DashMap; +use std::{sync::Arc, time::Duration}; +use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore}, + time::Instant, +}; +use tracing::info; pub mod errors { use crate::{ @@ -149,6 +155,9 @@ pub mod errors { #[error(transparent)] ApiError(ApiError), + + #[error("Timeout waiting to acquire wake compute lock")] + TimeoutError, } // This allows more useful interactions than `#[from]`. @@ -158,6 +167,17 @@ pub mod errors { } } + impl From for WakeComputeError { + fn from(_: tokio::sync::AcquireError) -> Self { + WakeComputeError::TimeoutError + } + } + impl From for WakeComputeError { + fn from(_: tokio::time::error::Elapsed) -> Self { + WakeComputeError::TimeoutError + } + } + impl UserFacingError for WakeComputeError { fn to_string_client(&self) -> String { use WakeComputeError::*; @@ -167,6 +187,8 @@ pub mod errors { BadComputeAddress(_) => REQUEST_FAILED.to_owned(), // However, API might return a meaningful error. ApiError(e) => e.to_string_client(), + + TimeoutError => "timeout while acquiring the compute resource lock".to_owned(), } } } @@ -233,3 +255,145 @@ pub struct ApiCaches { /// Cache for the `wake_compute` API method. pub node_info: NodeInfoCache, } + +/// Various caches for [`console`](super). +pub struct ApiLocks { + name: &'static str, + node_locks: DashMap, Arc>, + permits: usize, + timeout: Duration, + registered: prometheus::IntCounter, + unregistered: prometheus::IntCounter, + reclamation_lag: prometheus::Histogram, + lock_acquire_lag: prometheus::Histogram, +} + +impl ApiLocks { + pub fn new( + name: &'static str, + permits: usize, + shards: usize, + timeout: Duration, + ) -> prometheus::Result { + let registered = prometheus::IntCounter::with_opts( + prometheus::Opts::new( + "semaphores_registered", + "Number of semaphores registered in this api lock", + ) + .namespace(name), + )?; + prometheus::register(Box::new(registered.clone()))?; + let unregistered = prometheus::IntCounter::with_opts( + prometheus::Opts::new( + "semaphores_unregistered", + "Number of semaphores unregistered in this api lock", + ) + .namespace(name), + )?; + prometheus::register(Box::new(unregistered.clone()))?; + let reclamation_lag = prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "reclamation_lag_seconds", + "Time it takes to reclaim unused semaphores in the api lock", + ) + .namespace(name) + // 1us -> 65ms + // benchmarks on my mac indicate it's usually in the range of 256us and 512us + .buckets(prometheus::exponential_buckets(1e-6, 2.0, 16)?), + )?; + prometheus::register(Box::new(reclamation_lag.clone()))?; + let lock_acquire_lag = prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "semaphore_acquire_seconds", + "Time it takes to reclaim unused semaphores in the api lock", + ) + .namespace(name) + // 0.1ms -> 6s + .buckets(prometheus::exponential_buckets(1e-4, 2.0, 16)?), + )?; + prometheus::register(Box::new(lock_acquire_lag.clone()))?; + + Ok(Self { + name, + node_locks: DashMap::with_shard_amount(shards), + permits, + timeout, + lock_acquire_lag, + registered, + unregistered, + reclamation_lag, + }) + } + + pub async fn get_wake_compute_permit( + &self, + key: &Arc, + ) -> Result { + if self.permits == 0 { + return Ok(WakeComputePermit { permit: None }); + } + let now = Instant::now(); + let semaphore = { + // get fast path + if let Some(semaphore) = self.node_locks.get(key) { + semaphore.clone() + } else { + self.node_locks + .entry(key.clone()) + .or_insert_with(|| { + self.registered.inc(); + Arc::new(Semaphore::new(self.permits)) + }) + .clone() + } + }; + let permit = tokio::time::timeout_at(now + self.timeout, semaphore.acquire_owned()).await; + + self.lock_acquire_lag + .observe((Instant::now() - now).as_secs_f64()); + + Ok(WakeComputePermit { + permit: Some(permit??), + }) + } + + pub async fn garbage_collect_worker(&self, epoch: std::time::Duration) { + if self.permits == 0 { + return; + } + + let mut interval = tokio::time::interval(epoch / (self.node_locks.shards().len()) as u32); + loop { + for (i, shard) in self.node_locks.shards().iter().enumerate() { + interval.tick().await; + // temporary lock a single shard and then clear any semaphores that aren't currently checked out + // race conditions: if strong_count == 1, there's no way that it can increase while the shard is locked + // therefore releasing it is safe from race conditions + info!( + name = self.name, + shard = i, + "performing epoch reclamation on api lock" + ); + let mut lock = shard.write(); + let timer = self.reclamation_lag.start_timer(); + let count = lock + .extract_if(|_, semaphore| Arc::strong_count(semaphore.get_mut()) == 1) + .count(); + drop(lock); + self.unregistered.inc_by(count as u64); + timer.observe_duration() + } + } + } +} + +pub struct WakeComputePermit { + // None if the lock is disabled + permit: Option, +} + +impl WakeComputePermit { + pub fn should_check_cache(&self) -> bool { + self.permit.is_some() + } +} diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 6229840c46..0dc7c71534 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -3,12 +3,12 @@ use super::{ super::messages::{ConsoleError, GetRoleSecret, WakeCompute}, errors::{ApiError, GetAuthInfoError, WakeComputeError}, - ApiCaches, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, + ApiCaches, ApiLocks, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, }; use crate::{auth::ClientCredentials, compute, http, scram}; use async_trait::async_trait; use futures::TryFutureExt; -use std::net::SocketAddr; +use std::{net::SocketAddr, sync::Arc}; use tokio::time::Instant; use tokio_postgres::config::SslMode; use tracing::{error, info, info_span, warn, Instrument}; @@ -17,12 +17,17 @@ use tracing::{error, info, info_span, warn, Instrument}; pub struct Api { endpoint: http::Endpoint, caches: &'static ApiCaches, + locks: &'static ApiLocks, jwt: String, } impl Api { /// Construct an API object containing the auth parameters. - pub fn new(endpoint: http::Endpoint, caches: &'static ApiCaches) -> Self { + pub fn new( + endpoint: http::Endpoint, + caches: &'static ApiCaches, + locks: &'static ApiLocks, + ) -> Self { let jwt: String = match std::env::var("NEON_PROXY_TO_CONTROLPLANE_TOKEN") { Ok(v) => v, Err(_) => "".to_string(), @@ -30,6 +35,7 @@ impl Api { Self { endpoint, caches, + locks, jwt, } } @@ -163,9 +169,22 @@ impl super::Api for Api { return Ok(cached); } + let key: Arc = key.into(); + + let permit = self.locks.get_wake_compute_permit(&key).await?; + + // after getting back a permit - it's possible the cache was filled + // double check + if permit.should_check_cache() { + if let Some(cached) = self.caches.node_info.get(&key) { + info!(key = &*key, "found cached compute node info"); + return Ok(cached); + } + } + let node = self.do_wake_compute(extra, creds).await?; - let (_, cached) = self.caches.node_info.insert(key.into(), node); - info!(key = key, "created a cache entry for compute node info"); + let (_, cached) = self.caches.node_info.insert(key.clone(), node); + info!(key = &*key, "created a cache entry for compute node info"); Ok(cached) } diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 54c3503c93..a1ebf03545 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -570,6 +570,7 @@ fn report_error(e: &WakeComputeError, retry: bool) { "api_console_other_server_error" } WakeComputeError::ApiError(ApiError::Console { .. }) => "api_console_other_error", + WakeComputeError::TimeoutError => "timeout_error", }; NUM_WAKEUP_FAILURES.with_label_values(&[retry, kind]).inc(); } diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index e2a65ad150..a088f1868b 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -25,6 +25,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd clap = { version = "4", features = ["derive", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8" } +dashmap = { version = "5", default-features = false, features = ["raw-api"] } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures = { version = "0.3" } From 893616051dc611006ef6ba098d708722dba3939c Mon Sep 17 00:00:00 2001 From: Anna Stepanyan Date: Thu, 9 Nov 2023 15:24:43 +0100 Subject: [PATCH 33/63] Update epic-template.md (#5709) replace the checkbox list with a a proper task list in the epic template NB: this PR does not change the code, it only touches the github issue templates --- .github/ISSUE_TEMPLATE/epic-template.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/epic-template.md b/.github/ISSUE_TEMPLATE/epic-template.md index 7707e0aa67..019e6e7345 100644 --- a/.github/ISSUE_TEMPLATE/epic-template.md +++ b/.github/ISSUE_TEMPLATE/epic-template.md @@ -17,8 +17,9 @@ assignees: '' ## Implementation ideas -## Tasks -- [ ] +```[tasklist] +### Tasks +``` ## Other related tasks and Epics From 842223b47f923d3c9efc65ee4501ca1e723a2aaf Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 9 Nov 2023 16:40:52 +0200 Subject: [PATCH 34/63] fix(metric): remove pageserver_wal_redo_wait_seconds (#5791) the meaning of the values recorded in this histogram changed with #5560 and we never had it visualized as a histogram, just the `increase(_sum)`. The histogram is not too interesting to look at, so remove it per discussion in [slack thread](https://neondb.slack.com/archives/C063LJFF26S/p1699008316109999?thread_ts=1698852436.637559&cid=C063LJFF26S). --- pageserver/src/metrics.rs | 10 ---------- pageserver/src/walredo.rs | 9 ++------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 429ab801d9..3d11daed96 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1225,15 +1225,6 @@ pub(crate) static WAL_REDO_TIME: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); -pub(crate) static WAL_REDO_WAIT_TIME: Lazy = Lazy::new(|| { - register_histogram!( - "pageserver_wal_redo_wait_seconds", - "Time spent waiting for access to the Postgres WAL redo process", - redo_histogram_time_buckets!(), - ) - .expect("failed to define a metric") -}); - pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy = Lazy::new(|| { register_histogram!( "pageserver_wal_redo_records_histogram", @@ -1928,7 +1919,6 @@ pub fn preinitialize_metrics() { &READ_NUM_FS_LAYERS, &WAIT_LSN_TIME, &WAL_REDO_TIME, - &WAL_REDO_WAIT_TIME, &WAL_REDO_RECORDS_HISTOGRAM, &WAL_REDO_BYTES_HISTOGRAM, ] diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 38f2c64420..4a9524b5e4 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -44,7 +44,6 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use crate::config::PageServerConf; use crate::metrics::{ WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, - WAL_REDO_WAIT_TIME, }; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; use crate::repository::Key; @@ -207,11 +206,8 @@ impl PostgresRedoManager { ) -> anyhow::Result { let (rel, blknum) = key_to_rel_block(key).context("invalid record")?; const MAX_RETRY_ATTEMPTS: u32 = 1; - let start_time = Instant::now(); let mut n_attempts = 0u32; loop { - let lock_time = Instant::now(); - // launch the WAL redo process on first use let proc: Arc = { let proc_guard = self.redo_process.read().unwrap(); @@ -236,7 +232,7 @@ impl PostgresRedoManager { } }; - WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64()); + let started_at = std::time::Instant::now(); // Relational WAL records are applied using wal-redo-postgres let buf_tag = BufferTag { rel, blknum }; @@ -244,8 +240,7 @@ impl PostgresRedoManager { .apply_wal_records(buf_tag, &base_img, records, wal_redo_timeout) .context("apply_wal_records"); - let end_time = Instant::now(); - let duration = end_time.duration_since(lock_time); + let duration = started_at.elapsed(); let len = records.len(); let nbytes = records.iter().fold(0, |acumulator, record| { From 4469b1a62c869de93cb2b3261ef690db7d640189 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 9 Nov 2023 10:47:03 -0500 Subject: [PATCH 35/63] Fix blob_io test (#5818) --- pageserver/src/tenant/blob_io.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index bedf09a40c..6de2e95055 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -327,7 +327,7 @@ mod tests { let mut sz: u16 = rng.gen(); // Make 50% of the arrays small if rng.gen() { - sz |= 63; + sz &= 63; } random_array(sz.into()) }) From e0821e1eab80fc4ea147abefeb0743110dd94689 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 9 Nov 2023 16:02:59 +0000 Subject: [PATCH 36/63] pageserver: refined Timeline shutdown (#5833) ## Problem We have observed the shutdown of a timeline taking a long time when a deletion arrives at a busy time for the system. This suggests that we are not respecting cancellation tokens promptly enough. ## Summary of changes - Refactor timeline shutdown so that rather than having a shutdown() function that takes a flag for optionally flushing, there are two distinct functions, one for graceful flushing shutdown, and another that does the "normal" shutdown where we're just setting a cancellation token and then tearing down as fast as we can. This makes things a bit easier to reason about, and enables us to remove the hand-written variant of shutdown that was maintained in `delete.rs` - Layer flush task checks cancellation token more carefully - Logical size calculation's handling of cancellation tokens is simplified: rather than passing one in, it respects the Timeline's cancellation token. This PR doesn't touch RemoteTimelineClient, which will be a key thing to fix as well, so that a slow remote storage op doesn't hold up shutdown. --- libs/utils/src/seqwait.rs | 3 + pageserver/src/http/routes.rs | 6 +- pageserver/src/page_service.rs | 6 +- pageserver/src/pgdatadir_mapping.rs | 4 +- pageserver/src/tenant.rs | 10 +- pageserver/src/tenant/size.rs | 19 +- pageserver/src/tenant/timeline.rs | 223 +++++++++++------- .../src/tenant/timeline/eviction_task.rs | 16 +- 8 files changed, 164 insertions(+), 123 deletions(-) diff --git a/libs/utils/src/seqwait.rs b/libs/utils/src/seqwait.rs index 5bc7ca91d6..effc9c67b5 100644 --- a/libs/utils/src/seqwait.rs +++ b/libs/utils/src/seqwait.rs @@ -125,6 +125,9 @@ where // Wake everyone with an error. let mut internal = self.internal.lock().unwrap(); + // Block any future waiters from starting + internal.shutdown = true; + // This will steal the entire waiters map. // When we drop it all waiters will be woken. mem::take(&mut internal.waiters) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 7a8f37f923..63016042cf 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -303,11 +303,7 @@ async fn build_timeline_info( // we're executing this function, we will outlive the timeline on-disk state. info.current_logical_size_non_incremental = Some( timeline - .get_current_logical_size_non_incremental( - info.last_record_lsn, - CancellationToken::new(), - ctx, - ) + .get_current_logical_size_non_incremental(info.last_record_lsn, ctx) .await?, ); } diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 2201d6c86b..ee5f1732e4 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -512,7 +512,11 @@ impl PageServerHandler { }; if let Err(e) = &response { - if timeline.cancel.is_cancelled() { + // Requests may fail as soon as we are Stopping, even if the Timeline's cancellation token wasn't fired yet, + // because wait_lsn etc will drop out + // is_stopping(): [`Timeline::flush_and_shutdown`] has entered + // is_canceled(): [`Timeline::shutdown`]` has entered + if timeline.cancel.is_cancelled() || timeline.is_stopping() { // If we fail to fulfil a request during shutdown, which may be _because_ of // shutdown, then do not send the error to the client. Instead just drop the // connection. diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index aa4d155bcc..9e8a6b02cc 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -21,7 +21,6 @@ use serde::{Deserialize, Serialize}; use std::collections::{hash_map, HashMap, HashSet}; use std::ops::ControlFlow; use std::ops::Range; -use tokio_util::sync::CancellationToken; use tracing::{debug, trace, warn}; use utils::{bin_ser::BeSer, lsn::Lsn}; @@ -578,7 +577,6 @@ impl Timeline { pub async fn get_current_logical_size_non_incremental( &self, lsn: Lsn, - cancel: CancellationToken, ctx: &RequestContext, ) -> Result { crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id(); @@ -590,7 +588,7 @@ impl Timeline { let mut total_size: u64 = 0; for (spcnode, dbnode) in dbdir.dbdirs.keys() { for rel in self.list_rels(*spcnode, *dbnode, lsn, ctx).await? { - if cancel.is_cancelled() { + if self.cancel.is_cancelled() { return Err(CalculateLogicalSizeError::Cancelled); } let relsize_key = rel_size_to_key(rel); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index dc07ea7346..758f8b15a1 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1841,7 +1841,13 @@ impl Tenant { timelines.values().for_each(|timeline| { let timeline = Arc::clone(timeline); let span = Span::current(); - js.spawn(async move { timeline.shutdown(freeze_and_flush).instrument(span).await }); + js.spawn(async move { + if freeze_and_flush { + timeline.flush_and_shutdown().instrument(span).await + } else { + timeline.shutdown().instrument(span).await + } + }); }) }; tracing::info!("Waiting for timelines..."); @@ -4727,7 +4733,7 @@ mod tests { // Keeps uninit mark in place let raw_tline = tline.raw_timeline().unwrap(); raw_tline - .shutdown(false) + .shutdown() .instrument(info_span!("test_shutdown", tenant_id=%raw_tline.tenant_id)) .await; std::mem::forget(tline); diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index e4df94b8e9..a85dc9231c 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use anyhow::{bail, Context}; use tokio::sync::oneshot::error::RecvError; use tokio::sync::Semaphore; -use tokio_util::sync::CancellationToken; use crate::context::RequestContext; use crate::pgdatadir_mapping::CalculateLogicalSizeError; @@ -350,10 +349,6 @@ async fn fill_logical_sizes( // our advantage with `?` error handling. let mut joinset = tokio::task::JoinSet::new(); - let cancel = tokio_util::sync::CancellationToken::new(); - // be sure to cancel all spawned tasks if we are dropped - let _dg = cancel.clone().drop_guard(); - // For each point that would benefit from having a logical size available, // spawn a Task to fetch it, unless we have it cached already. for seg in segments.iter() { @@ -371,15 +366,8 @@ async fn fill_logical_sizes( let parallel_size_calcs = Arc::clone(limit); let ctx = ctx.attached_child(); joinset.spawn( - calculate_logical_size( - parallel_size_calcs, - timeline, - lsn, - cause, - ctx, - cancel.child_token(), - ) - .in_current_span(), + calculate_logical_size(parallel_size_calcs, timeline, lsn, cause, ctx) + .in_current_span(), ); } e.insert(cached_size); @@ -487,14 +475,13 @@ async fn calculate_logical_size( lsn: utils::lsn::Lsn, cause: LogicalSizeCalculationCause, ctx: RequestContext, - cancel: CancellationToken, ) -> Result { let _permit = tokio::sync::Semaphore::acquire_owned(limit) .await .expect("global semaphore should not had been closed"); let size_res = timeline - .spawn_ondemand_logical_size_calculation(lsn, cause, ctx, cancel) + .spawn_ondemand_logical_size_calculation(lsn, cause, ctx) .instrument(info_span!("spawn_ondemand_logical_size_calculation")) .await?; Ok(TimelineAtLsnSizeResult(timeline, lsn, size_res)) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 36629e0655..bbb96cb172 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -36,7 +36,6 @@ use std::time::{Duration, Instant, SystemTime}; use crate::context::{ AccessStatsBehavior, DownloadBehavior, RequestContext, RequestContextBuilder, }; -use crate::deletion_queue::DeletionQueueClient; use crate::tenant::storage_layer::delta_layer::DeltaEntry; use crate::tenant::storage_layer::{ AsLayerDesc, DeltaLayerWriter, EvictionError, ImageLayerWriter, InMemoryLayer, Layer, @@ -50,6 +49,7 @@ use crate::tenant::{ metadata::{save_metadata, TimelineMetadata}, par_fsync, }; +use crate::{deletion_queue::DeletionQueueClient, tenant::remote_timeline_client::StopError}; use crate::config::PageServerConf; use crate::keyspace::{KeyPartitioning, KeySpace, KeySpaceRandomAccum}; @@ -247,7 +247,7 @@ pub struct Timeline { /// the flush finishes. You can use that to wait for the flush to finish. layer_flush_start_tx: tokio::sync::watch::Sender, /// to be notified when layer flushing has finished, subscribe to the layer_flush_done channel - layer_flush_done_tx: tokio::sync::watch::Sender<(u64, anyhow::Result<()>)>, + layer_flush_done_tx: tokio::sync::watch::Sender<(u64, Result<(), FlushLayerError>)>, /// Layer removal lock. /// A lock to ensure that no layer of the timeline is removed concurrently by other tasks. @@ -374,6 +374,19 @@ pub enum PageReconstructError { WalRedo(anyhow::Error), } +#[derive(thiserror::Error, Debug)] +enum FlushLayerError { + /// Timeline cancellation token was cancelled + #[error("timeline shutting down")] + Cancelled, + + #[error(transparent)] + PageReconstructError(#[from] PageReconstructError), + + #[error(transparent)] + Other(#[from] anyhow::Error), +} + impl std::fmt::Debug for PageReconstructError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { match self { @@ -891,15 +904,16 @@ impl Timeline { self.launch_eviction_task(background_jobs_can_start); } + /// Graceful shutdown, may do a lot of I/O as we flush any open layers to disk and then + /// also to remote storage. This method can easily take multiple seconds for a busy timeline. + /// + /// While we are flushing, we continue to accept read I/O. #[instrument(skip_all, fields(timeline_id=%self.timeline_id))] - pub async fn shutdown(self: &Arc, freeze_and_flush: bool) { + pub(crate) async fn flush_and_shutdown(&self) { debug_assert_current_span_has_tenant_and_timeline_id(); - // Signal any subscribers to our cancellation token to drop out - tracing::debug!("Cancelling CancellationToken"); - self.cancel.cancel(); - - // prevent writes to the InMemoryLayer + // Stop ingesting data, so that we are not still writing to an InMemoryLayer while + // trying to flush tracing::debug!("Waiting for WalReceiverManager..."); task_mgr::shutdown_tasks( Some(TaskKind::WalReceiverManager), @@ -908,40 +922,70 @@ impl Timeline { ) .await; + // Since we have shut down WAL ingest, we should not let anyone start waiting for the LSN to advance + self.last_record_lsn.shutdown(); + // now all writers to InMemory layer are gone, do the final flush if requested - if freeze_and_flush { - match self.freeze_and_flush().await { - Ok(()) => {} - Err(e) => { - warn!("failed to freeze and flush: {e:#}"); - return; // TODO: should probably drain remote timeline client anyways? + match self.freeze_and_flush().await { + Ok(_) => { + // drain the upload queue + if let Some(client) = self.remote_client.as_ref() { + // if we did not wait for completion here, it might be our shutdown process + // didn't wait for remote uploads to complete at all, as new tasks can forever + // be spawned. + // + // what is problematic is the shutting down of RemoteTimelineClient, because + // obviously it does not make sense to stop while we wait for it, but what + // about corner cases like s3 suddenly hanging up? + if let Err(e) = client.wait_completion().await { + // Non-fatal. Shutdown is infallible. Failures to flush just mean that + // we have some extra WAL replay to do next time the timeline starts. + warn!("failed to flush to remote storage: {e:#}"); + } } } - - // drain the upload queue - let res = if let Some(client) = self.remote_client.as_ref() { - // if we did not wait for completion here, it might be our shutdown process - // didn't wait for remote uploads to complete at all, as new tasks can forever - // be spawned. - // - // what is problematic is the shutting down of RemoteTimelineClient, because - // obviously it does not make sense to stop while we wait for it, but what - // about corner cases like s3 suddenly hanging up? - client.wait_completion().await - } else { - Ok(()) - }; - - if let Err(e) = res { - warn!("failed to await for frozen and flushed uploads: {e:#}"); + Err(e) => { + // Non-fatal. Shutdown is infallible. Failures to flush just mean that + // we have some extra WAL replay to do next time the timeline starts. + warn!("failed to freeze and flush: {e:#}"); } } + self.shutdown().await; + } + + /// Shut down immediately, without waiting for any open layers to flush to disk. This is a subset of + /// the graceful [`Timeline::flush_and_shutdown`] function. + pub(crate) async fn shutdown(&self) { + // Signal any subscribers to our cancellation token to drop out + tracing::debug!("Cancelling CancellationToken"); + self.cancel.cancel(); + // Page request handlers might be waiting for LSN to advance: they do not respect Timeline::cancel // while doing so. self.last_record_lsn.shutdown(); + // Shut down the layer flush task before the remote client, as one depends on the other + task_mgr::shutdown_tasks( + Some(TaskKind::LayerFlushTask), + Some(self.tenant_id), + Some(self.timeline_id), + ) + .await; + + // Shut down remote timeline client: this gracefully moves its metadata into its Stopping state in + // case our caller wants to use that for a deletion + if let Some(remote_client) = self.remote_client.as_ref() { + match remote_client.stop() { + Ok(()) => {} + Err(StopError::QueueUninitialized) => { + // Shutting down during initialization is legal + } + } + } + tracing::debug!("Waiting for tasks..."); + task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(self.timeline_id)).await; // Finally wait until any gate-holders are complete @@ -985,7 +1029,12 @@ impl Timeline { reason, backtrace: backtrace_str, }; - self.set_state(broken_state) + self.set_state(broken_state); + + // Although the Broken state is not equivalent to shutdown() (shutdown will be called + // later when this tenant is detach or the process shuts down), firing the cancellation token + // here avoids the need for other tasks to watch for the Broken state explicitly. + self.cancel.cancel(); } pub fn current_state(&self) -> TimelineState { @@ -1741,12 +1790,8 @@ impl Timeline { // delay will be terminated by a timeout regardless. let _completion = { self_clone.initial_logical_size_attempt.lock().expect("unexpected initial_logical_size_attempt poisoned").take() }; - // no extra cancellation here, because nothing really waits for this to complete compared - // to spawn_ondemand_logical_size_calculation. - let cancel = CancellationToken::new(); - let calculated_size = match self_clone - .logical_size_calculation_task(lsn, LogicalSizeCalculationCause::Initial, &background_ctx, cancel) + .logical_size_calculation_task(lsn, LogicalSizeCalculationCause::Initial, &background_ctx) .await { Ok(s) => s, @@ -1815,7 +1860,6 @@ impl Timeline { lsn: Lsn, cause: LogicalSizeCalculationCause, ctx: RequestContext, - cancel: CancellationToken, ) -> oneshot::Receiver> { let (sender, receiver) = oneshot::channel(); let self_clone = Arc::clone(self); @@ -1836,7 +1880,7 @@ impl Timeline { false, async move { let res = self_clone - .logical_size_calculation_task(lsn, cause, &ctx, cancel) + .logical_size_calculation_task(lsn, cause, &ctx) .await; let _ = sender.send(res).ok(); Ok(()) // Receiver is responsible for handling errors @@ -1852,58 +1896,28 @@ impl Timeline { lsn: Lsn, cause: LogicalSizeCalculationCause, ctx: &RequestContext, - cancel: CancellationToken, ) -> Result { span::debug_assert_current_span_has_tenant_and_timeline_id(); - let mut timeline_state_updates = self.subscribe_for_state_updates(); + let _guard = self.gate.enter(); + let self_calculation = Arc::clone(self); let mut calculation = pin!(async { - let cancel = cancel.child_token(); let ctx = ctx.attached_child(); self_calculation - .calculate_logical_size(lsn, cause, cancel, &ctx) + .calculate_logical_size(lsn, cause, &ctx) .await }); - let timeline_state_cancellation = async { - loop { - match timeline_state_updates.changed().await { - Ok(()) => { - let new_state = timeline_state_updates.borrow().clone(); - match new_state { - // we're running this job for active timelines only - TimelineState::Active => continue, - TimelineState::Broken { .. } - | TimelineState::Stopping - | TimelineState::Loading => { - break format!("aborted because timeline became inactive (new state: {new_state:?})") - } - } - } - Err(_sender_dropped_error) => { - // can't happen, the sender is not dropped as long as the Timeline exists - break "aborted because state watch was dropped".to_string(); - } - } - } - }; - - let taskmgr_shutdown_cancellation = async { - task_mgr::shutdown_watcher().await; - "aborted because task_mgr shutdown requested".to_string() - }; tokio::select! { res = &mut calculation => { res } - reason = timeline_state_cancellation => { - debug!(reason = reason, "cancelling calculation"); - cancel.cancel(); + _ = self.cancel.cancelled() => { + debug!("cancelling logical size calculation for timeline shutdown"); calculation.await } - reason = taskmgr_shutdown_cancellation => { - debug!(reason = reason, "cancelling calculation"); - cancel.cancel(); + _ = task_mgr::shutdown_watcher() => { + debug!("cancelling logical size calculation for task shutdown"); calculation.await } } @@ -1917,7 +1931,6 @@ impl Timeline { &self, up_to_lsn: Lsn, cause: LogicalSizeCalculationCause, - cancel: CancellationToken, ctx: &RequestContext, ) -> Result { info!( @@ -1960,7 +1973,7 @@ impl Timeline { }; let timer = storage_time_metrics.start_timer(); let logical_size = self - .get_current_logical_size_non_incremental(up_to_lsn, cancel, ctx) + .get_current_logical_size_non_incremental(up_to_lsn, ctx) .await?; debug!("calculated logical size: {logical_size}"); timer.stop_and_record(); @@ -2373,6 +2386,10 @@ impl Timeline { info!("started flush loop"); loop { tokio::select! { + _ = self.cancel.cancelled() => { + info!("shutting down layer flush task"); + break; + }, _ = task_mgr::shutdown_watcher() => { info!("shutting down layer flush task"); break; @@ -2384,6 +2401,14 @@ impl Timeline { let timer = self.metrics.flush_time_histo.start_timer(); let flush_counter = *layer_flush_start_rx.borrow(); let result = loop { + if self.cancel.is_cancelled() { + info!("dropping out of flush loop for timeline shutdown"); + // Note: we do not bother transmitting into [`layer_flush_done_tx`], because + // anyone waiting on that will respect self.cancel as well: they will stop + // waiting at the same time we as drop out of this loop. + return; + } + let layer_to_flush = { let guard = self.layers.read().await; guard.layer_map().frozen_layers.front().cloned() @@ -2392,9 +2417,18 @@ impl Timeline { let Some(layer_to_flush) = layer_to_flush else { break Ok(()); }; - if let Err(err) = self.flush_frozen_layer(layer_to_flush, ctx).await { - error!("could not flush frozen layer: {err:?}"); - break Err(err); + match self.flush_frozen_layer(layer_to_flush, ctx).await { + Ok(()) => {} + Err(FlushLayerError::Cancelled) => { + info!("dropping out of flush loop for timeline shutdown"); + return; + } + err @ Err( + FlushLayerError::Other(_) | FlushLayerError::PageReconstructError(_), + ) => { + error!("could not flush frozen layer: {err:?}"); + break err; + } } }; // Notify any listeners that we're done @@ -2443,7 +2477,17 @@ impl Timeline { } } trace!("waiting for flush to complete"); - rx.changed().await?; + tokio::select! { + rx_e = rx.changed() => { + rx_e?; + }, + // Cancellation safety: we are not leaving an I/O in-flight for the flush, we're just ignoring + // the notification from [`flush_loop`] that it completed. + _ = self.cancel.cancelled() => { + tracing::info!("Cancelled layer flush due on timeline shutdown"); + return Ok(()) + } + }; trace!("done") } } @@ -2458,7 +2502,7 @@ impl Timeline { self: &Arc, frozen_layer: Arc, ctx: &RequestContext, - ) -> anyhow::Result<()> { + ) -> Result<(), FlushLayerError> { // As a special case, when we have just imported an image into the repository, // instead of writing out a L0 delta layer, we directly write out image layer // files instead. This is possible as long as *all* the data imported into the @@ -2483,6 +2527,11 @@ impl Timeline { let (partitioning, _lsn) = self .repartition(self.initdb_lsn, self.get_compaction_target_size(), ctx) .await?; + + if self.cancel.is_cancelled() { + return Err(FlushLayerError::Cancelled); + } + // For image layers, we add them immediately into the layer map. ( self.create_image_layers(&partitioning, self.initdb_lsn, true, ctx) @@ -2514,6 +2563,10 @@ impl Timeline { ) }; + if self.cancel.is_cancelled() { + return Err(FlushLayerError::Cancelled); + } + let disk_consistent_lsn = Lsn(lsn_range.end.0 - 1); let old_disk_consistent_lsn = self.disk_consistent_lsn.load(); @@ -2523,6 +2576,10 @@ impl Timeline { let metadata = { let mut guard = self.layers.write().await; + if self.cancel.is_cancelled() { + return Err(FlushLayerError::Cancelled); + } + guard.finish_flush_l0_layer(delta_layer_to_add.as_ref(), &frozen_layer, &self.metrics); if disk_consistent_lsn != old_disk_consistent_lsn { diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index e3aad22e40..183fcb872f 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -326,8 +326,7 @@ impl Timeline { match state.last_layer_access_imitation { Some(ts) if ts.elapsed() < inter_imitate_period => { /* no need to run */ } _ => { - self.imitate_timeline_cached_layer_accesses(cancel, ctx) - .await; + self.imitate_timeline_cached_layer_accesses(ctx).await; state.last_layer_access_imitation = Some(tokio::time::Instant::now()) } } @@ -367,21 +366,12 @@ impl Timeline { /// Recompute the values which would cause on-demand downloads during restart. #[instrument(skip_all)] - async fn imitate_timeline_cached_layer_accesses( - &self, - cancel: &CancellationToken, - ctx: &RequestContext, - ) { + async fn imitate_timeline_cached_layer_accesses(&self, ctx: &RequestContext) { let lsn = self.get_last_record_lsn(); // imitiate on-restart initial logical size let size = self - .calculate_logical_size( - lsn, - LogicalSizeCalculationCause::EvictionTaskImitation, - cancel.clone(), - ctx, - ) + .calculate_logical_size(lsn, LogicalSizeCalculationCause::EvictionTaskImitation, ctx) .instrument(info_span!("calculate_logical_size")) .await; From f95f001b8bc19ec0697f47bff4a0a0b7c2ce5c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 9 Nov 2023 17:12:18 +0100 Subject: [PATCH 37/63] Lsn for get_timestamp_of_lsn should be string, not integer (#5840) The `get_timestamp_of_lsn` pageserver endpoint has been added in #5497, but the yml it added was wrong: the lsn is expected in hex format, not in integer (decimal) format. --- pageserver/src/http/openapi_spec.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 9bff0fd668..4d455243f0 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -352,7 +352,8 @@ paths: in: query required: true schema: - type: integer + type: string + format: hex description: A LSN to get the timestamp responses: "200": From f5344fb85a3e8f1fbf0352c12cb378cef5d20bcc Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 9 Nov 2023 23:31:53 +0200 Subject: [PATCH 38/63] temp: log all layer loading errors while we lose them (#5816) Temporary workaround while some errors are not being logged. Cc: #5815. --- pageserver/src/tenant/storage_layer/layer.rs | 6 +++++- test_runner/regress/test_broken_timeline.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index d72982a9a0..17df39733f 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -251,6 +251,7 @@ impl Layer { layer .get_value_reconstruct_data(key, lsn_range, reconstruct_data, &self.0, ctx) + .instrument(tracing::info_span!("get_value_reconstruct_data", layer=%self)) .await } @@ -1211,8 +1212,10 @@ impl DownloadedLayer { // this will be a permanent failure .context("load layer"); - if res.is_err() { + if let Err(e) = res.as_ref() { LAYER_IMPL_METRICS.inc_permanent_loading_failures(); + // TODO(#5815): we are not logging all errors, so temporarily log them here as well + tracing::error!("layer loading failed permanently: {e:#}"); } res }; @@ -1291,6 +1294,7 @@ impl ResidentLayer { } /// Loads all keys stored in the layer. Returns key, lsn and value size. + #[tracing::instrument(skip_all, fields(layer=%self))] pub(crate) async fn load_keys<'a>( &'a self, ctx: &RequestContext, diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index b1b47b3f2c..23839a4dd1 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -26,6 +26,7 @@ def test_local_corruption(neon_env_builder: NeonEnvBuilder): ".*will not become active. Current state: Broken.*", ".*failed to load metadata.*", ".*load failed.*load local timeline.*", + ".*layer loading failed permanently: load layer: .*", ] ) From 8dd29f1e27ffd3652727ebd1aed36fbccf9b77ad Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 9 Nov 2023 23:36:57 +0200 Subject: [PATCH 39/63] fix(pageserver): spawn all kinds of tenant shutdowns (#5841) Minor bugfix, something noticed while manual code-review. Use the same joinset for inprogress tenants so we can get the benefit of the buffering logging just as we get for attached tenants, and no single inprogress task can hold up shutdown of other tenants. --- pageserver/src/tenant/mgr.rs | 91 ++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 50 deletions(-) diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 07d1618272..c71eac0672 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -566,8 +566,10 @@ pub(crate) async fn shutdown_all_tenants() { async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { use utils::completion; - // Atomically, 1. extract the list of tenants to shut down and 2. prevent creation of new tenants. - let (in_progress_ops, tenants_to_shut_down) = { + let mut join_set = JoinSet::new(); + + // Atomically, 1. create the shutdown tasks and 2. prevent creation of new tenants. + let (total_in_progress, total_attached) = { let mut m = tenants.write().unwrap(); match &mut *m { TenantsMap::Initializing => { @@ -577,78 +579,67 @@ async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { } TenantsMap::Open(tenants) => { let mut shutdown_state = HashMap::new(); - let mut in_progress_ops = Vec::new(); - let mut tenants_to_shut_down = Vec::new(); + let mut total_in_progress = 0; + let mut total_attached = 0; - for (k, v) in tenants.drain() { + for (tenant_id, v) in tenants.drain() { match v { TenantSlot::Attached(t) => { - tenants_to_shut_down.push(t.clone()); - shutdown_state.insert(k, TenantSlot::Attached(t)); + shutdown_state.insert(tenant_id, TenantSlot::Attached(t.clone())); + join_set.spawn( + async move { + let freeze_and_flush = true; + + let res = { + let (_guard, shutdown_progress) = completion::channel(); + t.shutdown(shutdown_progress, freeze_and_flush).await + }; + + if let Err(other_progress) = res { + // join the another shutdown in progress + other_progress.wait().await; + } + + // we cannot afford per tenant logging here, because if s3 is degraded, we are + // going to log too many lines + debug!("tenant successfully stopped"); + } + .instrument(info_span!("shutdown", %tenant_id)), + ); + + total_attached += 1; } TenantSlot::Secondary => { - shutdown_state.insert(k, TenantSlot::Secondary); + shutdown_state.insert(tenant_id, TenantSlot::Secondary); } TenantSlot::InProgress(notify) => { // InProgress tenants are not visible in TenantsMap::ShuttingDown: we will // wait for their notifications to fire in this function. - in_progress_ops.push(notify); + join_set.spawn(async move { + notify.wait().await; + }); + + total_in_progress += 1; } } } *m = TenantsMap::ShuttingDown(shutdown_state); - (in_progress_ops, tenants_to_shut_down) + (total_in_progress, total_attached) } TenantsMap::ShuttingDown(_) => { - // TODO: it is possible that detach and shutdown happen at the same time. as a - // result, during shutdown we do not wait for detach. error!("already shutting down, this function isn't supposed to be called more than once"); return; } } }; + let started_at = std::time::Instant::now(); + info!( "Waiting for {} InProgress tenants and {} Attached tenants to shut down", - in_progress_ops.len(), - tenants_to_shut_down.len() + total_in_progress, total_attached ); - for barrier in in_progress_ops { - barrier.wait().await; - } - - info!( - "InProgress tenants shut down, waiting for {} Attached tenants to shut down", - tenants_to_shut_down.len() - ); - let started_at = std::time::Instant::now(); - let mut join_set = JoinSet::new(); - for tenant in tenants_to_shut_down { - let tenant_id = tenant.get_tenant_id(); - join_set.spawn( - async move { - let freeze_and_flush = true; - - let res = { - let (_guard, shutdown_progress) = completion::channel(); - tenant.shutdown(shutdown_progress, freeze_and_flush).await - }; - - if let Err(other_progress) = res { - // join the another shutdown in progress - other_progress.wait().await; - } - - // we cannot afford per tenant logging here, because if s3 is degraded, we are - // going to log too many lines - - debug!("tenant successfully stopped"); - } - .instrument(info_span!("shutdown", %tenant_id)), - ); - } - let total = join_set.len(); let mut panicked = 0; let mut buffering = true; @@ -661,7 +652,7 @@ async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { match joined { Ok(()) => {} Err(join_error) if join_error.is_cancelled() => { - unreachable!("we are not cancelling any of the futures"); + unreachable!("we are not cancelling any of the tasks"); } Err(join_error) if join_error.is_panic() => { // cannot really do anything, as this panic is likely a bug From 8e5e3971ba4f05f17b17fd3471b9eb839a4da0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 10 Nov 2023 13:38:44 +0100 Subject: [PATCH 40/63] find_lsn_for_timestamp fixes (#5844) Includes the changes of #3689 that address point 1 of #3689, plus some further improvements. In particular, this PR does: * set `min_lsn` to a safe value to create branches from (and verify it in tests) * return `min_lsn` instead of `max_lsn` for `NoData` and `Past` (verify it in test for `Past`, `NoData` is harder and not as important) * return `commit_lsn` instead of `max_lsn` for Future (and verify it in the tests) * add some comments Split out of #5686 to get something more minimal out to users. --- pageserver/src/pgdatadir_mapping.rs | 62 ++++++++++++++++++------- test_runner/regress/test_lsn_mapping.py | 62 +++++++++++++++++++++---- 2 files changed, 98 insertions(+), 26 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 9e8a6b02cc..09feba9b68 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -29,9 +29,33 @@ pub type BlockNumber = u32; #[derive(Debug)] pub enum LsnForTimestamp { + /// Found commits both before and after the given timestamp Present(Lsn), + + /// Found no commits after the given timestamp, this means + /// that the newest data in the branch is older than the given + /// timestamp. + /// + /// All commits <= LSN happened before the given timestamp Future(Lsn), + + /// The queried timestamp is past our horizon we look back at (PITR) + /// + /// All commits > LSN happened after the given timestamp, + /// but any commits < LSN might have happened before or after + /// the given timestamp. We don't know because no data before + /// the given lsn is available. Past(Lsn), + + /// We have found no commit with a timestamp, + /// so we can't return anything meaningful. + /// + /// The associated LSN is the lower bound value we can safely + /// create branches on, but no statement is made if it is + /// older or newer than the timestamp. + /// + /// This variant can e.g. be returned right after a + /// cluster import. NoData(Lsn), } @@ -324,7 +348,11 @@ impl Timeline { ctx: &RequestContext, ) -> Result { let gc_cutoff_lsn_guard = self.get_latest_gc_cutoff_lsn(); - let min_lsn = *gc_cutoff_lsn_guard; + // We use this method to figure out the branching LSN for the new branch, but the + // GC cutoff could be before the branching point and we cannot create a new branch + // with LSN < `ancestor_lsn`. Thus, pick the maximum of these two to be + // on the safe side. + let min_lsn = std::cmp::max(*gc_cutoff_lsn_guard, self.get_ancestor_lsn()); let max_lsn = self.get_last_record_lsn(); // LSNs are always 8-byte aligned. low/mid/high represent the @@ -354,30 +382,32 @@ impl Timeline { low = mid + 1; } } + // If `found_smaller == true`, `low` is the LSN of the last commit record + // before or at `search_timestamp` + // + // FIXME: it would be better to get the LSN of the previous commit. + // Otherwise, if you restore to the returned LSN, the database will + // include physical changes from later commits that will be marked + // as aborted, and will need to be vacuumed away. + let commit_lsn = Lsn((low - 1) * 8); match (found_smaller, found_larger) { (false, false) => { // This can happen if no commit records have been processed yet, e.g. // just after importing a cluster. - Ok(LsnForTimestamp::NoData(max_lsn)) - } - (true, false) => { - // Didn't find any commit timestamps larger than the request - Ok(LsnForTimestamp::Future(max_lsn)) + Ok(LsnForTimestamp::NoData(min_lsn)) } (false, true) => { // Didn't find any commit timestamps smaller than the request - Ok(LsnForTimestamp::Past(max_lsn)) + Ok(LsnForTimestamp::Past(min_lsn)) } - (true, true) => { - // low is the LSN of the first commit record *after* the search_timestamp, - // Back off by one to get to the point just before the commit. - // - // FIXME: it would be better to get the LSN of the previous commit. - // Otherwise, if you restore to the returned LSN, the database will - // include physical changes from later commits that will be marked - // as aborted, and will need to be vacuumed away. - Ok(LsnForTimestamp::Present(Lsn((low - 1) * 8))) + (true, false) => { + // Only found commits with timestamps smaller than the request. + // It's still a valid case for branch creation, return it. + // And `update_gc_info()` ignores LSN for a `LsnForTimestamp::Future` + // case, anyway. + Ok(LsnForTimestamp::Future(commit_lsn)) } + (true, true) => Ok(LsnForTimestamp::Present(commit_lsn)), } } diff --git a/test_runner/regress/test_lsn_mapping.py b/test_runner/regress/test_lsn_mapping.py index 726bfa5f29..f79c1c347c 100644 --- a/test_runner/regress/test_lsn_mapping.py +++ b/test_runner/regress/test_lsn_mapping.py @@ -79,13 +79,32 @@ def test_lsn_mapping_old(neon_env_builder: NeonEnvBuilder): def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() - new_timeline_id = env.neon_cli.create_branch("test_lsn_mapping") - endpoint_main = env.endpoints.create_start("test_lsn_mapping") - log.info("postgres is running on 'test_lsn_mapping' branch") + tenant_id, _ = env.neon_cli.create_tenant( + conf={ + # disable default GC and compaction + "gc_period": "1000 m", + "compaction_period": "0 s", + "gc_horizon": f"{1024 ** 2}", + "checkpoint_distance": f"{1024 ** 2}", + "compaction_target_size": f"{1024 ** 2}", + } + ) + + timeline_id = env.neon_cli.create_branch("test_lsn_mapping", tenant_id=tenant_id) + endpoint_main = env.endpoints.create_start("test_lsn_mapping", tenant_id=tenant_id) + timeline_id = endpoint_main.safe_psql("show neon.timeline_id")[0][0] + log.info("postgres is running on 'main' branch") cur = endpoint_main.connect().cursor() + + # Obtain an lsn before all write operations on this branch + start_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_lsn()")) + # Create table, and insert rows, each in a separate transaction # Disable synchronous_commit to make this initialization go faster. + # Disable `synchronous_commit` to make this initialization go faster. + # XXX: on my laptop this test takes 7s, and setting `synchronous_commit=off` + # doesn't change anything. # # Each row contains current insert LSN and the current timestamp, when # the row was inserted. @@ -104,40 +123,63 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): cur.execute("INSERT INTO foo VALUES (-1)") # Wait until WAL is received by pageserver - wait_for_last_flush_lsn(env, endpoint_main, env.initial_tenant, new_timeline_id) + last_flush_lsn = wait_for_last_flush_lsn(env, endpoint_main, tenant_id, timeline_id) with env.pageserver.http_client() as client: - # Check edge cases: timestamp in the future + # Check edge cases + # Timestamp is in the future probe_timestamp = tbl[-1][1] + timedelta(hours=1) result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) assert result["kind"] == "future" + # make sure that we return a well advanced lsn here + assert Lsn(result["lsn"]) > start_lsn - # timestamp too the far history + # Timestamp is in the unreachable past probe_timestamp = tbl[0][1] - timedelta(hours=10) result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) assert result["kind"] == "past" + # make sure that we return the minimum lsn here at the start of the range + assert Lsn(result["lsn"]) < start_lsn # Probe a bunch of timestamps in the valid range for i in range(1, len(tbl), 100): probe_timestamp = tbl[i][1] result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) + assert result["kind"] not in ["past", "nodata"] lsn = result["lsn"] # Call get_lsn_by_timestamp to get the LSN # Launch a new read-only node at that LSN, and check that only the rows # that were supposed to be committed at that point in time are visible. endpoint_here = env.endpoints.create_start( - branch_name="test_lsn_mapping", endpoint_id="ep-lsn_mapping_read", lsn=lsn + branch_name="test_lsn_mapping", + endpoint_id="ep-lsn_mapping_read", + lsn=lsn, + tenant_id=tenant_id, ) assert endpoint_here.safe_psql("SELECT max(x) FROM foo")[0][0] == i endpoint_here.stop_and_destroy() + # Do the "past" check again at a new branch to ensure that we don't return something before the branch cutoff + timeline_id_child = env.neon_cli.create_branch( + "test_lsn_mapping_child", tenant_id=tenant_id, ancestor_branch_name="test_lsn_mapping" + ) + + # Timestamp is in the unreachable past + probe_timestamp = tbl[0][1] - timedelta(hours=10) + result = client.timeline_get_lsn_by_timestamp( + tenant_id, timeline_id_child, f"{probe_timestamp.isoformat()}Z", 2 + ) + assert result["kind"] == "past" + # make sure that we return the minimum lsn here at the start of the range + assert Lsn(result["lsn"]) >= last_flush_lsn + # Test pageserver get_timestamp_of_lsn API def test_ts_of_lsn_api(neon_env_builder: NeonEnvBuilder): From 6e145a44fa6934021326448fee610a96ebbffcd2 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Fri, 10 Nov 2023 12:45:41 +0000 Subject: [PATCH 41/63] workflows/neon_extra_builds: run check-codestyle-rust & build-neon on arm64 (#5832) ## Problem Some developers use workstations with arm CPUs, and sometimes x86-64 code is not fully compatible with it (for example, https://github.com/neondatabase/neon/pull/5827). Although we don't have arm CPUs in the prod (yet?), it is worth having some basic checks for this architecture to have a better developer experience. Closes https://github.com/neondatabase/neon/issues/5829 ## Summary of changes - Run `check-codestyle-rust`-like & `build-neon`-like jobs on Arm runner - Add `run-extra-build-*` label to run all available extra builds --- .github/actionlint.yml | 2 + .github/workflows/neon_extra_builds.yml | 181 +++++++++++++++++++++++- 2 files changed, 181 insertions(+), 2 deletions(-) diff --git a/.github/actionlint.yml b/.github/actionlint.yml index e2ece5f230..362480f256 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -1,5 +1,7 @@ self-hosted-runner: labels: + - arm64 + - dev - gen3 - large - small diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index d7f5295c5b..0d7db8dfbc 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -21,7 +21,10 @@ env: jobs: check-macos-build: - if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') + if: | + contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' timeout-minutes: 90 runs-on: macos-latest @@ -112,8 +115,182 @@ jobs: - name: Check that no warnings are produced run: ./run_clippy.sh + check-linux-arm-build: + timeout-minutes: 90 + runs-on: [ self-hosted, dev, arm64 ] + + env: + # Use release build only, to have less debug info around + # Hence keeping target/ (and general cache size) smaller + BUILD_TYPE: release + CARGO_FEATURES: --features testing + CARGO_FLAGS: --locked --release + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} + + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init + + steps: + - name: Fix git ownership + run: | + # Workaround for `fatal: detected dubious ownership in repository at ...` + # + # Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers + # Ref https://github.com/actions/checkout/issues/785 + # + git config --global --add safe.directory ${{ github.workspace }} + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 1 + + - name: Set pg 14 revision for caching + id: pg_v14_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT + + - name: Set pg 15 revision for caching + id: pg_v15_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT + + - name: Set pg 16 revision for caching + id: pg_v16_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT + + - name: Set env variables + run: | + echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo" >> $GITHUB_ENV + + - name: Cache postgres v14 build + id: cache_pg_14 + uses: actions/cache@v3 + with: + path: pg_install/v14 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Cache postgres v15 build + id: cache_pg_15 + uses: actions/cache@v3 + with: + path: pg_install/v15 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Cache postgres v16 build + id: cache_pg_16 + uses: actions/cache@v3 + with: + path: pg_install/v16 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Build postgres v14 + if: steps.cache_pg_14.outputs.cache-hit != 'true' + run: mold -run make postgres-v14 -j$(nproc) + + - name: Build postgres v15 + if: steps.cache_pg_15.outputs.cache-hit != 'true' + run: mold -run make postgres-v15 -j$(nproc) + + - name: Build postgres v16 + if: steps.cache_pg_16.outputs.cache-hit != 'true' + run: mold -run make postgres-v16 -j$(nproc) + + - name: Build neon extensions + run: mold -run make neon-pg-ext -j$(nproc) + + - name: Build walproposer-lib + run: mold -run make walproposer-lib -j$(nproc) + + - name: Run cargo build + run: | + mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests + + - name: Run cargo test + run: | + cargo test $CARGO_FLAGS $CARGO_FEATURES + + # Run separate tests for real S3 + export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty + export REMOTE_STORAGE_S3_BUCKET=neon-github-public-dev + export REMOTE_STORAGE_S3_REGION=eu-central-1 + # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now + cargo test $CARGO_FLAGS --package remote_storage --test test_real_s3 + + # Run separate tests for real Azure Blob Storage + # XXX: replace region with `eu-central-1`-like region + export ENABLE_REAL_AZURE_REMOTE_STORAGE=y + export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}" + export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}" + export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}" + export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}" + # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now + cargo test $CARGO_FLAGS --package remote_storage --test test_real_azure + + check-codestyle-rust-arm: + timeout-minutes: 90 + runs-on: [ self-hosted, dev, arm64 ] + + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 1 + + # Some of our rust modules use FFI and need those to be checked + - name: Get postgres headers + run: make postgres-headers -j$(nproc) + + # cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations. + # This will catch compiler & clippy warnings in all feature combinations. + # TODO: use cargo hack for build and test as well, but, that's quite expensive. + # NB: keep clippy args in sync with ./run_clippy.sh + - run: | + CLIPPY_COMMON_ARGS="$( source .neon_clippy_args; echo "$CLIPPY_COMMON_ARGS")" + if [ "$CLIPPY_COMMON_ARGS" = "" ]; then + echo "No clippy args found in .neon_clippy_args" + exit 1 + fi + echo "CLIPPY_COMMON_ARGS=${CLIPPY_COMMON_ARGS}" >> $GITHUB_ENV + - name: Run cargo clippy (debug) + run: cargo hack --feature-powerset clippy $CLIPPY_COMMON_ARGS + - name: Run cargo clippy (release) + run: cargo hack --feature-powerset clippy --release $CLIPPY_COMMON_ARGS + + - name: Check documentation generation + run: cargo doc --workspace --no-deps --document-private-items + env: + RUSTDOCFLAGS: "-Dwarnings -Arustdoc::private_intra_doc_links" + + # Use `${{ !cancelled() }}` to run quck tests after the longer clippy run + - name: Check formatting + if: ${{ !cancelled() }} + run: cargo fmt --all -- --check + + # https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci + - name: Check rust dependencies + if: ${{ !cancelled() }} + run: | + cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date + cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack + + # https://github.com/EmbarkStudios/cargo-deny + - name: Check rust licenses/bans/advisories/sources + if: ${{ !cancelled() }} + run: cargo deny check + gather-rust-build-stats: - if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') + if: | + contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned From 71b380f90a8f9b5813097a6bf961ad9cb73c416c Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Fri, 10 Nov 2023 12:49:52 +0000 Subject: [PATCH 42/63] Set BUILD_TAG for build-neon job (#5847) ## Problem I've added `BUILD_TAG` to docker images. (https://github.com/neondatabase/neon/pull/5812), but forgot to add it to services that we build for tests ## Summary of changes - Set `BUILD_TAG` in `build-neon` job --- .github/workflows/build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index babf767fcf..6068177245 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -175,7 +175,7 @@ jobs: run: cargo deny check build-neon: - needs: [ check-permissions ] + needs: [ check-permissions, tag ] runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned @@ -187,6 +187,7 @@ jobs: env: BUILD_TYPE: ${{ matrix.build_type }} GIT_VERSION: ${{ github.event.pull_request.head.sha || github.sha }} + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} steps: - name: Fix git ownership From a6f892e200ef7bd548b640b5db430599f54d902e Mon Sep 17 00:00:00 2001 From: Rahul Modpur Date: Fri, 10 Nov 2023 18:35:22 +0530 Subject: [PATCH 43/63] metric: add started and killed walredo processes counter (#5809) In OOM situations, knowing exactly how many walredo processes there were at a time would help afterwards to understand why was pageserver OOM killed. Add `pageserver_wal_redo_process_total` metric to keep track of total wal redo process started, shutdown and killed since pageserver start. Closes #5722 --------- Signed-off-by: Rahul Modpur Co-authored-by: Joonas Koivunen Co-authored-by: Christian Schwarz --- pageserver/src/metrics.rs | 40 +++++++++++++++++++++++++++++++++++++++ pageserver/src/walredo.rs | 22 ++++++++++++--------- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 3d11daed96..4b52f07326 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1252,6 +1252,46 @@ pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy = Lazy::new(|| { .unwrap() }); +pub(crate) struct WalRedoProcessCounters { + pub(crate) started: IntCounter, + pub(crate) killed_by_cause: enum_map::EnumMap, +} + +#[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)] +pub(crate) enum WalRedoKillCause { + WalRedoProcessDrop, + NoLeakChildDrop, + Startup, +} + +impl Default for WalRedoProcessCounters { + fn default() -> Self { + let started = register_int_counter!( + "pageserver_wal_redo_process_started_total", + "Number of WAL redo processes started", + ) + .unwrap(); + + let killed = register_int_counter_vec!( + "pageserver_wal_redo_process_stopped_total", + "Number of WAL redo processes stopped", + &["cause"], + ) + .unwrap(); + Self { + started, + killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| { + let cause = ::from_usize(i); + let cause_str: &'static str = cause.into(); + killed.with_label_values(&[cause_str]) + })), + } + } +} + +pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy = + Lazy::new(WalRedoProcessCounters::default); + /// Similar to `prometheus::HistogramTimer` but does not record on drop. pub struct StorageTimeMetricsTimer { metrics: StorageTimeMetrics, diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 4a9524b5e4..ccdf621c30 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use crate::config::PageServerConf; use crate::metrics::{ - WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, + WalRedoKillCause, WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_PROCESS_COUNTERS, + WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, }; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; use crate::repository::Key; @@ -662,10 +663,10 @@ impl WalRedoProcess { .close_fds() .spawn_no_leak_child(tenant_id) .context("spawn process")?; - + WAL_REDO_PROCESS_COUNTERS.started.inc(); let mut child = scopeguard::guard(child, |child| { error!("killing wal-redo-postgres process due to a problem during launch"); - child.kill_and_wait(); + child.kill_and_wait(WalRedoKillCause::Startup); }); let stdin = child.stdin.take().unwrap(); @@ -996,7 +997,7 @@ impl Drop for WalRedoProcess { self.child .take() .expect("we only do this once") - .kill_and_wait(); + .kill_and_wait(WalRedoKillCause::WalRedoProcessDrop); self.stderr_logger_cancel.cancel(); // no way to wait for stderr_logger_task from Drop because that is async only } @@ -1032,16 +1033,19 @@ impl NoLeakChild { }) } - fn kill_and_wait(mut self) { + fn kill_and_wait(mut self, cause: WalRedoKillCause) { let child = match self.child.take() { Some(child) => child, None => return, }; - Self::kill_and_wait_impl(child); + Self::kill_and_wait_impl(child, cause); } - #[instrument(skip_all, fields(pid=child.id()))] - fn kill_and_wait_impl(mut child: Child) { + #[instrument(skip_all, fields(pid=child.id(), ?cause))] + fn kill_and_wait_impl(mut child: Child, cause: WalRedoKillCause) { + scopeguard::defer! { + WAL_REDO_PROCESS_COUNTERS.killed_by_cause[cause].inc(); + } let res = child.kill(); if let Err(e) = res { // This branch is very unlikely because: @@ -1086,7 +1090,7 @@ impl Drop for NoLeakChild { // This thread here is going to outlive of our dropper. let span = tracing::info_span!("walredo", %tenant_id); let _entered = span.enter(); - Self::kill_and_wait_impl(child); + Self::kill_and_wait_impl(child, WalRedoKillCause::NoLeakChildDrop); }) .await }); From d672e44eee7740277b670fc7e4667c9f0ea04355 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 10 Nov 2023 13:58:18 +0000 Subject: [PATCH 44/63] pageserver: error type for collect_keyspace (#5846) ## Problem This is a log hygiene fix, for an occasional test failure. warn-level logging in imitate_timeline_cached_layer_accesses can't distinguish actual errors from shutdown cases. ## Summary of changes Replaced anyhow::Error with an explicit CollectKeySpaceError type, that includes conversion from PageReconstructError::Cancelled. --- pageserver/src/http/routes.rs | 2 +- pageserver/src/pgdatadir_mapping.rs | 30 +++++++++++++++---- .../src/tenant/timeline/eviction_task.rs | 14 +++++++-- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 63016042cf..2915178104 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1478,7 +1478,7 @@ async fn timeline_collect_keyspace( let keys = timeline .collect_keyspace(at_lsn, &ctx) .await - .map_err(ApiError::InternalServerError)?; + .map_err(|e| ApiError::InternalServerError(e.into()))?; json_response(StatusCode::OK, Partitioning { keys, at_lsn }) } diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 09feba9b68..abdb4a8379 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -22,6 +22,7 @@ use std::collections::{hash_map, HashMap, HashSet}; use std::ops::ControlFlow; use std::ops::Range; use tracing::{debug, trace, warn}; +use utils::bin_ser::DeserializeError; use utils::{bin_ser::BeSer, lsn::Lsn}; /// Block number within a relation or SLRU. This matches PostgreSQL's BlockNumber type. @@ -67,6 +68,25 @@ pub enum CalculateLogicalSizeError { Other(#[from] anyhow::Error), } +#[derive(Debug, thiserror::Error)] +pub(crate) enum CollectKeySpaceError { + #[error(transparent)] + Decode(#[from] DeserializeError), + #[error(transparent)] + PageRead(PageReconstructError), + #[error("cancelled")] + Cancelled, +} + +impl From for CollectKeySpaceError { + fn from(err: PageReconstructError) -> Self { + match err { + PageReconstructError::Cancelled => Self::Cancelled, + err => Self::PageRead(err), + } + } +} + impl From for CalculateLogicalSizeError { fn from(pre: PageReconstructError) -> Self { match pre { @@ -635,11 +655,11 @@ impl Timeline { /// Get a KeySpace that covers all the Keys that are in use at the given LSN. /// Anything that's not listed maybe removed from the underlying storage (from /// that LSN forwards). - pub async fn collect_keyspace( + pub(crate) async fn collect_keyspace( &self, lsn: Lsn, ctx: &RequestContext, - ) -> anyhow::Result { + ) -> Result { // Iterate through key ranges, greedily packing them into partitions let mut result = KeySpaceAccum::new(); @@ -648,7 +668,7 @@ impl Timeline { // Fetch list of database dirs and iterate them let buf = self.get(DBDIR_KEY, lsn, ctx).await?; - let dbdir = DbDirectory::des(&buf).context("deserialization failure")?; + let dbdir = DbDirectory::des(&buf)?; let mut dbs: Vec<(Oid, Oid)> = dbdir.dbdirs.keys().cloned().collect(); dbs.sort_unstable(); @@ -681,7 +701,7 @@ impl Timeline { let slrudir_key = slru_dir_to_key(kind); result.add_key(slrudir_key); let buf = self.get(slrudir_key, lsn, ctx).await?; - let dir = SlruSegmentDirectory::des(&buf).context("deserialization failure")?; + let dir = SlruSegmentDirectory::des(&buf)?; let mut segments: Vec = dir.segments.iter().cloned().collect(); segments.sort_unstable(); for segno in segments { @@ -699,7 +719,7 @@ impl Timeline { // Then pg_twophase result.add_key(TWOPHASEDIR_KEY); let buf = self.get(TWOPHASEDIR_KEY, lsn, ctx).await?; - let twophase_dir = TwoPhaseDirectory::des(&buf).context("deserialization failure")?; + let twophase_dir = TwoPhaseDirectory::des(&buf)?; let mut xids: Vec = twophase_dir.xids.iter().cloned().collect(); xids.sort_unstable(); for xid in xids { diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 183fcb872f..79bc434a2a 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -26,6 +26,7 @@ use tracing::{debug, error, info, info_span, instrument, warn, Instrument}; use crate::{ context::{DownloadBehavior, RequestContext}, + pgdatadir_mapping::CollectKeySpaceError, task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}, tenant::{ config::{EvictionPolicy, EvictionPolicyLayerAccessThreshold}, @@ -397,9 +398,16 @@ impl Timeline { if size.is_err() { // ignore, see above comment } else { - warn!( - "failed to collect keyspace but succeeded in calculating logical size: {e:#}" - ); + match e { + CollectKeySpaceError::Cancelled => { + // Shutting down, ignore + } + err => { + warn!( + "failed to collect keyspace but succeeded in calculating logical size: {err:#}" + ); + } + } } } } From a05f104cce70fbd7330704662108ed8934408328 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 10 Nov 2023 16:05:21 +0200 Subject: [PATCH 45/63] build: remove async-std dependency (#5848) Introduced by accident (missing `default-features = false`) in e09d5ada6a. We directly need only `http_types::StatusCode`. --- Cargo.lock | 597 +++----------------------------------- Cargo.toml | 2 +- workspace_hack/Cargo.toml | 6 +- 3 files changed, 50 insertions(+), 555 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 738771f88b..44ea46efda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,60 +23,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aead" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" -dependencies = [ - "aes-soft", - "aesni", - "cipher", -] - -[[package]] -name = "aes-gcm" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - -[[package]] -name = "aes-soft" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" -dependencies = [ - "cipher", - "opaque-debug", -] - -[[package]] -name = "aesni" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" -dependencies = [ - "cipher", - "opaque-debug", -] - [[package]] name = "ahash" version = "0.8.3" @@ -198,7 +144,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.21", + "time", ] [[package]] @@ -248,55 +194,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-executor" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1da3ae8dabd9c00f453a329dfe1fb28da3c0a72e2478cdcd93171740c20499" -dependencies = [ - "async-lock", - "async-task", - "concurrent-queue", - "fastrand 2.0.0", - "futures-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" -dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite", - "log", - "parking", - "polling", - "rustix 0.37.25", - "slab", - "socket2 0.4.9", - "waker-fn", -] - [[package]] name = "async-lock" version = "2.8.0" @@ -306,32 +203,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - [[package]] name = "async-stream" version = "0.3.5" @@ -354,12 +225,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "async-task" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9441c6b2fe128a7c2bf680a44c34d0df31ce09e5b7e401fcca3faa483dbc921" - [[package]] name = "async-trait" version = "0.1.68" @@ -380,12 +245,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.1.0" @@ -415,7 +274,7 @@ dependencies = [ "http", "hyper", "ring", - "time 0.3.21", + "time", "tokio", "tower", "tracing", @@ -568,13 +427,13 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac 0.12.1", + "hmac", "http", "once_cell", "percent-encoding", "regex", - "sha2 0.10.6", - "time 0.3.21", + "sha2", + "time", "tracing", ] @@ -606,8 +465,8 @@ dependencies = [ "http-body", "md-5", "pin-project-lite", - "sha1 0.10.5", - "sha2 0.10.6", + "sha1", + "sha2", "tracing", ] @@ -752,7 +611,7 @@ dependencies = [ "num-integer", "ryu", "serde", - "time 0.3.21", + "time", ] [[package]] @@ -776,7 +635,7 @@ dependencies = [ "aws-smithy-http", "aws-smithy-types", "http", - "rustc_version 0.4.0", + "rustc_version", "tracing", ] @@ -806,7 +665,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1 0.10.5", + "sha1", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -851,10 +710,10 @@ dependencies = [ "quick-xml", "rand 0.8.5", "reqwest", - "rustc_version 0.4.0", + "rustc_version", "serde", "serde_json", - "time 0.3.21", + "time", "url", "uuid", ] @@ -874,7 +733,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "time 0.3.21", + "time", "tz-rs", "url", "uuid", @@ -891,13 +750,13 @@ dependencies = [ "azure_core", "bytes", "futures", - "hmac 0.12.1", + "hmac", "log", "serde", "serde_derive", "serde_json", - "sha2 0.10.6", - "time 0.3.21", + "sha2", + "time", "url", "uuid", ] @@ -917,7 +776,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "time 0.3.21", + "time", "url", "uuid", ] @@ -937,12 +796,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - [[package]] name = "base64" version = "0.13.1" @@ -1015,15 +868,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1033,22 +877,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c36a4d0d48574b3dd360b4b7d95cc651d2b6557b6402848a27d4b228a473e2a" -dependencies = [ - "async-channel", - "async-lock", - "async-task", - "fastrand 2.0.0", - "futures-io", - "futures-lite", - "piper", - "tracing", -] - [[package]] name = "bstr" version = "1.5.0" @@ -1193,15 +1021,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" -dependencies = [ - "generic-array", -] - [[package]] name = "clang-sys" version = "1.6.1" @@ -1419,23 +1238,6 @@ dependencies = [ "workspace_hack", ] -[[package]] -name = "cookie" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" -dependencies = [ - "aes-gcm", - "base64 0.13.1", - "hkdf", - "hmac 0.10.1", - "percent-encoding", - "rand 0.8.5", - "sha2 0.9.9", - "time 0.2.27", - "version_check", -] - [[package]] name = "core-foundation" version = "0.9.3" @@ -1461,19 +1263,13 @@ dependencies = [ "libc", ] -[[package]] -name = "cpuid-bool" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" - [[package]] name = "crc32c" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e" dependencies = [ - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -1605,25 +1401,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ctr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" -dependencies = [ - "cipher", -] - [[package]] name = "darling" version = "0.20.1" @@ -1702,32 +1479,17 @@ dependencies = [ "rusticata-macros", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "crypto-common", "subtle", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "displaydoc" version = "0.2.4" @@ -2094,16 +1856,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "ghash" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" -dependencies = [ - "opaque-debug", - "polyval", -] - [[package]] name = "gimli" version = "0.27.2" @@ -2138,18 +1890,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "h2" version = "0.3.19" @@ -2221,7 +1961,7 @@ source = "git+https://github.com/japaric/heapless.git?rev=644653bf3b831c6bb4963b dependencies = [ "atomic-polyfill", "hash32", - "rustc_version 0.4.0", + "rustc_version", "spin 0.9.8", "stable_deref_trait", ] @@ -2263,33 +2003,13 @@ dependencies = [ "thiserror", ] -[[package]] -name = "hkdf" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" -dependencies = [ - "digest 0.9.0", - "hmac 0.10.1", -] - -[[package]] -name = "hmac" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" -dependencies = [ - "crypto-mac", - "digest 0.9.0", -] - [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -2333,9 +2053,7 @@ checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" dependencies = [ "anyhow", "async-channel", - "async-std", "base64 0.13.1", - "cookie", "futures-lite", "infer", "pin-project-lite", @@ -2649,15 +2367,6 @@ dependencies = [ "libc", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2713,9 +2422,6 @@ name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -dependencies = [ - "value-bag", -] [[package]] name = "match_cfg" @@ -2744,7 +2450,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -2990,7 +2696,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "sha2 0.10.6", + "sha2", "thiserror", "url", ] @@ -3025,12 +2731,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.55" @@ -3384,10 +3084,10 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0ca0b5a68607598bf3bad68f32227a8164f6254833f84eafaac409cd6746c31" dependencies = [ - "digest 0.10.7", - "hmac 0.12.1", + "digest", + "hmac", "password-hash", - "sha2 0.10.6", + "sha2", ] [[package]] @@ -3481,17 +3181,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" -dependencies = [ - "atomic-waker", - "fastrand 2.0.0", - "futures-io", -] - [[package]] name = "pkg-config" version = "0.3.27" @@ -3526,33 +3215,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "polyval" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" -dependencies = [ - "cpuid-bool", - "opaque-debug", - "universal-hash", -] - [[package]] name = "postgres" version = "0.19.4" @@ -3586,12 +3248,12 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator", - "hmac 0.12.1", + "hmac", "lazy_static", "md-5", "memchr", "rand 0.8.5", - "sha2 0.10.6", + "sha2", "stringprep", ] @@ -3820,7 +3482,7 @@ dependencies = [ "hashbrown 0.13.2", "hashlink", "hex", - "hmac 0.12.1", + "hmac", "hostname", "humantime", "hyper", @@ -3853,7 +3515,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", - "sha2 0.10.6", + "sha2", "socket2 0.5.3", "sync_wrapper", "thiserror", @@ -3995,7 +3657,7 @@ checksum = "4954fbc00dcd4d8282c987710e50ba513d351400dbdd00e803a05172a90d8976" dependencies = [ "pem 2.0.1", "ring", - "time 0.3.21", + "time", "yasna", ] @@ -4252,7 +3914,7 @@ dependencies = [ "futures", "futures-timer", "rstest_macros", - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -4267,7 +3929,7 @@ dependencies = [ "quote", "regex", "relative-path", - "rustc_version 0.4.0", + "rustc_version", "syn 2.0.28", "unicode-ident", ] @@ -4284,22 +3946,13 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.17", + "semver", ] [[package]] @@ -4561,27 +4214,12 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "sentry" version = "0.31.6" @@ -4622,7 +4260,7 @@ dependencies = [ "hostname", "libc", "os_info", - "rustc_version 0.4.0", + "rustc_version", "sentry-core", "uname", ] @@ -4674,7 +4312,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "time 0.3.21", + "time", "url", "uuid", ] @@ -4775,7 +4413,7 @@ dependencies = [ "serde", "serde_json", "serde_with_macros", - "time 0.3.21", + "time", ] [[package]] @@ -4790,15 +4428,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - [[package]] name = "sha1" version = "0.10.5" @@ -4807,26 +4436,7 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "digest", ] [[package]] @@ -4837,7 +4447,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -4894,7 +4504,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.21", + "time", ] [[package]] @@ -4959,70 +4569,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1 0.6.1", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - [[package]] name = "storage_broker" version = "0.1.0" @@ -5256,21 +4808,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros 0.1.1", - "version_check", - "winapi", -] - [[package]] name = "time" version = "0.3.21" @@ -5283,7 +4820,7 @@ dependencies = [ "num_threads", "serde", "time-core", - "time-macros 0.2.9", + "time-macros", ] [[package]] @@ -5292,16 +4829,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - [[package]] name = "time-macros" version = "0.2.9" @@ -5311,19 +4838,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn 1.0.109", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -5685,7 +5199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.21", + "time", "tracing-subscriber", ] @@ -5820,7 +5334,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha1 0.10.5", + "sha1", "thiserror", "url", "utf-8", @@ -5892,16 +5406,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "universal-hash" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -6019,12 +5523,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "value-bag" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" - [[package]] name = "vcpkg" version = "0.2.15" @@ -6514,11 +6012,10 @@ dependencies = [ "serde", "serde_json", "smallvec", - "standback", "syn 1.0.109", "syn 2.0.28", - "time 0.3.21", - "time-macros 0.2.9", + "time", + "time-macros", "tokio", "tokio-rustls", "tokio-util", @@ -6546,7 +6043,7 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.21", + "time", ] [[package]] @@ -6570,7 +6067,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.21", + "time", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e528489f1e..363d4c6fe4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,7 @@ hex = "0.4" hex-literal = "0.4" hmac = "0.12.1" hostname = "0.3.1" -http-types = "2" +http-types = { version = "2", default-features = false } humantime = "2.1" humantime-serde = "1.1.1" hyper = "0.14" diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index a088f1868b..3535c2cf94 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -39,7 +39,7 @@ hex = { version = "0.4", features = ["serde"] } hyper = { version = "0.14", features = ["full"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] } +log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } nom = { version = "7" } num-bigint = { version = "0.4" } @@ -56,7 +56,6 @@ scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } smallvec = { version = "1", default-features = false, features = ["write"] } -standback = { version = "0.2", default-features = false, features = ["std"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-rustls = { version = "0.24" } @@ -77,14 +76,13 @@ cc = { version = "1", default-features = false, features = ["parallel"] } either = { version = "1" } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] } +log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } nom = { version = "7" } prost = { version = "0.11" } regex = { version = "1" } regex-syntax = { version = "0.7" } serde = { version = "1", features = ["alloc", "derive"] } -standback = { version = "0.2", default-features = false, features = ["std"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "full", "visit", "visit-mut"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } From b7f45204a28f69e344e74d653b68c7011bc4a6da Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 10 Nov 2023 19:02:22 +0200 Subject: [PATCH 46/63] build: deny async-std and friends (#5849) rationale: some crates pull these in as default; hopefully these hints will require less cleanup-after and Cargo.lock file watching. follow-up to #5848. --- .github/workflows/build_and_test.yml | 2 +- deny.toml | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6068177245..ff7d8c1a62 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -172,7 +172,7 @@ jobs: # https://github.com/EmbarkStudios/cargo-deny - name: Check rust licenses/bans/advisories/sources if: ${{ !cancelled() }} - run: cargo deny check + run: cargo deny check --hide-inclusion-graph build-neon: needs: [ check-permissions, tag ] diff --git a/deny.toml b/deny.toml index f4ea0d4dac..079dcac679 100644 --- a/deny.toml +++ b/deny.toml @@ -74,10 +74,30 @@ highlight = "all" workspace-default-features = "allow" external-default-features = "allow" allow = [] -deny = [] + skip = [] skip-tree = [] +[[bans.deny]] +# we use tokio, the same rationale applies for async-{io,waker,global-executor,executor,channel,lock}, smol +# if you find yourself here while adding a dependency, try "default-features = false", ask around on #rust +name = "async-std" + +[[bans.deny]] +name = "async-io" + +[[bans.deny]] +name = "async-waker" + +[[bans.deny]] +name = "async-global-executor" + +[[bans.deny]] +name = "async-executor" + +[[bans.deny]] +name = "smol" + # This section is considered when running `cargo deny check sources`. # More documentation about the 'sources' section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html From 74d150ba45cc230e2403cc7a016230e01334972f Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 10 Nov 2023 21:10:54 +0200 Subject: [PATCH 47/63] build: upgrade ahash (#5851) `cargo deny` was complaining the version 0.8.3 was yanked (for possible DoS attack [wiki]), but the latest version (0.8.5) also includes aarch64 fixes which may or may not be relevant. Our usage of ahash limits to proxy, but I don't think we are at any risk. [wiki]: https://github.com/tkaitchuck/aHash/wiki/Yanked-versions --- Cargo.lock | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44ea46efda..4cada013d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,13 +25,14 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" dependencies = [ "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -6070,6 +6071,26 @@ dependencies = [ "time", ] +[[package]] +name = "zerocopy" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7af71d8643341260a65f89fa60c0eeaa907f34544d8f6d9b0df72f069b5e74" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9731702e2f0617ad526794ae28fbc6f6ca8849b5ba729666c2a5bc4b6ddee2cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + [[package]] name = "zeroize" version = "1.6.0" From f7249b9018d582cb2b03546836e04f44fbb03644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Sat, 11 Nov 2023 01:32:00 +0100 Subject: [PATCH 48/63] Fix comment in find_lsn_for_timestamp (#5855) We still subtract 1 from low to compute `commit_lsn`. the comment moved/added by #5844 should point this out. --- pageserver/src/pgdatadir_mapping.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index abdb4a8379..827278af72 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -402,8 +402,9 @@ impl Timeline { low = mid + 1; } } - // If `found_smaller == true`, `low` is the LSN of the last commit record - // before or at `search_timestamp` + // If `found_smaller == true`, `low = t + 1` where `t` is the target LSN, + // so the LSN of the last commit record before or at `search_timestamp`. + // Remove one from `low` to get `t`. // // FIXME: it would be better to get the LSN of the previous commit. // Otherwise, if you restore to the returned LSN, the database will From 7709c91fe57428a82ee4b9a21cd331d5abcaa81d Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 14 Nov 2023 09:51:51 +0000 Subject: [PATCH 49/63] neon_local: use remote storage by default, add `cargo neon tenant migrate` (#5760) ## Problem Currently the only way to exercise tenant migration is via python test code. We need a convenient way for developers to do it directly in a neon local environment. ## Summary of changes - Add a `--num-pageservers` argument to `cargo neon init` so that it's easy to run with multiple pageservers - Modify default pageserver overrides in neon_local to set up `LocalFs` remote storage, as any migration/attach/detach stuff doesn't work in the legacy local storage mode. This also unblocks removing the pageserver's support for the legacy local mode. - Add a new `cargo neon tenant migrate` command that orchestrates tenant migration, including endpoints. --- control_plane/src/attachment_service.rs | 41 +++- control_plane/src/bin/attachment_service.rs | 20 +- control_plane/src/bin/neon_local.rs | 110 ++++++++--- control_plane/src/lib.rs | 1 + control_plane/src/pageserver.rs | 53 ++++- control_plane/src/tenant_migration.rs | 202 ++++++++++++++++++++ test_runner/fixtures/neon_fixtures.py | 2 + 7 files changed, 388 insertions(+), 41 deletions(-) create mode 100644 control_plane/src/tenant_migration.rs diff --git a/control_plane/src/attachment_service.rs b/control_plane/src/attachment_service.rs index fcefe0e431..822ac7d8a6 100644 --- a/control_plane/src/attachment_service.rs +++ b/control_plane/src/attachment_service.rs @@ -9,6 +9,7 @@ pub struct AttachmentService { env: LocalEnv, listen: String, path: PathBuf, + client: reqwest::blocking::Client, } const COMMAND: &str = "attachment_service"; @@ -24,6 +25,16 @@ pub struct AttachHookResponse { pub gen: Option, } +#[derive(Serialize, Deserialize)] +pub struct InspectRequest { + pub tenant_id: TenantId, +} + +#[derive(Serialize, Deserialize)] +pub struct InspectResponse { + pub attachment: Option<(u32, NodeId)>, +} + impl AttachmentService { pub fn from_env(env: &LocalEnv) -> Self { let path = env.base_data_dir.join("attachments.json"); @@ -42,6 +53,9 @@ impl AttachmentService { env: env.clone(), path, listen, + client: reqwest::blocking::ClientBuilder::new() + .build() + .expect("Failed to construct http client"), } } @@ -84,16 +98,13 @@ impl AttachmentService { .unwrap() .join("attach-hook") .unwrap(); - let client = reqwest::blocking::ClientBuilder::new() - .build() - .expect("Failed to construct http client"); let request = AttachHookRequest { tenant_id, node_id: Some(pageserver_id), }; - let response = client.post(url).json(&request).send()?; + let response = self.client.post(url).json(&request).send()?; if response.status() != StatusCode::OK { return Err(anyhow!("Unexpected status {}", response.status())); } @@ -101,4 +112,26 @@ impl AttachmentService { let response = response.json::()?; Ok(response.gen) } + + pub fn inspect(&self, tenant_id: TenantId) -> anyhow::Result> { + use hyper::StatusCode; + + let url = self + .env + .control_plane_api + .clone() + .unwrap() + .join("inspect") + .unwrap(); + + let request = InspectRequest { tenant_id }; + + let response = self.client.post(url).json(&request).send()?; + if response.status() != StatusCode::OK { + return Err(anyhow!("Unexpected status {}", response.status())); + } + + let response = response.json::()?; + Ok(response.attachment) + } } diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index 3205703c7d..ddd50ff51d 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -32,7 +32,9 @@ use pageserver_api::control_api::{ ValidateResponseTenant, }; -use control_plane::attachment_service::{AttachHookRequest, AttachHookResponse}; +use control_plane::attachment_service::{ + AttachHookRequest, AttachHookResponse, InspectRequest, InspectResponse, +}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -255,12 +257,28 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap ) } +async fn handle_inspect(mut req: Request) -> Result, ApiError> { + let inspect_req = json_request::(&mut req).await?; + + let state = get_state(&req).inner.clone(); + let locked = state.write().await; + let tenant_state = locked.tenants.get(&inspect_req.tenant_id); + + json_response( + StatusCode::OK, + InspectResponse { + attachment: tenant_state.and_then(|s| s.pageserver.map(|ps| (s.generation, ps))), + }, + ) +} + fn make_router(persistent_state: PersistentState) -> RouterBuilder { endpoint::make_router() .data(Arc::new(State::new(persistent_state))) .post("/re-attach", |r| request_span(r, handle_re_attach)) .post("/validate", |r| request_span(r, handle_validate)) .post("/attach-hook", |r| request_span(r, handle_attach_hook)) + .post("/inspect", |r| request_span(r, handle_inspect)) } #[tokio::main] diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index bc68eeaa8a..87ea519a9e 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -11,13 +11,14 @@ use compute_api::spec::ComputeMode; use control_plane::attachment_service::AttachmentService; use control_plane::endpoint::ComputeControlPlane; use control_plane::local_env::LocalEnv; -use control_plane::pageserver::PageServerNode; +use control_plane::pageserver::{PageServerNode, PAGESERVER_REMOTE_STORAGE_DIR}; use control_plane::safekeeper::SafekeeperNode; +use control_plane::tenant_migration::migrate_tenant; use control_plane::{broker, local_env}; use pageserver_api::models::TimelineInfo; use pageserver_api::{ - DEFAULT_HTTP_LISTEN_ADDR as DEFAULT_PAGESERVER_HTTP_ADDR, - DEFAULT_PG_LISTEN_ADDR as DEFAULT_PAGESERVER_PG_ADDR, + DEFAULT_HTTP_LISTEN_PORT as DEFAULT_PAGESERVER_HTTP_PORT, + DEFAULT_PG_LISTEN_PORT as DEFAULT_PAGESERVER_PG_PORT, }; use postgres_backend::AuthType; use safekeeper_api::{ @@ -46,8 +47,8 @@ const DEFAULT_PG_VERSION: &str = "15"; const DEFAULT_PAGESERVER_CONTROL_PLANE_API: &str = "http://127.0.0.1:1234/"; -fn default_conf() -> String { - format!( +fn default_conf(num_pageservers: u16) -> String { + let mut template = format!( r#" # Default built-in configuration, defined in main.rs control_plane_api = '{DEFAULT_PAGESERVER_CONTROL_PLANE_API}' @@ -55,21 +56,33 @@ control_plane_api = '{DEFAULT_PAGESERVER_CONTROL_PLANE_API}' [broker] listen_addr = '{DEFAULT_BROKER_ADDR}' -[[pageservers]] -id = {DEFAULT_PAGESERVER_ID} -listen_pg_addr = '{DEFAULT_PAGESERVER_PG_ADDR}' -listen_http_addr = '{DEFAULT_PAGESERVER_HTTP_ADDR}' -pg_auth_type = '{trust_auth}' -http_auth_type = '{trust_auth}' - [[safekeepers]] id = {DEFAULT_SAFEKEEPER_ID} pg_port = {DEFAULT_SAFEKEEPER_PG_PORT} http_port = {DEFAULT_SAFEKEEPER_HTTP_PORT} "#, - trust_auth = AuthType::Trust, - ) + ); + + for i in 0..num_pageservers { + let pageserver_id = NodeId(DEFAULT_PAGESERVER_ID.0 + i as u64); + let pg_port = DEFAULT_PAGESERVER_PG_PORT + i; + let http_port = DEFAULT_PAGESERVER_HTTP_PORT + i; + + template += &format!( + r#" +[[pageservers]] +id = {pageserver_id} +listen_pg_addr = '127.0.0.1:{pg_port}' +listen_http_addr = '127.0.0.1:{http_port}' +pg_auth_type = '{trust_auth}' +http_auth_type = '{trust_auth}' +"#, + trust_auth = AuthType::Trust, + ) + } + + template } /// @@ -295,6 +308,9 @@ fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result anyhow::Result { + let num_pageservers = init_match + .get_one::("num-pageservers") + .expect("num-pageservers arg has a default"); // Create config file let toml_file: String = if let Some(config_path) = init_match.get_one::("config") { // load and parse the file @@ -306,7 +322,7 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result { })? } else { // Built-in default config - default_conf() + default_conf(*num_pageservers) }; let pg_version = init_match @@ -320,6 +336,9 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result { env.init(pg_version, force) .context("Failed to initialize neon repository")?; + // Create remote storage location for default LocalFs remote storage + std::fs::create_dir_all(env.base_data_dir.join(PAGESERVER_REMOTE_STORAGE_DIR))?; + // Initialize pageserver, create initial tenant and timeline. for ps_conf in &env.pageservers { PageServerNode::from_env(&env, ps_conf) @@ -433,6 +452,15 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an .with_context(|| format!("Tenant config failed for tenant with id {tenant_id}"))?; println!("tenant {tenant_id} successfully configured on the pageserver"); } + Some(("migrate", matches)) => { + let tenant_id = get_tenant_id(matches, env)?; + let new_pageserver = get_pageserver(env, matches)?; + let new_pageserver_id = new_pageserver.conf.id; + + migrate_tenant(env, tenant_id, new_pageserver)?; + println!("tenant {tenant_id} migrated to {}", new_pageserver_id); + } + Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), None => bail!("no tenant subcommand provided"), } @@ -867,20 +895,20 @@ fn handle_mappings(sub_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Res } } +fn get_pageserver(env: &local_env::LocalEnv, args: &ArgMatches) -> Result { + let node_id = if let Some(id_str) = args.get_one::("pageserver-id") { + NodeId(id_str.parse().context("while parsing pageserver id")?) + } else { + DEFAULT_PAGESERVER_ID + }; + + Ok(PageServerNode::from_env( + env, + env.get_pageserver_conf(node_id)?, + )) +} + fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { - fn get_pageserver(env: &local_env::LocalEnv, args: &ArgMatches) -> Result { - let node_id = if let Some(id_str) = args.get_one::("pageserver-id") { - NodeId(id_str.parse().context("while parsing pageserver id")?) - } else { - DEFAULT_PAGESERVER_ID - }; - - Ok(PageServerNode::from_env( - env, - env.get_pageserver_conf(node_id)?, - )) - } - match sub_match.subcommand() { Some(("start", subcommand_args)) => { if let Err(e) = get_pageserver(env, subcommand_args)? @@ -917,6 +945,20 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul } } + Some(("migrate", subcommand_args)) => { + let pageserver = get_pageserver(env, subcommand_args)?; + //TODO what shutdown strategy should we use here? + if let Err(e) = pageserver.stop(false) { + eprintln!("pageserver stop failed: {}", e); + exit(1); + } + + if let Err(e) = pageserver.start(&pageserver_config_overrides(subcommand_args)) { + eprintln!("pageserver start failed: {e}"); + exit(1); + } + } + Some(("status", subcommand_args)) => { match get_pageserver(env, subcommand_args)?.check_status() { Ok(_) => println!("Page server is up and running"), @@ -1224,6 +1266,13 @@ fn cli() -> Command { .help("Force initialization even if the repository is not empty") .required(false); + let num_pageservers_arg = Arg::new("num-pageservers") + .value_parser(value_parser!(u16)) + .long("num-pageservers") + .help("How many pageservers to create (default 1)") + .required(false) + .default_value("1"); + Command::new("Neon CLI") .arg_required_else_help(true) .version(GIT_VERSION) @@ -1231,6 +1280,7 @@ fn cli() -> Command { Command::new("init") .about("Initialize a new Neon repository, preparing configs for services to start with") .arg(pageserver_config_args.clone()) + .arg(num_pageservers_arg.clone()) .arg( Arg::new("config") .long("config") @@ -1301,6 +1351,10 @@ fn cli() -> Command { .subcommand(Command::new("config") .arg(tenant_id_arg.clone()) .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))) + .subcommand(Command::new("migrate") + .about("Migrate a tenant from one pageserver to another") + .arg(tenant_id_arg.clone()) + .arg(pageserver_id_arg.clone())) ) .subcommand( Command::new("pageserver") diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index bb79d36bfc..52a0e20429 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -14,3 +14,4 @@ pub mod local_env; pub mod pageserver; pub mod postgresql_conf; pub mod safekeeper; +pub mod tenant_migration; diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 0746dde4ef..e13a234e89 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -15,7 +15,9 @@ use std::{io, result}; use anyhow::{bail, Context}; use camino::Utf8PathBuf; -use pageserver_api::models::{self, TenantInfo, TimelineInfo}; +use pageserver_api::models::{ + self, LocationConfig, TenantInfo, TenantLocationConfigRequest, TimelineInfo, +}; use postgres_backend::AuthType; use postgres_connection::{parse_host_port, PgConnectionConfig}; use reqwest::blocking::{Client, RequestBuilder, Response}; @@ -31,6 +33,9 @@ use utils::{ use crate::local_env::PageServerConf; use crate::{background_process, local_env::LocalEnv}; +/// Directory within .neon which will be used by default for LocalFs remote storage. +pub const PAGESERVER_REMOTE_STORAGE_DIR: &str = "local_fs_remote_storage/pageserver"; + #[derive(Error, Debug)] pub enum PageserverHttpError { #[error("Reqwest error: {0}")] @@ -98,8 +103,10 @@ impl PageServerNode { } } - // pageserver conf overrides defined by neon_local configuration. - fn neon_local_overrides(&self) -> Vec { + /// Merge overrides provided by the user on the command line with our default overides derived from neon_local configuration. + /// + /// These all end up on the command line of the `pageserver` binary. + fn neon_local_overrides(&self, cli_overrides: &[&str]) -> Vec { let id = format!("id={}", self.conf.id); // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc. let pg_distrib_dir_param = format!( @@ -132,12 +139,25 @@ impl PageServerNode { )); } + if !cli_overrides + .iter() + .any(|c| c.starts_with("remote_storage")) + { + overrides.push(format!( + "remote_storage={{local_path='../{PAGESERVER_REMOTE_STORAGE_DIR}'}}" + )); + } + if self.conf.http_auth_type != AuthType::Trust || self.conf.pg_auth_type != AuthType::Trust { // Keys are generated in the toplevel repo dir, pageservers' workdirs // are one level below that, so refer to keys with ../ overrides.push("auth_validation_public_key_path='../auth_public_key.pem'".to_owned()); } + + // Apply the user-provided overrides + overrides.extend(cli_overrides.iter().map(|&c| c.to_owned())); + overrides } @@ -203,9 +223,6 @@ impl PageServerNode { } fn start_node(&self, config_overrides: &[&str], update_config: bool) -> anyhow::Result { - let mut overrides = self.neon_local_overrides(); - overrides.extend(config_overrides.iter().map(|&c| c.to_owned())); - let datadir = self.repo_path(); print!( "Starting pageserver node {} at '{}' in {:?}", @@ -248,8 +265,7 @@ impl PageServerNode { ) -> Vec> { let mut args = vec![Cow::Borrowed("-D"), Cow::Borrowed(datadir_path_str)]; - let mut overrides = self.neon_local_overrides(); - overrides.extend(config_overrides.iter().map(|&c| c.to_owned())); + let overrides = self.neon_local_overrides(config_overrides); for config_override in overrides { args.push(Cow::Borrowed("-c")); args.push(Cow::Owned(config_override)); @@ -501,6 +517,27 @@ impl PageServerNode { Ok(()) } + pub fn location_config( + &self, + tenant_id: TenantId, + config: LocationConfig, + ) -> anyhow::Result<()> { + let req_body = TenantLocationConfigRequest { tenant_id, config }; + + self.http_request( + Method::PUT, + format!( + "{}/tenant/{}/location_config", + self.http_base_url, tenant_id + ), + )? + .json(&req_body) + .send()? + .error_from_body()?; + + Ok(()) + } + pub fn timeline_list(&self, tenant_id: &TenantId) -> anyhow::Result> { let timeline_infos: Vec = self .http_request( diff --git a/control_plane/src/tenant_migration.rs b/control_plane/src/tenant_migration.rs new file mode 100644 index 0000000000..d28d1f9fe8 --- /dev/null +++ b/control_plane/src/tenant_migration.rs @@ -0,0 +1,202 @@ +//! +//! Functionality for migrating tenants across pageservers: unlike most of neon_local, this code +//! isn't scoped to a particular physical service, as it needs to update compute endpoints to +//! point to the new pageserver. +//! +use crate::local_env::LocalEnv; +use crate::{ + attachment_service::AttachmentService, endpoint::ComputeControlPlane, + pageserver::PageServerNode, +}; +use pageserver_api::models::{ + LocationConfig, LocationConfigMode, LocationConfigSecondary, TenantConfig, +}; +use std::collections::HashMap; +use std::time::Duration; +use utils::{ + generation::Generation, + id::{TenantId, TimelineId}, + lsn::Lsn, +}; + +/// Given an attached pageserver, retrieve the LSN for all timelines +fn get_lsns( + tenant_id: TenantId, + pageserver: &PageServerNode, +) -> anyhow::Result> { + let timelines = pageserver.timeline_list(&tenant_id)?; + Ok(timelines + .into_iter() + .map(|t| (t.timeline_id, t.last_record_lsn)) + .collect()) +} + +/// Wait for the timeline LSNs on `pageserver` to catch up with or overtake +/// `baseline`. +fn await_lsn( + tenant_id: TenantId, + pageserver: &PageServerNode, + baseline: HashMap, +) -> anyhow::Result<()> { + loop { + let latest = match get_lsns(tenant_id, pageserver) { + Ok(l) => l, + Err(e) => { + println!( + "🕑 Can't get LSNs on pageserver {} yet, waiting ({e})", + pageserver.conf.id + ); + std::thread::sleep(Duration::from_millis(500)); + continue; + } + }; + + let mut any_behind: bool = false; + for (timeline_id, baseline_lsn) in &baseline { + match latest.get(timeline_id) { + Some(latest_lsn) => { + println!("🕑 LSN origin {baseline_lsn} vs destination {latest_lsn}"); + if latest_lsn < baseline_lsn { + any_behind = true; + } + } + None => { + // Expected timeline isn't yet visible on migration destination. + // (IRL we would have to account for timeline deletion, but this + // is just test helper) + any_behind = true; + } + } + } + + if !any_behind { + println!("✅ LSN caught up. Proceeding..."); + break; + } else { + std::thread::sleep(Duration::from_millis(500)); + } + } + + Ok(()) +} + +/// This function spans multiple services, to demonstrate live migration of a tenant +/// between pageservers: +/// - Coordinate attach/secondary/detach on pageservers +/// - call into attachment_service for generations +/// - reconfigure compute endpoints to point to new attached pageserver +pub fn migrate_tenant( + env: &LocalEnv, + tenant_id: TenantId, + dest_ps: PageServerNode, +) -> anyhow::Result<()> { + // Get a new generation + let attachment_service = AttachmentService::from_env(env); + + let previous = attachment_service.inspect(tenant_id)?; + let mut baseline_lsns = None; + if let Some((generation, origin_ps_id)) = &previous { + let origin_ps = PageServerNode::from_env(env, env.get_pageserver_conf(*origin_ps_id)?); + + if origin_ps_id == &dest_ps.conf.id { + println!("🔁 Already attached to {origin_ps_id}, freshening..."); + let gen = attachment_service.attach_hook(tenant_id, dest_ps.conf.id)?; + let dest_conf = LocationConfig { + mode: LocationConfigMode::AttachedSingle, + generation: gen.map(Generation::new), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + dest_ps.location_config(tenant_id, dest_conf)?; + println!("✅ Migration complete"); + return Ok(()); + } + + println!("🔁 Switching origin pageserver {origin_ps_id} to stale mode"); + + let stale_conf = LocationConfig { + mode: LocationConfigMode::AttachedStale, + generation: Some(Generation::new(*generation)), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + origin_ps.location_config(tenant_id, stale_conf)?; + + baseline_lsns = Some(get_lsns(tenant_id, &origin_ps)?); + } + + let gen = attachment_service.attach_hook(tenant_id, dest_ps.conf.id)?; + let dest_conf = LocationConfig { + mode: LocationConfigMode::AttachedMulti, + generation: gen.map(Generation::new), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + + println!("🔁 Attaching to pageserver {}", dest_ps.conf.id); + dest_ps.location_config(tenant_id, dest_conf)?; + + if let Some(baseline) = baseline_lsns { + println!("🕑 Waiting for LSN to catch up..."); + await_lsn(tenant_id, &dest_ps, baseline)?; + } + + let cplane = ComputeControlPlane::load(env.clone())?; + for (endpoint_name, endpoint) in &cplane.endpoints { + if endpoint.tenant_id == tenant_id { + println!( + "🔁 Reconfiguring endpoint {} to use pageserver {}", + endpoint_name, dest_ps.conf.id + ); + endpoint.reconfigure(Some(dest_ps.conf.id))?; + } + } + + for other_ps_conf in &env.pageservers { + if other_ps_conf.id == dest_ps.conf.id { + continue; + } + + let other_ps = PageServerNode::from_env(env, other_ps_conf); + let other_ps_tenants = other_ps.tenant_list()?; + + // Check if this tenant is attached + let found = other_ps_tenants + .into_iter() + .map(|t| t.id) + .any(|i| i == tenant_id); + if !found { + continue; + } + + // Downgrade to a secondary location + let secondary_conf = LocationConfig { + mode: LocationConfigMode::Secondary, + generation: None, + secondary_conf: Some(LocationConfigSecondary { warm: true }), + tenant_conf: TenantConfig::default(), + }; + + println!( + "💤 Switching to secondary mode on pageserver {}", + other_ps.conf.id + ); + other_ps.location_config(tenant_id, secondary_conf)?; + } + + println!( + "🔁 Switching to AttachedSingle mode on pageserver {}", + dest_ps.conf.id + ); + let dest_conf = LocationConfig { + mode: LocationConfigMode::AttachedSingle, + generation: gen.map(Generation::new), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + dest_ps.location_config(tenant_id, dest_conf)?; + + println!("✅ Migration complete"); + + Ok(()) +} diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 4057f620a0..b45b0a12c0 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1871,6 +1871,8 @@ def append_pageserver_param_overrides( params_to_update.append( f"--pageserver-config-override=remote_storage={remote_storage_toml_table}" ) + else: + params_to_update.append('--pageserver-config-override=remote_storage=""') env_overrides = os.getenv("NEON_PAGESERVER_OVERRIDES") if env_overrides is not None: From 31a54d663cfebeb8e08e96ba00db06f0a3c16939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 14 Nov 2023 16:36:47 +0100 Subject: [PATCH 50/63] Migrate links from wiki to notion (#5862) See the slack discussion: https://neondb.slack.com/archives/C033A2WE6BZ/p1696429688621489?thread_ts=1695647103.117499 --- .github/PULL_REQUEST_TEMPLATE/release-pr.md | 2 +- docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md | 2 +- scripts/export_import_between_pageservers.py | 2 +- test_runner/regress/test_compatibility.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/release-pr.md b/.github/PULL_REQUEST_TEMPLATE/release-pr.md index 1e18fd5d44..44b3094c24 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-pr.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-pr.md @@ -3,7 +3,7 @@ **NB: this PR must be merged only by 'Create a merge commit'!** ### Checklist when preparing for release -- [ ] Read or refresh [the release flow guide](https://github.com/neondatabase/cloud/wiki/Release:-general-flow) +- [ ] Read or refresh [the release flow guide](https://www.notion.so/neondatabase/Release-general-flow-61f2e39fd45d4d14a70c7749604bd70b) - [ ] Ask in the [cloud Slack channel](https://neondb.slack.com/archives/C033A2WE6BZ) that you are going to rollout the release. Any blockers? - [ ] Does this release contain any db migrations? Destructive ones? What is the rollback plan? diff --git a/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md b/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md index 9f22fc1ee4..836c91fb25 100644 --- a/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md +++ b/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md @@ -177,7 +177,7 @@ I e during migration create_branch can be called on old pageserver and newly cre The difference of simplistic approach from one described above is that it calls ignore on source tenant first and then calls attach on target pageserver. Approach above does it in opposite order thus opening a possibility for race conditions we strive to avoid. -The approach largely follows this guide: +The approach largely follows this guide: The happy path sequence: diff --git a/scripts/export_import_between_pageservers.py b/scripts/export_import_between_pageservers.py index fca645078a..77e4310eac 100755 --- a/scripts/export_import_between_pageservers.py +++ b/scripts/export_import_between_pageservers.py @@ -18,7 +18,7 @@ # 6. We wait for the new pageserver's remote_consistent_lsn to catch up # # For more context on how to use this, see: -# https://github.com/neondatabase/cloud/wiki/Storage-format-migration +# https://www.notion.so/neondatabase/Storage-format-migration-9a8eba33ccf8417ea8cf50e6a0c542cf import argparse import os diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 161662bc99..98f6677c00 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -449,7 +449,7 @@ def check_neon_works( ) # Check that project can be recovered from WAL - # loosely based on https://github.com/neondatabase/cloud/wiki/Recovery-from-WAL + # loosely based on https://www.notion.so/neondatabase/Storage-Recovery-from-WAL-d92c0aac0ebf40df892b938045d7d720 tenant_id = snapshot_config["default_tenant_id"] timeline_id = dict(snapshot_config["branch_name_mappings"]["main"])[tenant_id] pageserver_port = snapshot_config["pageservers"][0]["listen_http_addr"].split(":")[-1] From 462f04d3779d2399cebd4af8ef9087ec6de88f45 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 14 Nov 2023 19:04:34 +0200 Subject: [PATCH 51/63] Smaller test addition and change (#5858) - trivial serialization roundtrip test for `pageserver::repository::Value` - add missing `start_paused = true` to 15s test making it <0s test - completely unrelated future clippy lint avoidance (helps beta channel users) --- compute_tools/src/extension_server.rs | 78 +++++++++++++-------------- pageserver/src/repository.rs | 65 ++++++++++++++++++++++ pageserver/src/tenant/mgr.rs | 2 +- 3 files changed, 105 insertions(+), 40 deletions(-) diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 2278509c1f..9732d8adea 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -133,45 +133,6 @@ fn parse_pg_version(human_version: &str) -> &str { panic!("Unsuported postgres version {human_version}"); } -#[cfg(test)] -mod tests { - use super::parse_pg_version; - - #[test] - fn test_parse_pg_version() { - assert_eq!(parse_pg_version("PostgreSQL 15.4"), "v15"); - assert_eq!(parse_pg_version("PostgreSQL 15.14"), "v15"); - assert_eq!( - parse_pg_version("PostgreSQL 15.4 (Ubuntu 15.4-0ubuntu0.23.04.1)"), - "v15" - ); - - assert_eq!(parse_pg_version("PostgreSQL 14.15"), "v14"); - assert_eq!(parse_pg_version("PostgreSQL 14.0"), "v14"); - assert_eq!( - parse_pg_version("PostgreSQL 14.9 (Debian 14.9-1.pgdg120+1"), - "v14" - ); - - assert_eq!(parse_pg_version("PostgreSQL 16devel"), "v16"); - assert_eq!(parse_pg_version("PostgreSQL 16beta1"), "v16"); - assert_eq!(parse_pg_version("PostgreSQL 16rc2"), "v16"); - assert_eq!(parse_pg_version("PostgreSQL 16extra"), "v16"); - } - - #[test] - #[should_panic] - fn test_parse_pg_unsupported_version() { - parse_pg_version("PostgreSQL 13.14"); - } - - #[test] - #[should_panic] - fn test_parse_pg_incorrect_version_format() { - parse_pg_version("PostgreSQL 14"); - } -} - // download the archive for a given extension, // unzip it, and place files in the appropriate locations (share/lib) pub async fn download_extension( @@ -285,3 +246,42 @@ pub fn init_remote_storage(remote_ext_config: &str) -> anyhow::Result {{ + let orig: Value = $orig; + + let actual = Value::ser(&orig).unwrap(); + let expected: &[u8] = &$expected; + + assert_eq!(utils::Hex(&actual), utils::Hex(expected)); + + let deser = Value::des(&actual).unwrap(); + + assert_eq!(orig, deser); + }}; + } + + #[test] + fn image_roundtrip() { + let image = Bytes::from_static(b"foobar"); + let image = Value::Image(image); + + #[rustfmt::skip] + let expected = [ + // top level discriminator of 4 bytes + 0x00, 0x00, 0x00, 0x00, + // 8 byte length + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // foobar + 0x66, 0x6f, 0x6f, 0x62, 0x61, 0x72 + ]; + + roundtrip!(image, expected); + } + + #[test] + fn walrecord_postgres_roundtrip() { + let rec = NeonWalRecord::Postgres { + will_init: true, + rec: Bytes::from_static(b"foobar"), + }; + let rec = Value::WalRecord(rec); + + #[rustfmt::skip] + let expected = [ + // flattened discriminator of total 8 bytes + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + // will_init + 0x01, + // 8 byte length + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // foobar + 0x66, 0x6f, 0x6f, 0x62, 0x61, 0x72 + ]; + + roundtrip!(rec, expected); + } +} + /// /// Result of performing GC /// diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index c71eac0672..4363dab375 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -1875,7 +1875,7 @@ mod tests { use super::{super::harness::TenantHarness, TenantsMap}; - #[tokio::test] + #[tokio::test(start_paused = true)] async fn shutdown_awaits_in_progress_tenant() { // Test that if an InProgress tenant is in the map during shutdown, the shutdown will gracefully // wait for it to complete before proceeding. From 2f0d245c2a63d3d82c1007e9f40493f244e0be50 Mon Sep 17 00:00:00 2001 From: khanova <32508607+khanova@users.noreply.github.com> Date: Wed, 15 Nov 2023 10:15:59 +0100 Subject: [PATCH 52/63] Proxy control plane rate limiter (#5785) ## Problem Proxy might overload the control plane. ## Summary of changes Implement rate limiter for proxy<->control plane connection. Resolves https://github.com/neondatabase/neon/issues/5707 Used implementation ideas from https://github.com/conradludgate/squeeze/ --- Cargo.lock | 1 + Cargo.toml | 1 + proxy/Cargo.toml | 1 + proxy/src/bin/proxy.rs | 24 +- proxy/src/http.rs | 5 +- proxy/src/lib.rs | 1 + proxy/src/proxy.rs | 24 +- proxy/src/rate_limiter.rs | 6 + proxy/src/rate_limiter/aimd.rs | 199 ++++++++ proxy/src/rate_limiter/limit_algorithm.rs | 98 ++++ proxy/src/rate_limiter/limiter.rs | 441 ++++++++++++++++++ proxy/src/usage_metrics.rs | 4 +- test_runner/fixtures/neon_fixtures.py | 23 + test_runner/regress/test_proxy.py | 24 + .../regress/test_proxy_rate_limiter.py | 84 ++++ 15 files changed, 930 insertions(+), 6 deletions(-) create mode 100644 proxy/src/rate_limiter.rs create mode 100644 proxy/src/rate_limiter/aimd.rs create mode 100644 proxy/src/rate_limiter/limit_algorithm.rs create mode 100644 proxy/src/rate_limiter/limiter.rs create mode 100644 test_runner/regress/test_proxy_rate_limiter.py diff --git a/Cargo.lock b/Cargo.lock index 4cada013d7..a083af020a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3519,6 +3519,7 @@ dependencies = [ "sha2", "socket2 0.5.3", "sync_wrapper", + "task-local-extensions", "thiserror", "tls-listener", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 363d4c6fe4..bfdb0442ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,6 +136,7 @@ strum_macros = "0.24" svg_fmt = "0.4.1" sync_wrapper = "0.1.2" tar = "0.4" +task-local-extensions = "0.1.4" test-context = "0.1" thiserror = "1.0" tls-listener = { version = "0.7", features = ["rustls", "hyper-h1"] } diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 92498d3ecd..0ec7efd316 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -51,6 +51,7 @@ serde_json.workspace = true sha2.workspace = true socket2.workspace = true sync_wrapper.workspace = true +task-local-extensions.workspace = true thiserror.workspace = true tls-listener.workspace = true tokio-postgres.workspace = true diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 7d1b7eaaae..570cf0943a 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -4,6 +4,7 @@ use proxy::config::AuthenticationConfig; use proxy::config::HttpConfig; use proxy::console; use proxy::http; +use proxy::rate_limiter::RateLimiterConfig; use proxy::usage_metrics; use anyhow::bail; @@ -95,6 +96,20 @@ struct ProxyCliArgs { /// Require that all incoming requests have a Proxy Protocol V2 packet **and** have an IP address associated. #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] require_client_ip: bool, + /// Disable dynamic rate limiter and store the metrics to ensure its production behaviour. + #[clap(long, default_value_t = true, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] + disable_dynamic_rate_limiter: bool, + /// Rate limit algorithm. Makes sense only if `disable_rate_limiter` is `false`. + #[clap(value_enum, long, default_value_t = proxy::rate_limiter::RateLimitAlgorithm::Aimd)] + rate_limit_algorithm: proxy::rate_limiter::RateLimitAlgorithm, + /// Timeout for rate limiter. If it didn't manage to aquire a permit in this time, it will return an error. + #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] + rate_limiter_timeout: tokio::time::Duration, + /// Initial limit for dynamic rate limiter. Makes sense only if `rate_limit_algorithm` is *not* `None`. + #[clap(long, default_value_t = 100)] + initial_limit: usize, + #[clap(flatten)] + aimd_config: proxy::rate_limiter::AimdConfig, } #[tokio::main] @@ -213,6 +228,13 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { and metric-collection-interval must be specified" ), }; + let rate_limiter_config = RateLimiterConfig { + disable: args.disable_dynamic_rate_limiter, + algorithm: args.rate_limit_algorithm, + timeout: args.rate_limiter_timeout, + initial_limit: args.initial_limit, + aimd_config: Some(args.aimd_config), + }; let auth_backend = match &args.auth_backend { AuthBackend::Console => { @@ -237,7 +259,7 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { tokio::spawn(locks.garbage_collect_worker(epoch)); let url = args.auth_endpoint.parse()?; - let endpoint = http::Endpoint::new(url, http::new_client()); + let endpoint = http::Endpoint::new(url, http::new_client(rate_limiter_config)); let api = console::provider::neon::Api::new(endpoint, caches, locks); auth::BackendType::Console(Cow::Owned(api), ()) diff --git a/proxy/src/http.rs b/proxy/src/http.rs index 14a9072a45..159b949da3 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -13,13 +13,13 @@ pub use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; use tokio::time::Instant; use tracing::trace; -use crate::url::ApiUrl; +use crate::{rate_limiter, url::ApiUrl}; use reqwest_middleware::RequestBuilder; /// This is the preferred way to create new http clients, /// because it takes care of observability (OpenTelemetry). /// We deliberately don't want to replace this with a public static. -pub fn new_client() -> ClientWithMiddleware { +pub fn new_client(rate_limiter_config: rate_limiter::RateLimiterConfig) -> ClientWithMiddleware { let client = reqwest::ClientBuilder::new() .dns_resolver(Arc::new(GaiResolver::default())) .connection_verbose(true) @@ -28,6 +28,7 @@ pub fn new_client() -> ClientWithMiddleware { reqwest_middleware::ClientBuilder::new(client) .with(reqwest_tracing::TracingMiddleware::default()) + .with(rate_limiter::Limiter::new(rate_limiter_config)) .build() } diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index 4a95e473f6..a22600cbb3 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -19,6 +19,7 @@ pub mod logging; pub mod parse; pub mod protocol2; pub mod proxy; +pub mod rate_limiter; pub mod sasl; pub mod scram; pub mod serverless; diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index a1ebf03545..2d38acd05b 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -19,7 +19,10 @@ use itertools::Itertools; use metrics::{exponential_buckets, register_int_counter_vec, IntCounterVec}; use once_cell::sync::{Lazy, OnceCell}; use pq_proto::{BeMessage as Be, FeStartupPacket, StartupMessageParams}; -use prometheus::{register_histogram_vec, HistogramVec}; +use prometheus::{ + register_histogram, register_histogram_vec, register_int_gauge_vec, Histogram, HistogramVec, + IntGaugeVec, +}; use regex::Regex; use std::{error::Error, io, ops::ControlFlow, sync::Arc, time::Instant}; use tokio::{ @@ -107,6 +110,25 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { .unwrap() }); +pub static RATE_LIMITER_ACQUIRE_LATENCY: Lazy = Lazy::new(|| { + register_histogram!( + "semaphore_control_plane_token_acquire_seconds", + "Time it took for proxy to establish a connection to the compute endpoint", + // largest bucket = 2^16 * 0.5ms = 32s + exponential_buckets(0.0005, 2.0, 16).unwrap(), + ) + .unwrap() +}); + +pub static RATE_LIMITER_LIMIT: Lazy = Lazy::new(|| { + register_int_gauge_vec!( + "semaphore_control_plane_limit", + "Current limit of the semaphore control plane", + &["limit"], // 2 counters + ) + .unwrap() +}); + pub struct LatencyTimer { // time since the stopwatch was started start: Option, diff --git a/proxy/src/rate_limiter.rs b/proxy/src/rate_limiter.rs new file mode 100644 index 0000000000..5622c44a68 --- /dev/null +++ b/proxy/src/rate_limiter.rs @@ -0,0 +1,6 @@ +mod aimd; +mod limit_algorithm; +mod limiter; +pub use aimd::Aimd; +pub use limit_algorithm::{AimdConfig, Fixed, RateLimitAlgorithm, RateLimiterConfig}; +pub use limiter::Limiter; diff --git a/proxy/src/rate_limiter/aimd.rs b/proxy/src/rate_limiter/aimd.rs new file mode 100644 index 0000000000..c6c532ae53 --- /dev/null +++ b/proxy/src/rate_limiter/aimd.rs @@ -0,0 +1,199 @@ +use std::usize; + +use async_trait::async_trait; + +use super::limit_algorithm::{AimdConfig, LimitAlgorithm, Sample}; + +use super::limiter::Outcome; + +/// Loss-based congestion avoidance. +/// +/// Additive-increase, multiplicative decrease. +/// +/// Adds available currency when: +/// 1. no load-based errors are observed, and +/// 2. the utilisation of the current limit is high. +/// +/// Reduces available concurrency by a factor when load-based errors are detected. +pub struct Aimd { + min_limit: usize, + max_limit: usize, + decrease_factor: f32, + increase_by: usize, + min_utilisation_threshold: f32, +} + +impl Aimd { + pub fn new(config: AimdConfig) -> Self { + Self { + min_limit: config.aimd_min_limit, + max_limit: config.aimd_max_limit, + decrease_factor: config.aimd_decrease_factor, + increase_by: config.aimd_increase_by, + min_utilisation_threshold: config.aimd_min_utilisation_threshold, + } + } + + pub fn decrease_factor(self, factor: f32) -> Self { + assert!((0.5..1.0).contains(&factor)); + Self { + decrease_factor: factor, + ..self + } + } + + pub fn increase_by(self, increase: usize) -> Self { + assert!(increase > 0); + Self { + increase_by: increase, + ..self + } + } + + pub fn with_max_limit(self, max: usize) -> Self { + assert!(max > 0); + Self { + max_limit: max, + ..self + } + } + + /// A threshold below which the limit won't be increased. 0.5 = 50%. + pub fn with_min_utilisation_threshold(self, min_util: f32) -> Self { + assert!(min_util > 0. && min_util < 1.); + Self { + min_utilisation_threshold: min_util, + ..self + } + } +} + +#[async_trait] +impl LimitAlgorithm for Aimd { + async fn update(&mut self, old_limit: usize, sample: Sample) -> usize { + use Outcome::*; + match sample.outcome { + Success => { + let utilisation = sample.in_flight as f32 / old_limit as f32; + + if utilisation > self.min_utilisation_threshold { + let limit = old_limit + self.increase_by; + limit.clamp(self.min_limit, self.max_limit) + } else { + old_limit + } + } + Overload => { + let limit = old_limit as f32 * self.decrease_factor; + + // Floor instead of round, so the limit reduces even with small numbers. + // E.g. round(2 * 0.9) = 2, but floor(2 * 0.9) = 1 + let limit = limit.floor() as usize; + + limit.clamp(self.min_limit, self.max_limit) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tokio::sync::Notify; + + use super::*; + + use crate::rate_limiter::{Limiter, RateLimiterConfig}; + + #[tokio::test] + async fn should_decrease_limit_on_overload() { + let config = RateLimiterConfig { + initial_limit: 10, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let release_notifier = Arc::new(Notify::new()); + + let limiter = Limiter::new(config).with_release_notifier(release_notifier.clone()); + + let token = limiter.try_acquire().unwrap(); + limiter.release(token, Some(Outcome::Overload)).await; + release_notifier.notified().await; + assert_eq!(limiter.state().limit(), 5, "overload: decrease"); + } + + #[tokio::test] + async fn should_increase_limit_on_success_when_using_gt_util_threshold() { + let config = RateLimiterConfig { + initial_limit: 4, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + aimd_min_utilisation_threshold: 0.5, + aimd_increase_by: 1, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + let _token = limiter.try_acquire().unwrap(); + let _token = limiter.try_acquire().unwrap(); + + limiter.release(token, Some(Outcome::Success)).await; + assert_eq!(limiter.state().limit(), 5, "success: increase"); + } + + #[tokio::test] + async fn should_not_change_limit_on_success_when_using_lt_util_threshold() { + let config = RateLimiterConfig { + initial_limit: 4, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + aimd_min_utilisation_threshold: 0.5, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + + limiter.release(token, Some(Outcome::Success)).await; + assert_eq!( + limiter.state().limit(), + 4, + "success: ignore when < half limit" + ); + } + + #[tokio::test] + async fn should_not_change_limit_when_no_outcome() { + let config = RateLimiterConfig { + initial_limit: 10, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + aimd_min_utilisation_threshold: 0.5, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + limiter.release(token, None).await; + assert_eq!(limiter.state().limit(), 10, "ignore"); + } +} diff --git a/proxy/src/rate_limiter/limit_algorithm.rs b/proxy/src/rate_limiter/limit_algorithm.rs new file mode 100644 index 0000000000..5cd2d5ebb7 --- /dev/null +++ b/proxy/src/rate_limiter/limit_algorithm.rs @@ -0,0 +1,98 @@ +//! Algorithms for controlling concurrency limits. +use async_trait::async_trait; +use std::time::Duration; + +use super::{limiter::Outcome, Aimd}; + +/// An algorithm for controlling a concurrency limit. +#[async_trait] +pub trait LimitAlgorithm: Send + Sync + 'static { + /// Update the concurrency limit in response to a new job completion. + async fn update(&mut self, old_limit: usize, sample: Sample) -> usize; +} + +/// The result of a job (or jobs), including the [Outcome] (loss) and latency (delay). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Sample { + pub(crate) latency: Duration, + /// Jobs in flight when the sample was taken. + pub(crate) in_flight: usize, + pub(crate) outcome: Outcome, +} + +#[derive(Clone, Copy, Debug, Default, clap::ValueEnum)] +pub enum RateLimitAlgorithm { + Fixed, + #[default] + Aimd, +} + +pub struct Fixed; + +#[async_trait] +impl LimitAlgorithm for Fixed { + async fn update(&mut self, old_limit: usize, _sample: Sample) -> usize { + old_limit + } +} + +#[derive(Clone, Copy, Debug)] +pub struct RateLimiterConfig { + pub disable: bool, + pub algorithm: RateLimitAlgorithm, + pub timeout: Duration, + pub initial_limit: usize, + pub aimd_config: Option, +} + +impl RateLimiterConfig { + pub fn create_rate_limit_algorithm(self) -> Box { + match self.algorithm { + RateLimitAlgorithm::Fixed => Box::new(Fixed), + RateLimitAlgorithm::Aimd => Box::new(Aimd::new(self.aimd_config.unwrap())), // For aimd algorithm config is mandatory. + } + } +} + +impl Default for RateLimiterConfig { + fn default() -> Self { + Self { + disable: true, + algorithm: RateLimitAlgorithm::Aimd, + timeout: Duration::from_secs(1), + initial_limit: 100, + aimd_config: Some(AimdConfig::default()), + } + } +} + +#[derive(clap::Parser, Clone, Copy, Debug)] +pub struct AimdConfig { + /// Minimum limit for AIMD algorithm. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 1)] + pub aimd_min_limit: usize, + /// Maximum limit for AIMD algorithm. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 1500)] + pub aimd_max_limit: usize, + /// Increase AIMD increase by value in case of success. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 10)] + pub aimd_increase_by: usize, + /// Decrease AIMD decrease by value in case of timout/429. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 0.9)] + pub aimd_decrease_factor: f32, + /// A threshold below which the limit won't be increased. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 0.8)] + pub aimd_min_utilisation_threshold: f32, +} + +impl Default for AimdConfig { + fn default() -> Self { + Self { + aimd_min_limit: 1, + aimd_max_limit: 1500, + aimd_increase_by: 10, + aimd_decrease_factor: 0.9, + aimd_min_utilisation_threshold: 0.8, + } + } +} diff --git a/proxy/src/rate_limiter/limiter.rs b/proxy/src/rate_limiter/limiter.rs new file mode 100644 index 0000000000..3a9fed3919 --- /dev/null +++ b/proxy/src/rate_limiter/limiter.rs @@ -0,0 +1,441 @@ +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +use tokio::sync::{Mutex as AsyncMutex, Semaphore, SemaphorePermit}; +use tokio::time::{timeout, Instant}; +use tracing::info; + +use super::{ + limit_algorithm::{LimitAlgorithm, Sample}, + RateLimiterConfig, +}; + +/// Limits the number of concurrent jobs. +/// +/// Concurrency is limited through the use of [Token]s. Acquire a token to run a job, and release the +/// token once the job is finished. +/// +/// The limit will be automatically adjusted based on observed latency (delay) and/or failures +/// caused by overload (loss). +pub struct Limiter { + limit_algo: AsyncMutex>, + semaphore: std::sync::Arc, + config: RateLimiterConfig, + + // ONLY WRITE WHEN LIMIT_ALGO IS LOCKED + limits: AtomicUsize, + + // ONLY USE ATOMIC ADD/SUB + in_flight: Arc, + + #[cfg(test)] + notifier: Option>, +} + +/// A concurrency token, required to run a job. +/// +/// Release the token back to the [Limiter] after the job is complete. +#[derive(Debug)] +pub struct Token<'t> { + permit: Option>, + start: Instant, + in_flight: Arc, +} + +/// A snapshot of the state of the [Limiter]. +/// +/// Not guaranteed to be consistent under high concurrency. +#[derive(Debug, Clone, Copy)] +pub struct LimiterState { + limit: usize, + available: usize, + in_flight: usize, +} + +/// Whether a job succeeded or failed as a result of congestion/overload. +/// +/// Errors not considered to be caused by overload should be ignored. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// The job succeeded, or failed in a way unrelated to overload. + Success, + /// The job failed because of overload, e.g. it timed out or an explicit backpressure signal + /// was observed. + Overload, +} + +impl Outcome { + fn from_reqwest_error(error: &reqwest_middleware::Error) -> Self { + match error { + reqwest_middleware::Error::Middleware(_) => Outcome::Success, + reqwest_middleware::Error::Reqwest(e) => { + if let Some(status) = e.status() { + if status.is_server_error() + || reqwest::StatusCode::TOO_MANY_REQUESTS.as_u16() == status + { + Outcome::Overload + } else { + Outcome::Success + } + } else { + Outcome::Success + } + } + } + } + fn from_reqwest_response(response: &reqwest::Response) -> Self { + if response.status().is_server_error() + || response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS + { + Outcome::Overload + } else { + Outcome::Success + } + } +} + +impl Limiter { + /// Create a limiter with a given limit control algorithm. + pub fn new(config: RateLimiterConfig) -> Self { + assert!(config.initial_limit > 0); + Self { + limit_algo: AsyncMutex::new(config.create_rate_limit_algorithm()), + semaphore: Arc::new(Semaphore::new(config.initial_limit)), + config, + limits: AtomicUsize::new(config.initial_limit), + in_flight: Arc::new(AtomicUsize::new(0)), + #[cfg(test)] + notifier: None, + } + } + // pub fn new(limit_algorithm: T, timeout: Duration, initial_limit: usize) -> Self { + // assert!(initial_limit > 0); + + // Self { + // limit_algo: AsyncMutex::new(limit_algorithm), + // semaphore: Arc::new(Semaphore::new(initial_limit)), + // timeout, + // limits: AtomicUsize::new(initial_limit), + // in_flight: Arc::new(AtomicUsize::new(0)), + // #[cfg(test)] + // notifier: None, + // } + // } + + /// In some cases [Token]s are acquired asynchronously when updating the limit. + #[cfg(test)] + pub fn with_release_notifier(mut self, n: std::sync::Arc) -> Self { + self.notifier = Some(n); + self + } + + /// Try to immediately acquire a concurrency [Token]. + /// + /// Returns `None` if there are none available. + pub fn try_acquire(&self) -> Option { + let result = if self.config.disable { + // If the rate limiter is disabled, we can always acquire a token. + Some(Token::new(None, self.in_flight.clone())) + } else { + self.semaphore + .try_acquire() + .map(|permit| Token::new(Some(permit), self.in_flight.clone())) + .ok() + }; + if result.is_some() { + self.in_flight.fetch_add(1, Ordering::AcqRel); + } + result + } + + /// Try to acquire a concurrency [Token], waiting for `duration` if there are none available. + /// + /// Returns `None` if there are none available after `duration`. + pub async fn acquire_timeout(&self, duration: Duration) -> Option> { + info!("acquiring token: {:?}", self.semaphore.available_permits()); + let result = if self.config.disable { + // If the rate limiter is disabled, we can always acquire a token. + Some(Token::new(None, self.in_flight.clone())) + } else { + match timeout(duration, self.semaphore.acquire()).await { + Ok(maybe_permit) => maybe_permit + .map(|permit| Token::new(Some(permit), self.in_flight.clone())) + .ok(), + Err(_) => None, + } + }; + if result.is_some() { + self.in_flight.fetch_add(1, Ordering::AcqRel); + } + result + } + + /// Return the concurrency [Token], along with the outcome of the job. + /// + /// The [Outcome] of the job, and the time taken to perform it, may be used + /// to update the concurrency limit. + /// + /// Set the outcome to `None` to ignore the job. + pub async fn release(&self, mut token: Token<'_>, outcome: Option) { + tracing::info!("outcome is {:?}", outcome); + let in_flight = self.in_flight.load(Ordering::Acquire); + let old_limit = self.limits.load(Ordering::Acquire); + let available = if self.config.disable { + 0 // This is not used in the algorithm and can be anything. If the config disable it makes sense to set it to 0. + } else { + self.semaphore.available_permits() + }; + let total = in_flight + available; + + let mut algo = self.limit_algo.lock().await; + + let new_limit = if let Some(outcome) = outcome { + let sample = Sample { + latency: token.start.elapsed(), + in_flight, + outcome, + }; + algo.update(old_limit, sample).await + } else { + old_limit + }; + tracing::info!("new limit is {}", new_limit); + let actual_limit = if new_limit < total { + token.forget(); + total.saturating_sub(1) + } else { + if !self.config.disable { + self.semaphore.add_permits(new_limit.saturating_sub(total)); + } + new_limit + }; + crate::proxy::RATE_LIMITER_LIMIT + .with_label_values(&["expected"]) + .set(new_limit as i64); + crate::proxy::RATE_LIMITER_LIMIT + .with_label_values(&["actual"]) + .set(actual_limit as i64); + self.limits.store(new_limit, Ordering::Release); + #[cfg(test)] + if let Some(n) = &self.notifier { + n.notify_one(); + } + } + + /// The current state of the limiter. + pub fn state(&self) -> LimiterState { + let limit = self.limits.load(Ordering::Relaxed); + let in_flight = self.in_flight.load(Ordering::Relaxed); + LimiterState { + limit, + available: limit.saturating_sub(in_flight), + in_flight, + } + } +} + +impl<'t> Token<'t> { + fn new(permit: Option>, in_flight: Arc) -> Self { + Self { + permit, + start: Instant::now(), + in_flight, + } + } + + #[cfg(test)] + pub fn set_latency(&mut self, latency: Duration) { + use std::ops::Sub; + + self.start = Instant::now().sub(latency); + } + + pub fn forget(&mut self) { + if let Some(permit) = self.permit.take() { + permit.forget(); + } + } +} + +impl Drop for Token<'_> { + fn drop(&mut self) { + self.in_flight.fetch_sub(1, Ordering::AcqRel); + } +} + +impl LimiterState { + /// The current concurrency limit. + pub fn limit(&self) -> usize { + self.limit + } + /// The amount of concurrency available to use. + pub fn available(&self) -> usize { + self.available + } + /// The number of jobs in flight. + pub fn in_flight(&self) -> usize { + self.in_flight + } +} + +#[async_trait::async_trait] +impl reqwest_middleware::Middleware for Limiter { + async fn handle( + &self, + req: reqwest::Request, + extensions: &mut task_local_extensions::Extensions, + next: reqwest_middleware::Next<'_>, + ) -> reqwest_middleware::Result { + let start = Instant::now(); + let token = self + .acquire_timeout(self.config.timeout) + .await + .ok_or_else(|| { + reqwest_middleware::Error::Middleware( + // TODO: Should we map it into user facing errors? + crate::console::errors::ApiError::Console { + status: crate::http::StatusCode::TOO_MANY_REQUESTS, + text: "Too many requests".into(), + } + .into(), + ) + })?; + info!(duration = ?start.elapsed(), "waiting for token to connect to the control plane"); + crate::proxy::RATE_LIMITER_ACQUIRE_LATENCY.observe(start.elapsed().as_secs_f64()); + match next.run(req, extensions).await { + Ok(response) => { + self.release(token, Some(Outcome::from_reqwest_response(&response))) + .await; + Ok(response) + } + Err(e) => { + self.release(token, Some(Outcome::from_reqwest_error(&e))) + .await; + Err(e) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{pin::pin, task::Context, time::Duration}; + + use futures::{task::noop_waker_ref, Future}; + + use super::{Limiter, Outcome}; + use crate::rate_limiter::RateLimitAlgorithm; + + #[tokio::test] + async fn it_works() { + let config = super::RateLimiterConfig { + algorithm: RateLimitAlgorithm::Fixed, + timeout: Duration::from_secs(1), + initial_limit: 10, + disable: false, + ..Default::default() + }; + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + + limiter.release(token, Some(Outcome::Success)).await; + + assert_eq!(limiter.state().limit(), 10); + } + + #[tokio::test] + async fn is_fair() { + let config = super::RateLimiterConfig { + algorithm: RateLimitAlgorithm::Fixed, + timeout: Duration::from_secs(1), + initial_limit: 1, + disable: false, + ..Default::default() + }; + let limiter = Limiter::new(config); + + // === TOKEN 1 === + let token1 = limiter.try_acquire().unwrap(); + + let mut token2_fut = pin!(limiter.acquire_timeout(Duration::from_secs(1))); + assert!( + token2_fut + .as_mut() + .poll(&mut Context::from_waker(noop_waker_ref())) + .is_pending(), + "token is acquired by token1" + ); + + let mut token3_fut = pin!(limiter.acquire_timeout(Duration::from_secs(1))); + assert!( + token3_fut + .as_mut() + .poll(&mut Context::from_waker(noop_waker_ref())) + .is_pending(), + "token is acquired by token1" + ); + + limiter.release(token1, Some(Outcome::Success)).await; + // === END TOKEN 1 === + + // === TOKEN 2 === + assert!( + limiter.try_acquire().is_none(), + "token is acquired by token2" + ); + + assert!( + token3_fut + .as_mut() + .poll(&mut Context::from_waker(noop_waker_ref())) + .is_pending(), + "token is acquired by token2" + ); + + let token2 = token2_fut.await.unwrap(); + + limiter.release(token2, Some(Outcome::Success)).await; + // === END TOKEN 2 === + + // === TOKEN 3 === + assert!( + limiter.try_acquire().is_none(), + "token is acquired by token3" + ); + + let token3 = token3_fut.await.unwrap(); + limiter.release(token3, Some(Outcome::Success)).await; + // === END TOKEN 3 === + + // === TOKEN 4 === + let token4 = limiter.try_acquire().unwrap(); + limiter.release(token4, Some(Outcome::Success)).await; + } + + #[tokio::test] + async fn disable() { + let config = super::RateLimiterConfig { + algorithm: RateLimitAlgorithm::Fixed, + timeout: Duration::from_secs(1), + initial_limit: 1, + disable: true, + ..Default::default() + }; + let limiter = Limiter::new(config); + + // === TOKEN 1 === + let token1 = limiter.try_acquire().unwrap(); + let token2 = limiter.try_acquire().unwrap(); + let state = limiter.state(); + assert_eq!(state.limit(), 1); + assert_eq!(state.in_flight(), 2); // For disabled limiter, it's expected. + limiter.release(token1, None).await; + limiter.release(token2, None).await; + } +} diff --git a/proxy/src/usage_metrics.rs b/proxy/src/usage_metrics.rs index cfeec5622b..180b5f7199 100644 --- a/proxy/src/usage_metrics.rs +++ b/proxy/src/usage_metrics.rs @@ -249,7 +249,7 @@ mod tests { use url::Url; use super::{collect_metrics_iteration, Ids, Metrics}; - use crate::http; + use crate::{http, rate_limiter::RateLimiterConfig}; #[tokio::test] async fn metrics() { @@ -279,7 +279,7 @@ mod tests { tokio::spawn(server); let metrics = Metrics::default(); - let client = http::new_client(); + let client = http::new_client(RateLimiterConfig::default()); let endpoint = Url::parse(&format!("http://{addr}")).unwrap(); let now = Utc::now(); diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index b45b0a12c0..6737ca5fe3 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2179,6 +2179,29 @@ class NeonProxy(PgProtocol): *["--allow-self-signed-compute", "true"], ] + class Console(AuthBackend): + def __init__(self, endpoint: str, fixed_rate_limit: Optional[int] = None): + self.endpoint = endpoint + self.fixed_rate_limit = fixed_rate_limit + + def extra_args(self) -> list[str]: + args = [ + # Console auth backend params + *["--auth-backend", "console"], + *["--auth-endpoint", self.endpoint], + ] + if self.fixed_rate_limit is not None: + args += [ + *["--disable-dynamic-rate-limiter", "false"], + *["--rate-limit-algorithm", "aimd"], + *["--initial-limit", str(1)], + *["--rate-limiter-timeout", "1s"], + *["--aimd-min-limit", "0"], + *["--aimd-increase-by", "1"], + *["--wake-compute-cache", "size=0"], # Disable cache to test rate limiter. + ] + return args + @dataclass(frozen=True) class Postgres(AuthBackend): pg_conn_url: str diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index c93cdf637a..0f2cd9768f 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -1,3 +1,4 @@ +import asyncio import json import subprocess import time @@ -11,6 +12,29 @@ from fixtures.neon_fixtures import PSQL, NeonProxy, VanillaPostgres GET_CONNECTION_PID_QUERY = "SELECT pid FROM pg_stat_activity WHERE state = 'active'" +@pytest.mark.asyncio +async def test_http_pool_begin_1(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth with password 'http' superuser") + + def query(*args) -> Any: + static_proxy.http_query( + "SELECT pg_sleep(10);", + args, + user="http_auth", + password="http", + expected_code=200, + ) + + query() + loop = asyncio.get_running_loop() + tasks = [loop.run_in_executor(None, query) for _ in range(10)] + # Wait for all the tasks to complete + completed, pending = await asyncio.wait(tasks) + # Get the results + results = [task.result() for task in completed] + print(results) + + def test_proxy_select_1(static_proxy: NeonProxy): """ A simplest smoke test: check proxy against a local postgres instance. diff --git a/test_runner/regress/test_proxy_rate_limiter.py b/test_runner/regress/test_proxy_rate_limiter.py new file mode 100644 index 0000000000..f39f0cad07 --- /dev/null +++ b/test_runner/regress/test_proxy_rate_limiter.py @@ -0,0 +1,84 @@ +import asyncio +import time +from pathlib import Path +from typing import Iterator + +import pytest +from fixtures.neon_fixtures import ( + PSQL, + NeonProxy, +) +from fixtures.port_distributor import PortDistributor +from pytest_httpserver import HTTPServer +from werkzeug.wrappers.response import Response + + +def waiting_handler(status_code: int) -> Response: + # wait more than timeout to make sure that both (two) connections are open. + # It would be better to use a barrier here, but I don't know how to do that together with pytest-httpserver. + time.sleep(2) + return Response(status=status_code) + + +@pytest.fixture(scope="function") +def proxy_with_rate_limit( + port_distributor: PortDistributor, + neon_binpath: Path, + httpserver_listen_address, + test_output_dir: Path, +) -> Iterator[NeonProxy]: + """Neon proxy that routes directly to vanilla postgres.""" + + proxy_port = port_distributor.get_port() + mgmt_port = port_distributor.get_port() + http_port = port_distributor.get_port() + external_http_port = port_distributor.get_port() + (host, port) = httpserver_listen_address + endpoint = f"http://{host}:{port}/billing/api/v1/usage_events" + + with NeonProxy( + neon_binpath=neon_binpath, + test_output_dir=test_output_dir, + proxy_port=proxy_port, + http_port=http_port, + mgmt_port=mgmt_port, + external_http_port=external_http_port, + auth_backend=NeonProxy.Console(endpoint, fixed_rate_limit=5), + ) as proxy: + proxy.start() + yield proxy + + +@pytest.mark.asyncio +async def test_proxy_rate_limit( + httpserver: HTTPServer, + proxy_with_rate_limit: NeonProxy, +): + uri = "/billing/api/v1/usage_events/proxy_get_role_secret" + # mock control plane service + httpserver.expect_ordered_request(uri, method="GET").respond_with_handler( + lambda _: Response(status=200) + ) + httpserver.expect_ordered_request(uri, method="GET").respond_with_handler( + lambda _: waiting_handler(429) + ) + httpserver.expect_ordered_request(uri, method="GET").respond_with_handler( + lambda _: waiting_handler(500) + ) + + psql = PSQL(host=proxy_with_rate_limit.host, port=proxy_with_rate_limit.proxy_port) + f = await psql.run("select 42;") + await proxy_with_rate_limit.find_auth_link(uri, f) + # Limit should be 2. + + # Run two queries in parallel. + f1, f2 = await asyncio.gather(psql.run("select 42;"), psql.run("select 42;")) + await proxy_with_rate_limit.find_auth_link(uri, f1) + await proxy_with_rate_limit.find_auth_link(uri, f2) + + # Now limit should be 0. + f = await psql.run("select 42;") + await proxy_with_rate_limit.find_auth_link(uri, f) + + # There last query shouldn't reach the http-server. + assert httpserver.assertions == [] From 5cd5b93066520f30e2b0041ad64858b612cb0e70 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 11:08:49 +0000 Subject: [PATCH 53/63] build(deps): bump aiohttp from 3.8.5 to 3.8.6 (#5864) --- poetry.lock | 178 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 90 insertions(+), 90 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0beae18cdf..58ab4e70f2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,98 +2,98 @@ [[package]] name = "aiohttp" -version = "3.8.5" +version = "3.8.6" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.6" files = [ - {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, - {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, - {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"}, - {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"}, - {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"}, - {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"}, - {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"}, - {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"}, - {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"}, - {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"}, - {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"}, - {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"}, - {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"}, - {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"}, - {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, - {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, + {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41d55fc043954cddbbd82503d9cc3f4814a40bcef30b3569bc7b5e34130718c1"}, + {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1d84166673694841d8953f0a8d0c90e1087739d24632fe86b1a08819168b4566"}, + {file = "aiohttp-3.8.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:253bf92b744b3170eb4c4ca2fa58f9c4b87aeb1df42f71d4e78815e6e8b73c9e"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fd194939b1f764d6bb05490987bfe104287bbf51b8d862261ccf66f48fb4096"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c5f938d199a6fdbdc10bbb9447496561c3a9a565b43be564648d81e1102ac22"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2817b2f66ca82ee699acd90e05c95e79bbf1dc986abb62b61ec8aaf851e81c93"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fa375b3d34e71ccccf172cab401cd94a72de7a8cc01847a7b3386204093bb47"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9de50a199b7710fa2904be5a4a9b51af587ab24c8e540a7243ab737b45844543"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e1d8cb0b56b3587c5c01de3bf2f600f186da7e7b5f7353d1bf26a8ddca57f965"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8e31e9db1bee8b4f407b77fd2507337a0a80665ad7b6c749d08df595d88f1cf5"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7bc88fc494b1f0311d67f29fee6fd636606f4697e8cc793a2d912ac5b19aa38d"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ec00c3305788e04bf6d29d42e504560e159ccaf0be30c09203b468a6c1ccd3b2"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad1407db8f2f49329729564f71685557157bfa42b48f4b93e53721a16eb813ed"}, + {file = "aiohttp-3.8.6-cp310-cp310-win32.whl", hash = "sha256:ccc360e87341ad47c777f5723f68adbb52b37ab450c8bc3ca9ca1f3e849e5fe2"}, + {file = "aiohttp-3.8.6-cp310-cp310-win_amd64.whl", hash = "sha256:93c15c8e48e5e7b89d5cb4613479d144fda8344e2d886cf694fd36db4cc86865"}, + {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e2f9cc8e5328f829f6e1fb74a0a3a939b14e67e80832975e01929e320386b34"}, + {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e6a00ffcc173e765e200ceefb06399ba09c06db97f401f920513a10c803604ca"}, + {file = "aiohttp-3.8.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:41bdc2ba359032e36c0e9de5a3bd00d6fb7ea558a6ce6b70acedf0da86458321"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14cd52ccf40006c7a6cd34a0f8663734e5363fd981807173faf3a017e202fec9"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2d5b785c792802e7b275c420d84f3397668e9d49ab1cb52bd916b3b3ffcf09ad"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1bed815f3dc3d915c5c1e556c397c8667826fbc1b935d95b0ad680787896a358"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96603a562b546632441926cd1293cfcb5b69f0b4159e6077f7c7dbdfb686af4d"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d76e8b13161a202d14c9584590c4df4d068c9567c99506497bdd67eaedf36403"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e3f1e3f1a1751bb62b4a1b7f4e435afcdade6c17a4fd9b9d43607cebd242924a"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:76b36b3124f0223903609944a3c8bf28a599b2cc0ce0be60b45211c8e9be97f8"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:a2ece4af1f3c967a4390c284797ab595a9f1bc1130ef8b01828915a05a6ae684"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:16d330b3b9db87c3883e565340d292638a878236418b23cc8b9b11a054aaa887"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:42c89579f82e49db436b69c938ab3e1559e5a4409eb8639eb4143989bc390f2f"}, + {file = "aiohttp-3.8.6-cp311-cp311-win32.whl", hash = "sha256:efd2fcf7e7b9d7ab16e6b7d54205beded0a9c8566cb30f09c1abe42b4e22bdcb"}, + {file = "aiohttp-3.8.6-cp311-cp311-win_amd64.whl", hash = "sha256:3b2ab182fc28e7a81f6c70bfbd829045d9480063f5ab06f6e601a3eddbbd49a0"}, + {file = "aiohttp-3.8.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fdee8405931b0615220e5ddf8cd7edd8592c606a8e4ca2a00704883c396e4479"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d25036d161c4fe2225d1abff2bd52c34ed0b1099f02c208cd34d8c05729882f0"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d791245a894be071d5ab04bbb4850534261a7d4fd363b094a7b9963e8cdbd31"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0cccd1de239afa866e4ce5c789b3032442f19c261c7d8a01183fd956b1935349"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f13f60d78224f0dace220d8ab4ef1dbc37115eeeab8c06804fec11bec2bbd07"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a9b5a0606faca4f6cc0d338359d6fa137104c337f489cd135bb7fbdbccb1e39"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:13da35c9ceb847732bf5c6c5781dcf4780e14392e5d3b3c689f6d22f8e15ae31"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:4d4cbe4ffa9d05f46a28252efc5941e0462792930caa370a6efaf491f412bc66"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:229852e147f44da0241954fc6cb910ba074e597f06789c867cb7fb0621e0ba7a"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:713103a8bdde61d13490adf47171a1039fd880113981e55401a0f7b42c37d071"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:45ad816b2c8e3b60b510f30dbd37fe74fd4a772248a52bb021f6fd65dff809b6"}, + {file = "aiohttp-3.8.6-cp36-cp36m-win32.whl", hash = "sha256:2b8d4e166e600dcfbff51919c7a3789ff6ca8b3ecce16e1d9c96d95dd569eb4c"}, + {file = "aiohttp-3.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:0912ed87fee967940aacc5306d3aa8ba3a459fcd12add0b407081fbefc931e53"}, + {file = "aiohttp-3.8.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e2a988a0c673c2e12084f5e6ba3392d76c75ddb8ebc6c7e9ead68248101cd446"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf3fd9f141700b510d4b190094db0ce37ac6361a6806c153c161dc6c041ccda"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3161ce82ab85acd267c8f4b14aa226047a6bee1e4e6adb74b798bd42c6ae1f80"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95fc1bf33a9a81469aa760617b5971331cdd74370d1214f0b3109272c0e1e3c"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c43ecfef7deaf0617cee936836518e7424ee12cb709883f2c9a1adda63cc460"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca80e1b90a05a4f476547f904992ae81eda5c2c85c66ee4195bb8f9c5fb47f28"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:90c72ebb7cb3a08a7f40061079817133f502a160561d0675b0a6adf231382c92"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bb54c54510e47a8c7c8e63454a6acc817519337b2b78606c4e840871a3e15349"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:de6a1c9f6803b90e20869e6b99c2c18cef5cc691363954c93cb9adeb26d9f3ae"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:a3628b6c7b880b181a3ae0a0683698513874df63783fd89de99b7b7539e3e8a8"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:fc37e9aef10a696a5a4474802930079ccfc14d9f9c10b4662169671ff034b7df"}, + {file = "aiohttp-3.8.6-cp37-cp37m-win32.whl", hash = "sha256:f8ef51e459eb2ad8e7a66c1d6440c808485840ad55ecc3cafefadea47d1b1ba2"}, + {file = "aiohttp-3.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:b2fe42e523be344124c6c8ef32a011444e869dc5f883c591ed87f84339de5976"}, + {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9e2ee0ac5a1f5c7dd3197de309adfb99ac4617ff02b0603fd1e65b07dc772e4b"}, + {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01770d8c04bd8db568abb636c1fdd4f7140b284b8b3e0b4584f070180c1e5c62"}, + {file = "aiohttp-3.8.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c68330a59506254b556b99a91857428cab98b2f84061260a67865f7f52899f5"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89341b2c19fb5eac30c341133ae2cc3544d40d9b1892749cdd25892bbc6ac951"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71783b0b6455ac8f34b5ec99d83e686892c50498d5d00b8e56d47f41b38fbe04"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f628dbf3c91e12f4d6c8b3f092069567d8eb17814aebba3d7d60c149391aee3a"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b04691bc6601ef47c88f0255043df6f570ada1a9ebef99c34bd0b72866c217ae"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee912f7e78287516df155f69da575a0ba33b02dd7c1d6614dbc9463f43066e3"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9c19b26acdd08dd239e0d3669a3dddafd600902e37881f13fbd8a53943079dbc"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:99c5ac4ad492b4a19fc132306cd57075c28446ec2ed970973bbf036bcda1bcc6"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f0f03211fd14a6a0aed2997d4b1c013d49fb7b50eeb9ffdf5e51f23cfe2c77fa"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:8d399dade330c53b4106160f75f55407e9ae7505263ea86f2ccca6bfcbdb4921"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ec4fd86658c6a8964d75426517dc01cbf840bbf32d055ce64a9e63a40fd7b771"}, + {file = "aiohttp-3.8.6-cp38-cp38-win32.whl", hash = "sha256:33164093be11fcef3ce2571a0dccd9041c9a93fa3bde86569d7b03120d276c6f"}, + {file = "aiohttp-3.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:bdf70bfe5a1414ba9afb9d49f0c912dc524cf60141102f3a11143ba3d291870f"}, + {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d52d5dc7c6682b720280f9d9db41d36ebe4791622c842e258c9206232251ab2b"}, + {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ac39027011414dbd3d87f7edb31680e1f430834c8cef029f11c66dad0670aa5"}, + {file = "aiohttp-3.8.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3f5c7ce535a1d2429a634310e308fb7d718905487257060e5d4598e29dc17f0b"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b30e963f9e0d52c28f284d554a9469af073030030cef8693106d918b2ca92f54"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:918810ef188f84152af6b938254911055a72e0f935b5fbc4c1a4ed0b0584aed1"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:002f23e6ea8d3dd8d149e569fd580c999232b5fbc601c48d55398fbc2e582e8c"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fcf3eabd3fd1a5e6092d1242295fa37d0354b2eb2077e6eb670accad78e40e1"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:255ba9d6d5ff1a382bb9a578cd563605aa69bec845680e21c44afc2670607a95"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d67f8baed00870aa390ea2590798766256f31dc5ed3ecc737debb6e97e2ede78"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:86f20cee0f0a317c76573b627b954c412ea766d6ada1a9fcf1b805763ae7feeb"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:39a312d0e991690ccc1a61f1e9e42daa519dcc34ad03eb6f826d94c1190190dd"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e827d48cf802de06d9c935088c2924e3c7e7533377d66b6f31ed175c1620e05e"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bd111d7fc5591ddf377a408ed9067045259ff2770f37e2d94e6478d0f3fc0c17"}, + {file = "aiohttp-3.8.6-cp39-cp39-win32.whl", hash = "sha256:caf486ac1e689dda3502567eb89ffe02876546599bbf915ec94b1fa424eeffd4"}, + {file = "aiohttp-3.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:3f0e27e5b733803333bb2371249f41cf42bae8884863e8e8965ec69bebe53132"}, + {file = "aiohttp-3.8.6.tar.gz", hash = "sha256:b0cf2a4501bff9330a8a5248b4ce951851e415bdcce9dc158e76cfd55e15085c"}, ] [package.dependencies] @@ -2719,4 +2719,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "74649cf47c52f21b01b096a42044750b1c9677576b405be0489c2909127a9bf1" +content-hash = "0834e5cb69e5457741d4f476c3e49a4dc83598b5730685c8755da651b96ad3ec" diff --git a/pyproject.toml b/pyproject.toml index ebf69c636e..396edabe10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ psutil = "^5.9.4" types-psutil = "^5.9.5.12" types-toml = "^0.10.8.6" pytest-httpserver = "^1.0.8" -aiohttp = "3.8.5" +aiohttp = "3.8.6" pytest-rerunfailures = "^11.1.2" types-pytest-lazy-fixture = "^0.6.3.3" pytest-split = "^0.8.1" From f84ac2b98d2a67950f53bc08b730d387a9c25a21 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Wed, 15 Nov 2023 12:40:21 +0100 Subject: [PATCH 54/63] Fix baseline commit and branch for code coverage (#5769) ## Problem `HEAD` commit for a PR is a phantom merge commit which skews the baseline commit for coverage reports. See https://github.com/neondatabase/neon/pull/5751#issuecomment-1790717867 ## Summary of changes - Use commit hash instead of `HEAD` for finding baseline commits for code coverage - Use the base branch for PRs or the current branch for pushes --- .github/workflows/build_and_test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ff7d8c1a62..ed7c0d3dfa 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -586,10 +586,13 @@ jobs: id: upload-coverage-report-new env: BUCKET: neon-github-public-dev + # A differential coverage report is available only for PRs. + # (i.e. for pushes into main/release branches we have a regular coverage report) COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} run: | - BASELINE="$(git merge-base HEAD origin/main)" CURRENT="${COMMIT_SHA}" + BASELINE="$(git merge-base $BASE_SHA $CURRENT)" cp /tmp/coverage/report/lcov.info ./${CURRENT}.info From ab631e67927d9be4a538e659a286fa70e50c8777 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 15 Nov 2023 22:20:21 +0100 Subject: [PATCH 55/63] pageserver: make TenantsMap shard-aware (#5819) ## Problem When using TenantId as the key, we are unable to handle multiple tenant shards attached to the same pageserver for the same tenant ID. This is an expected scenario if we have e.g. 8 shards and 5 pageservers. ## Summary of changes - TenantsMap is now a BTreeMap instead of a HashMap: this enables looking up by range. In future, we will need this for page_service, as incoming requests will just specify the Key, and we'll have to figure out which shard to route it to. - A new key type TenantShardId is introduced, to act as the key in TenantsMap, and as the id type in external APIs. Its human readable serialization is backward compatible with TenantId, and also forward-compatible as long as sharding is not actually used (when we construct a TenantShardId with ShardCount(0), it serializes to an old-fashioned TenantId). - Essential tenant APIs are updated to accept TenantShardIds: tenant/timeline create, tenant delete, and /location_conf. These are the APIs that will enable driving sharded tenants. Other apis like /attach /detach /load /ignore will not work with sharding: those will soon be deprecated and replaced with /location_conf as part of the live migration work. Closes: #5787 --- Cargo.lock | 2 + control_plane/src/pageserver.rs | 3 +- libs/pageserver_api/Cargo.toml | 4 + libs/pageserver_api/src/key.rs | 142 +++++++++++++ libs/pageserver_api/src/lib.rs | 2 + libs/pageserver_api/src/models.rs | 4 +- libs/pageserver_api/src/shard.rs | 321 +++++++++++++++++++++++++++++ pageserver/src/http/routes.rs | 62 +++--- pageserver/src/repository.rs | 142 +------------ pageserver/src/tenant/mgr.rs | 331 ++++++++++++++++++++---------- 10 files changed, 744 insertions(+), 269 deletions(-) create mode 100644 libs/pageserver_api/src/key.rs create mode 100644 libs/pageserver_api/src/shard.rs diff --git a/Cargo.lock b/Cargo.lock index a083af020a..841c60c7e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2994,10 +2994,12 @@ name = "pageserver_api" version = "0.1.0" dependencies = [ "anyhow", + "bincode", "byteorder", "bytes", "const_format", "enum-map", + "hex", "postgres_ffi", "serde", "serde_json", diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index e13a234e89..237df48543 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -18,6 +18,7 @@ use camino::Utf8PathBuf; use pageserver_api::models::{ self, LocationConfig, TenantInfo, TenantLocationConfigRequest, TimelineInfo, }; +use pageserver_api::shard::TenantShardId; use postgres_backend::AuthType; use postgres_connection::{parse_host_port, PgConnectionConfig}; use reqwest::blocking::{Client, RequestBuilder, Response}; @@ -408,7 +409,7 @@ impl PageServerNode { }; let request = models::TenantCreateRequest { - new_tenant_id, + new_tenant_id: TenantShardId::unsharded(new_tenant_id), generation, config, }; diff --git a/libs/pageserver_api/Cargo.toml b/libs/pageserver_api/Cargo.toml index f97ec54e91..df9796b039 100644 --- a/libs/pageserver_api/Cargo.toml +++ b/libs/pageserver_api/Cargo.toml @@ -17,5 +17,9 @@ postgres_ffi.workspace = true enum-map.workspace = true strum.workspace = true strum_macros.workspace = true +hex.workspace = true workspace_hack.workspace = true + +[dev-dependencies] +bincode.workspace = true diff --git a/libs/pageserver_api/src/key.rs b/libs/pageserver_api/src/key.rs new file mode 100644 index 0000000000..b5350d6384 --- /dev/null +++ b/libs/pageserver_api/src/key.rs @@ -0,0 +1,142 @@ +use anyhow::{bail, Result}; +use byteorder::{ByteOrder, BE}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// Key used in the Repository kv-store. +/// +/// The Repository treats this as an opaque struct, but see the code in pgdatadir_mapping.rs +/// for what we actually store in these fields. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct Key { + pub field1: u8, + pub field2: u32, + pub field3: u32, + pub field4: u32, + pub field5: u8, + pub field6: u32, +} + +pub const KEY_SIZE: usize = 18; + +impl Key { + /// 'field2' is used to store tablespaceid for relations and small enum numbers for other relish. + /// As long as Neon does not support tablespace (because of lack of access to local file system), + /// we can assume that only some predefined namespace OIDs are used which can fit in u16 + pub fn to_i128(&self) -> i128 { + assert!(self.field2 < 0xFFFF || self.field2 == 0xFFFFFFFF || self.field2 == 0x22222222); + (((self.field1 & 0xf) as i128) << 120) + | (((self.field2 & 0xFFFF) as i128) << 104) + | ((self.field3 as i128) << 72) + | ((self.field4 as i128) << 40) + | ((self.field5 as i128) << 32) + | self.field6 as i128 + } + + pub const fn from_i128(x: i128) -> Self { + Key { + field1: ((x >> 120) & 0xf) as u8, + field2: ((x >> 104) & 0xFFFF) as u32, + field3: (x >> 72) as u32, + field4: (x >> 40) as u32, + field5: (x >> 32) as u8, + field6: x as u32, + } + } + + pub fn next(&self) -> Key { + self.add(1) + } + + pub fn add(&self, x: u32) -> Key { + let mut key = *self; + + let r = key.field6.overflowing_add(x); + key.field6 = r.0; + if r.1 { + let r = key.field5.overflowing_add(1); + key.field5 = r.0; + if r.1 { + let r = key.field4.overflowing_add(1); + key.field4 = r.0; + if r.1 { + let r = key.field3.overflowing_add(1); + key.field3 = r.0; + if r.1 { + let r = key.field2.overflowing_add(1); + key.field2 = r.0; + if r.1 { + let r = key.field1.overflowing_add(1); + key.field1 = r.0; + assert!(!r.1); + } + } + } + } + } + key + } + + pub fn from_slice(b: &[u8]) -> Self { + Key { + field1: b[0], + field2: u32::from_be_bytes(b[1..5].try_into().unwrap()), + field3: u32::from_be_bytes(b[5..9].try_into().unwrap()), + field4: u32::from_be_bytes(b[9..13].try_into().unwrap()), + field5: b[13], + field6: u32::from_be_bytes(b[14..18].try_into().unwrap()), + } + } + + pub fn write_to_byte_slice(&self, buf: &mut [u8]) { + buf[0] = self.field1; + BE::write_u32(&mut buf[1..5], self.field2); + BE::write_u32(&mut buf[5..9], self.field3); + BE::write_u32(&mut buf[9..13], self.field4); + buf[13] = self.field5; + BE::write_u32(&mut buf[14..18], self.field6); + } +} + +impl fmt::Display for Key { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:02X}{:08X}{:08X}{:08X}{:02X}{:08X}", + self.field1, self.field2, self.field3, self.field4, self.field5, self.field6 + ) + } +} + +impl Key { + pub const MIN: Key = Key { + field1: u8::MIN, + field2: u32::MIN, + field3: u32::MIN, + field4: u32::MIN, + field5: u8::MIN, + field6: u32::MIN, + }; + pub const MAX: Key = Key { + field1: u8::MAX, + field2: u32::MAX, + field3: u32::MAX, + field4: u32::MAX, + field5: u8::MAX, + field6: u32::MAX, + }; + + pub fn from_hex(s: &str) -> Result { + if s.len() != 36 { + bail!("parse error"); + } + Ok(Key { + field1: u8::from_str_radix(&s[0..2], 16)?, + field2: u32::from_str_radix(&s[2..10], 16)?, + field3: u32::from_str_radix(&s[10..18], 16)?, + field4: u32::from_str_radix(&s[18..26], 16)?, + field5: u8::from_str_radix(&s[26..28], 16)?, + field6: u32::from_str_radix(&s[28..36], 16)?, + }) + } +} diff --git a/libs/pageserver_api/src/lib.rs b/libs/pageserver_api/src/lib.rs index e49f7d00c1..511c5ed208 100644 --- a/libs/pageserver_api/src/lib.rs +++ b/libs/pageserver_api/src/lib.rs @@ -4,8 +4,10 @@ use const_format::formatcp; /// Public API types pub mod control_api; +pub mod key; pub mod models; pub mod reltag; +pub mod shard; pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000; pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}"); diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index cb99dc0a55..71e32e479f 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -16,7 +16,7 @@ use utils::{ lsn::Lsn, }; -use crate::reltag::RelTag; +use crate::{reltag::RelTag, shard::TenantShardId}; use anyhow::bail; use bytes::{BufMut, Bytes, BytesMut}; @@ -187,7 +187,7 @@ pub struct TimelineCreateRequest { #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantCreateRequest { - pub new_tenant_id: TenantId, + pub new_tenant_id: TenantShardId, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub generation: Option, diff --git a/libs/pageserver_api/src/shard.rs b/libs/pageserver_api/src/shard.rs new file mode 100644 index 0000000000..32a834a26a --- /dev/null +++ b/libs/pageserver_api/src/shard.rs @@ -0,0 +1,321 @@ +use std::{ops::RangeInclusive, str::FromStr}; + +use hex::FromHex; +use serde::{Deserialize, Serialize}; +use utils::id::TenantId; + +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug)] +pub struct ShardNumber(pub u8); + +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug)] +pub struct ShardCount(pub u8); + +impl ShardCount { + pub const MAX: Self = Self(u8::MAX); +} + +impl ShardNumber { + pub const MAX: Self = Self(u8::MAX); +} + +/// TenantShardId identify the units of work for the Pageserver. +/// +/// These are written as `-`, for example: +/// +/// # The second shard in a two-shard tenant +/// 072f1291a5310026820b2fe4b2968934-0102 +/// +/// Historically, tenants could not have multiple shards, and were identified +/// by TenantId. To support this, TenantShardId has a special legacy +/// mode where `shard_count` is equal to zero: this represents a single-sharded +/// tenant which should be written as a TenantId with no suffix. +/// +/// The human-readable encoding of TenantShardId, such as used in API URLs, +/// is both forward and backward compatible: a legacy TenantId can be +/// decoded as a TenantShardId, and when re-encoded it will be parseable +/// as a TenantId. +/// +/// Note that the binary encoding is _not_ backward compatible, because +/// at the time sharding is introduced, there are no existing binary structures +/// containing TenantId that we need to handle. +#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy)] +pub struct TenantShardId { + pub tenant_id: TenantId, + pub shard_number: ShardNumber, + pub shard_count: ShardCount, +} + +impl TenantShardId { + pub fn unsharded(tenant_id: TenantId) -> Self { + Self { + tenant_id, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + } + } + + /// The range of all TenantShardId that belong to a particular TenantId. This is useful when + /// you have a BTreeMap of TenantShardId, and are querying by TenantId. + pub fn tenant_range(tenant_id: TenantId) -> RangeInclusive { + RangeInclusive::new( + Self { + tenant_id, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + }, + Self { + tenant_id, + shard_number: ShardNumber::MAX, + shard_count: ShardCount::MAX, + }, + ) + } + + pub fn shard_slug(&self) -> String { + format!("{:02x}{:02x}", self.shard_number.0, self.shard_count.0) + } +} + +impl std::fmt::Display for TenantShardId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.shard_count != ShardCount(0) { + write!( + f, + "{}-{:02x}{:02x}", + self.tenant_id, self.shard_number.0, self.shard_count.0 + ) + } else { + // Legacy case (shard_count == 0) -- format as just the tenant id. Note that this + // is distinct from the normal single shard case (shard count == 1). + self.tenant_id.fmt(f) + } + } +} + +impl std::fmt::Debug for TenantShardId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Debug is the same as Display: the compact hex representation + write!(f, "{}", self) + } +} + +impl std::str::FromStr for TenantShardId { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + // Expect format: 16 byte TenantId, '-', 1 byte shard number, 1 byte shard count + if s.len() == 32 { + // Legacy case: no shard specified + Ok(Self { + tenant_id: TenantId::from_str(s)?, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + }) + } else if s.len() == 37 { + let bytes = s.as_bytes(); + let tenant_id = TenantId::from_hex(&bytes[0..32])?; + let mut shard_parts: [u8; 2] = [0u8; 2]; + hex::decode_to_slice(&bytes[33..37], &mut shard_parts)?; + Ok(Self { + tenant_id, + shard_number: ShardNumber(shard_parts[0]), + shard_count: ShardCount(shard_parts[1]), + }) + } else { + Err(hex::FromHexError::InvalidStringLength) + } + } +} + +impl From<[u8; 18]> for TenantShardId { + fn from(b: [u8; 18]) -> Self { + let tenant_id_bytes: [u8; 16] = b[0..16].try_into().unwrap(); + + Self { + tenant_id: TenantId::from(tenant_id_bytes), + shard_number: ShardNumber(b[16]), + shard_count: ShardCount(b[17]), + } + } +} + +impl Serialize for TenantShardId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + let mut packed: [u8; 18] = [0; 18]; + packed[0..16].clone_from_slice(&self.tenant_id.as_arr()); + packed[16] = self.shard_number.0; + packed[17] = self.shard_count.0; + + packed.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for TenantShardId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> serde::de::Visitor<'de> for IdVisitor { + type Value = TenantShardId; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 18])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 18] = Deserialize::deserialize(s)?; + Ok(TenantShardId::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + TenantShardId::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 18, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use bincode; + use utils::{id::TenantId, Hex}; + + use super::*; + + const EXAMPLE_TENANT_ID: &str = "1f359dd625e519a1a4e8d7509690f6fc"; + + #[test] + fn tenant_shard_id_string() -> Result<(), hex::FromHexError> { + let example = TenantShardId { + tenant_id: TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(), + shard_count: ShardCount(10), + shard_number: ShardNumber(7), + }; + + let encoded = format!("{example}"); + + let expected = format!("{EXAMPLE_TENANT_ID}-070a"); + assert_eq!(&encoded, &expected); + + let decoded = TenantShardId::from_str(&encoded)?; + + assert_eq!(example, decoded); + + Ok(()) + } + + #[test] + fn tenant_shard_id_binary() -> Result<(), hex::FromHexError> { + let example = TenantShardId { + tenant_id: TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(), + shard_count: ShardCount(10), + shard_number: ShardNumber(7), + }; + + let encoded = bincode::serialize(&example).unwrap(); + let expected: [u8; 18] = [ + 0x1f, 0x35, 0x9d, 0xd6, 0x25, 0xe5, 0x19, 0xa1, 0xa4, 0xe8, 0xd7, 0x50, 0x96, 0x90, + 0xf6, 0xfc, 0x07, 0x0a, + ]; + assert_eq!(Hex(&encoded), Hex(&expected)); + + let decoded = bincode::deserialize(&encoded).unwrap(); + + assert_eq!(example, decoded); + + Ok(()) + } + + #[test] + fn tenant_shard_id_backward_compat() -> Result<(), hex::FromHexError> { + // Test that TenantShardId can decode a TenantId in human + // readable form + let example = TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(); + let encoded = format!("{example}"); + + assert_eq!(&encoded, EXAMPLE_TENANT_ID); + + let decoded = TenantShardId::from_str(&encoded)?; + + assert_eq!(example, decoded.tenant_id); + assert_eq!(decoded.shard_count, ShardCount(0)); + assert_eq!(decoded.shard_number, ShardNumber(0)); + + Ok(()) + } + + #[test] + fn tenant_shard_id_forward_compat() -> Result<(), hex::FromHexError> { + // Test that a legacy TenantShardId encodes into a form that + // can be decoded as TenantId + let example_tenant_id = TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(); + let example = TenantShardId::unsharded(example_tenant_id); + let encoded = format!("{example}"); + + assert_eq!(&encoded, EXAMPLE_TENANT_ID); + + let decoded = TenantId::from_str(&encoded)?; + + assert_eq!(example_tenant_id, decoded); + + Ok(()) + } + + #[test] + fn tenant_shard_id_legacy_binary() -> Result<(), hex::FromHexError> { + // Unlike in human readable encoding, binary encoding does not + // do any special handling of legacy unsharded TenantIds: this test + // is equivalent to the main test for binary encoding, just verifying + // that the same behavior applies when we have used `unsharded()` to + // construct a TenantShardId. + let example = TenantShardId::unsharded(TenantId::from_str(EXAMPLE_TENANT_ID).unwrap()); + let encoded = bincode::serialize(&example).unwrap(); + + let expected: [u8; 18] = [ + 0x1f, 0x35, 0x9d, 0xd6, 0x25, 0xe5, 0x19, 0xa1, 0xa4, 0xe8, 0xd7, 0x50, 0x96, 0x90, + 0xf6, 0xfc, 0x00, 0x00, + ]; + assert_eq!(Hex(&encoded), Hex(&expected)); + + let decoded = bincode::deserialize::(&encoded).unwrap(); + assert_eq!(example, decoded); + + Ok(()) + } +} diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 2915178104..aa2b017471 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -16,6 +16,7 @@ use pageserver_api::models::{ DownloadRemoteLayersTaskSpawnRequest, LocationConfigMode, TenantAttachRequest, TenantLoadRequest, TenantLocationConfigRequest, }; +use pageserver_api::shard::TenantShardId; use remote_storage::GenericRemoteStorage; use tenant_size_model::{SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; @@ -419,9 +420,9 @@ async fn timeline_create_handler( mut request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; let request_data: TimelineCreateRequest = json_request(&mut request).await?; - check_permission(&request, Some(tenant_id))?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let new_timeline_id = request_data.new_timeline_id; @@ -430,7 +431,7 @@ async fn timeline_create_handler( let state = get_state(&request); async { - let tenant = mgr::get_tenant(tenant_id, true)?; + let tenant = state.tenant_manager.get_attached_tenant_shard(tenant_shard_id, true)?; match tenant.create_timeline( new_timeline_id, request_data.ancestor_timeline_id.map(TimelineId::from), @@ -464,7 +465,10 @@ async fn timeline_create_handler( Err(tenant::CreateTimelineError::Other(err)) => Err(ApiError::InternalServerError(err)), } } - .instrument(info_span!("timeline_create", %tenant_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version)) + .instrument(info_span!("timeline_create", + tenant_id = %tenant_shard_id.tenant_id, + shard = %tenant_shard_id.shard_slug(), + timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version)) .await } @@ -660,14 +664,15 @@ async fn timeline_delete_handler( request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; - check_permission(&request, Some(tenant_id))?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); + let state = get_state(&request); - mgr::delete_timeline(tenant_id, timeline_id, &ctx) - .instrument(info_span!("timeline_delete", %tenant_id, %timeline_id)) + state.tenant_manager.delete_timeline(tenant_shard_id, timeline_id, &ctx) + .instrument(info_span!("timeline_delete", tenant_id=%tenant_shard_id.tenant_id, shard=%tenant_shard_id.shard_slug(), %timeline_id)) .await?; json_response(StatusCode::ACCEPTED, ()) @@ -681,11 +686,14 @@ async fn tenant_detach_handler( check_permission(&request, Some(tenant_id))?; let detach_ignored: Option = parse_query_param(&request, "detach_ignored")?; + // This is a legacy API (`/location_conf` is the replacement). It only supports unsharded tenants + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + let state = get_state(&request); let conf = state.conf; mgr::detach_tenant( conf, - tenant_id, + tenant_shard_id, detach_ignored.unwrap_or(false), &state.deletion_queue_client, ) @@ -802,13 +810,16 @@ async fn tenant_delete_handler( _cancel: CancellationToken, ) -> Result, ApiError> { // TODO openapi spec - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - check_permission(&request, Some(tenant_id))?; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let state = get_state(&request); - mgr::delete_tenant(state.conf, state.remote_storage.clone(), tenant_id) - .instrument(info_span!("tenant_delete_handler", %tenant_id)) + mgr::delete_tenant(state.conf, state.remote_storage.clone(), tenant_shard_id) + .instrument(info_span!("tenant_delete_handler", + tenant_id = %tenant_shard_id.tenant_id, + shard = tenant_shard_id.shard_slug() + )) .await?; json_response(StatusCode::ACCEPTED, ()) @@ -1138,9 +1149,10 @@ async fn put_tenant_location_config_handler( mut request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; + let request_data: TenantLocationConfigRequest = json_request(&mut request).await?; - let tenant_id = request_data.tenant_id; - check_permission(&request, Some(tenant_id))?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); let state = get_state(&request); @@ -1149,9 +1161,13 @@ async fn put_tenant_location_config_handler( // The `Detached` state is special, it doesn't upsert a tenant, it removes // its local disk content and drops it from memory. if let LocationConfigMode::Detached = request_data.config.mode { - if let Err(e) = mgr::detach_tenant(conf, tenant_id, true, &state.deletion_queue_client) - .instrument(info_span!("tenant_detach", %tenant_id)) - .await + if let Err(e) = + mgr::detach_tenant(conf, tenant_shard_id, true, &state.deletion_queue_client) + .instrument(info_span!("tenant_detach", + tenant_id = %tenant_shard_id.tenant_id, + shard = tenant_shard_id.shard_slug() + )) + .await { match e { TenantStateError::SlotError(TenantSlotError::NotFound(_)) => { @@ -1168,7 +1184,7 @@ async fn put_tenant_location_config_handler( state .tenant_manager - .upsert_location(tenant_id, location_conf, &ctx) + .upsert_location(tenant_shard_id, location_conf, &ctx) .await // TODO: badrequest assumes the caller was asking for something unreasonable, but in // principle we might have hit something like concurrent API calls to the same tenant, @@ -1752,7 +1768,7 @@ pub fn make_router( .get("/v1/tenant", |r| api_handler(r, tenant_list_handler)) .post("/v1/tenant", |r| api_handler(r, tenant_create_handler)) .get("/v1/tenant/:tenant_id", |r| api_handler(r, tenant_status)) - .delete("/v1/tenant/:tenant_id", |r| { + .delete("/v1/tenant/:tenant_shard_id", |r| { api_handler(r, tenant_delete_handler) }) .get("/v1/tenant/:tenant_id/synthetic_size", |r| { @@ -1764,13 +1780,13 @@ pub fn make_router( .get("/v1/tenant/:tenant_id/config", |r| { api_handler(r, get_tenant_config_handler) }) - .put("/v1/tenant/:tenant_id/location_config", |r| { + .put("/v1/tenant/:tenant_shard_id/location_config", |r| { api_handler(r, put_tenant_location_config_handler) }) .get("/v1/tenant/:tenant_id/timeline", |r| { api_handler(r, timeline_list_handler) }) - .post("/v1/tenant/:tenant_id/timeline", |r| { + .post("/v1/tenant/:tenant_shard_id/timeline", |r| { api_handler(r, timeline_create_handler) }) .post("/v1/tenant/:tenant_id/attach", |r| { @@ -1814,7 +1830,7 @@ pub fn make_router( "/v1/tenant/:tenant_id/timeline/:timeline_id/download_remote_layers", |r| api_handler(r, timeline_download_remote_layers_handler_get), ) - .delete("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| { + .delete("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| { api_handler(r, timeline_delete_handler) }) .get("/v1/tenant/:tenant_id/timeline/:timeline_id/layer", |r| { diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index dc75aeeb50..24f47df92e 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -1,106 +1,11 @@ use crate::walrecord::NeonWalRecord; -use anyhow::{bail, Result}; -use byteorder::{ByteOrder, BE}; +use anyhow::Result; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use std::fmt; use std::ops::{AddAssign, Range}; use std::time::Duration; -/// Key used in the Repository kv-store. -/// -/// The Repository treats this as an opaque struct, but see the code in pgdatadir_mapping.rs -/// for what we actually store in these fields. -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] -pub struct Key { - pub field1: u8, - pub field2: u32, - pub field3: u32, - pub field4: u32, - pub field5: u8, - pub field6: u32, -} - -pub const KEY_SIZE: usize = 18; - -impl Key { - /// 'field2' is used to store tablespaceid for relations and small enum numbers for other relish. - /// As long as Neon does not support tablespace (because of lack of access to local file system), - /// we can assume that only some predefined namespace OIDs are used which can fit in u16 - pub fn to_i128(&self) -> i128 { - assert!(self.field2 < 0xFFFF || self.field2 == 0xFFFFFFFF || self.field2 == 0x22222222); - (((self.field1 & 0xf) as i128) << 120) - | (((self.field2 & 0xFFFF) as i128) << 104) - | ((self.field3 as i128) << 72) - | ((self.field4 as i128) << 40) - | ((self.field5 as i128) << 32) - | self.field6 as i128 - } - - pub const fn from_i128(x: i128) -> Self { - Key { - field1: ((x >> 120) & 0xf) as u8, - field2: ((x >> 104) & 0xFFFF) as u32, - field3: (x >> 72) as u32, - field4: (x >> 40) as u32, - field5: (x >> 32) as u8, - field6: x as u32, - } - } - - pub fn next(&self) -> Key { - self.add(1) - } - - pub fn add(&self, x: u32) -> Key { - let mut key = *self; - - let r = key.field6.overflowing_add(x); - key.field6 = r.0; - if r.1 { - let r = key.field5.overflowing_add(1); - key.field5 = r.0; - if r.1 { - let r = key.field4.overflowing_add(1); - key.field4 = r.0; - if r.1 { - let r = key.field3.overflowing_add(1); - key.field3 = r.0; - if r.1 { - let r = key.field2.overflowing_add(1); - key.field2 = r.0; - if r.1 { - let r = key.field1.overflowing_add(1); - key.field1 = r.0; - assert!(!r.1); - } - } - } - } - } - key - } - - pub fn from_slice(b: &[u8]) -> Self { - Key { - field1: b[0], - field2: u32::from_be_bytes(b[1..5].try_into().unwrap()), - field3: u32::from_be_bytes(b[5..9].try_into().unwrap()), - field4: u32::from_be_bytes(b[9..13].try_into().unwrap()), - field5: b[13], - field6: u32::from_be_bytes(b[14..18].try_into().unwrap()), - } - } - - pub fn write_to_byte_slice(&self, buf: &mut [u8]) { - buf[0] = self.field1; - BE::write_u32(&mut buf[1..5], self.field2); - BE::write_u32(&mut buf[5..9], self.field3); - BE::write_u32(&mut buf[9..13], self.field4); - buf[13] = self.field5; - BE::write_u32(&mut buf[14..18], self.field6); - } -} +pub use pageserver_api::key::{Key, KEY_SIZE}; pub fn key_range_size(key_range: &Range) -> u32 { let start = key_range.start; @@ -129,49 +34,6 @@ pub fn singleton_range(key: Key) -> Range { key..key.next() } -impl fmt::Display for Key { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{:02X}{:08X}{:08X}{:08X}{:02X}{:08X}", - self.field1, self.field2, self.field3, self.field4, self.field5, self.field6 - ) - } -} - -impl Key { - pub const MIN: Key = Key { - field1: u8::MIN, - field2: u32::MIN, - field3: u32::MIN, - field4: u32::MIN, - field5: u8::MIN, - field6: u32::MIN, - }; - pub const MAX: Key = Key { - field1: u8::MAX, - field2: u32::MAX, - field3: u32::MAX, - field4: u32::MAX, - field5: u8::MAX, - field6: u32::MAX, - }; - - pub fn from_hex(s: &str) -> Result { - if s.len() != 36 { - bail!("parse error"); - } - Ok(Key { - field1: u8::from_str_radix(&s[0..2], 16)?, - field2: u32::from_str_radix(&s[2..10], 16)?, - field3: u32::from_str_radix(&s[10..18], 16)?, - field4: u32::from_str_radix(&s[18..26], 16)?, - field5: u8::from_str_radix(&s[26..28], 16)?, - field6: u32::from_str_radix(&s[28..36], 16)?, - }) - } -} - /// A 'value' stored for a one Key. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(test, derive(PartialEq))] diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 4363dab375..a766cca0c5 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -2,9 +2,10 @@ //! page server. use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; +use pageserver_api::shard::TenantShardId; use rand::{distributions::Alphanumeric, Rng}; use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::ops::Deref; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -30,6 +31,7 @@ use crate::metrics::TENANT_MANAGER as METRICS; use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; use crate::tenant::delete::DeleteTenantFlow; +use crate::tenant::span::debug_assert_current_span_has_tenant_id; use crate::tenant::{create_tenant_files, AttachedTenantConf, SpawnMode, Tenant, TenantState}; use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME, TEMP_FILE_SUFFIX}; @@ -87,10 +89,37 @@ pub(crate) enum TenantsMap { Initializing, /// [`init_tenant_mgr`] is done, all on-disk tenants have been loaded. /// New tenants can be added using [`tenant_map_acquire_slot`]. - Open(HashMap), + Open(BTreeMap), /// The pageserver has entered shutdown mode via [`shutdown_all_tenants`]. /// Existing tenants are still accessible, but no new tenants can be created. - ShuttingDown(HashMap), + ShuttingDown(BTreeMap), +} + +/// Helper for mapping shard-unaware functions to a sharding-aware map +/// TODO(sharding): all users of this must be made shard-aware. +fn exactly_one_or_none<'a>( + map: &'a BTreeMap, + tenant_id: &TenantId, +) -> Option<(&'a TenantShardId, &'a TenantSlot)> { + let mut slots = map.range(TenantShardId::tenant_range(*tenant_id)); + + // Retrieve the first two slots in the range: if both are populated, we must panic because the caller + // needs a shard-naive view of the world in which only one slot can exist for a TenantId at a time. + let slot_a = slots.next(); + let slot_b = slots.next(); + match (slot_a, slot_b) { + (None, None) => None, + (Some(slot), None) => { + // Exactly one matching slot + Some(slot) + } + (Some(_slot_a), Some(_slot_b)) => { + // Multiple shards for this tenant: cannot handle this yet. + // TODO(sharding): callers of get() should be shard-aware. + todo!("Attaching multiple shards in teh same tenant to the same pageserver") + } + (None, Some(_)) => unreachable!(), + } } impl TenantsMap { @@ -101,7 +130,8 @@ impl TenantsMap { match self { TenantsMap::Initializing => None, TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => { - m.get(tenant_id).and_then(TenantSlot::get_attached) + // TODO(sharding): callers of get() should be shard-aware. + exactly_one_or_none(m, tenant_id).and_then(|(_, slot)| slot.get_attached()) } } } @@ -109,7 +139,10 @@ impl TenantsMap { pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option { match self { TenantsMap::Initializing => None, - TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.remove(tenant_id), + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => { + let key = exactly_one_or_none(m, tenant_id).map(|(k, _)| *k); + key.and_then(|key| m.remove(&key)) + } } } @@ -383,7 +416,7 @@ pub async fn init_tenant_mgr( init_order: InitializationOrder, cancel: CancellationToken, ) -> anyhow::Result { - let mut tenants = HashMap::new(); + let mut tenants = BTreeMap::new(); let ctx = RequestContext::todo_child(TaskKind::Startup, DownloadBehavior::Warn); @@ -404,7 +437,7 @@ pub async fn init_tenant_mgr( warn!(%tenant_id, "Marking tenant broken, failed to {e:#}"); tenants.insert( - tenant_id, + TenantShardId::unsharded(tenant_id), TenantSlot::Attached(Tenant::create_broken_tenant( conf, tenant_id, @@ -427,7 +460,7 @@ pub async fn init_tenant_mgr( // tenants, because they do no remote writes and hence require no // generation number info!(%tenant_id, "Loaded tenant in secondary mode"); - tenants.insert(tenant_id, TenantSlot::Secondary); + tenants.insert(TenantShardId::unsharded(tenant_id), TenantSlot::Secondary); } LocationMode::Attached(_) => { // TODO: augment re-attach API to enable the control plane to @@ -470,7 +503,10 @@ pub async fn init_tenant_mgr( &ctx, ) { Ok(tenant) => { - tenants.insert(tenant.tenant_id(), TenantSlot::Attached(tenant)); + tenants.insert( + TenantShardId::unsharded(tenant.tenant_id()), + TenantSlot::Attached(tenant), + ); } Err(e) => { error!(%tenant_id, "Failed to start tenant: {e:#}"); @@ -573,19 +609,19 @@ async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { let mut m = tenants.write().unwrap(); match &mut *m { TenantsMap::Initializing => { - *m = TenantsMap::ShuttingDown(HashMap::default()); + *m = TenantsMap::ShuttingDown(BTreeMap::default()); info!("tenants map is empty"); return; } TenantsMap::Open(tenants) => { - let mut shutdown_state = HashMap::new(); + let mut shutdown_state = BTreeMap::new(); let mut total_in_progress = 0; let mut total_attached = 0; - for (tenant_id, v) in tenants.drain() { + for (tenant_shard_id, v) in std::mem::take(tenants).into_iter() { match v { TenantSlot::Attached(t) => { - shutdown_state.insert(tenant_id, TenantSlot::Attached(t.clone())); + shutdown_state.insert(tenant_shard_id, TenantSlot::Attached(t.clone())); join_set.spawn( async move { let freeze_and_flush = true; @@ -604,13 +640,13 @@ async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { // going to log too many lines debug!("tenant successfully stopped"); } - .instrument(info_span!("shutdown", %tenant_id)), + .instrument(info_span!("shutdown", tenant_id=%tenant_shard_id.tenant_id, shard=%tenant_shard_id.shard_slug())), ); total_attached += 1; } TenantSlot::Secondary => { - shutdown_state.insert(tenant_id, TenantSlot::Secondary); + shutdown_state.insert(tenant_shard_id, TenantSlot::Secondary); } TenantSlot::InProgress(notify) => { // InProgress tenants are not visible in TenantsMap::ShuttingDown: we will @@ -690,19 +726,22 @@ async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { pub(crate) async fn create_tenant( conf: &'static PageServerConf, tenant_conf: TenantConfOpt, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, generation: Generation, resources: TenantSharedResources, ctx: &RequestContext, ) -> Result, TenantMapInsertError> { let location_conf = LocationConf::attached_single(tenant_conf, generation); - let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; - let tenant_path = super::create_tenant_files(conf, &location_conf, &tenant_id).await?; + let slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; + // TODO(sharding): make local paths shard-aware + let tenant_path = + super::create_tenant_files(conf, &location_conf, &tenant_shard_id.tenant_id).await?; let created_tenant = tenant_spawn( conf, - tenant_id, + tenant_shard_id.tenant_id, &tenant_path, resources, AttachedTenantConf::try_from(location_conf)?, @@ -715,11 +754,7 @@ pub(crate) async fn create_tenant( // See https://github.com/neondatabase/neon/issues/4233 let created_tenant_id = created_tenant.tenant_id(); - if tenant_id != created_tenant_id { - return Err(TenantMapInsertError::Other(anyhow::anyhow!( - "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {created_tenant_id})", - ))); - } + debug_assert_eq!(created_tenant_id, tenant_shard_id.tenant_id); slot_guard.upsert(TenantSlot::Attached(created_tenant.clone()))?; @@ -755,21 +790,70 @@ pub(crate) async fn set_new_tenant_config( } impl TenantManager { - #[instrument(skip_all, fields(%tenant_id))] + /// Gets the attached tenant from the in-memory data, erroring if it's absent, in secondary mode, or is not fitting to the query. + /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. + /// + /// This method is cancel-safe. + pub(crate) fn get_attached_tenant_shard( + &self, + tenant_shard_id: TenantShardId, + active_only: bool, + ) -> Result, GetTenantError> { + let locked = self.tenants.read().unwrap(); + + let peek_slot = tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read)?; + + match peek_slot { + Some(TenantSlot::Attached(tenant)) => match tenant.current_state() { + TenantState::Broken { + reason, + backtrace: _, + } if active_only => Err(GetTenantError::Broken(reason)), + TenantState::Active => Ok(Arc::clone(tenant)), + _ => { + if active_only { + Err(GetTenantError::NotActive(tenant_shard_id.tenant_id)) + } else { + Ok(Arc::clone(tenant)) + } + } + }, + Some(TenantSlot::InProgress(_)) => { + Err(GetTenantError::NotActive(tenant_shard_id.tenant_id)) + } + None | Some(TenantSlot::Secondary) => { + Err(GetTenantError::NotFound(tenant_shard_id.tenant_id)) + } + } + } + + pub(crate) async fn delete_timeline( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + _ctx: &RequestContext, + ) -> Result<(), DeleteTimelineError> { + let tenant = self.get_attached_tenant_shard(tenant_shard_id, true)?; + DeleteTimelineFlow::run(&tenant, timeline_id, false).await?; + Ok(()) + } + pub(crate) async fn upsert_location( &self, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, new_location_config: LocationConf, ctx: &RequestContext, ) -> Result<(), anyhow::Error> { - info!("configuring tenant location {tenant_id} to state {new_location_config:?}"); + debug_assert_current_span_has_tenant_id(); + info!("configuring tenant location to state {new_location_config:?}"); // Special case fast-path for updates to Tenant: if our upsert is only updating configuration, // then we do not need to set the slot to InProgress, we can just call into the // existng tenant. { let locked = self.tenants.read().unwrap(); - let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Write)?; + let peek_slot = + tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Write)?; match (&new_location_config.mode, peek_slot) { (LocationMode::Attached(attach_conf), Some(TenantSlot::Attached(tenant))) => { if attach_conf.generation == tenant.generation { @@ -800,7 +884,7 @@ impl TenantManager { // the tenant is inaccessible to the outside world while we are doing this, but that is sensible: // the state is ill-defined while we're in transition. Transitions are async, but fast: we do // not do significant I/O, and shutdowns should be prompt via cancellation tokens. - let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::Any)?; + let mut slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::Any)?; if let Some(TenantSlot::Attached(tenant)) = slot_guard.get_old_value() { // The case where we keep a Tenant alive was covered above in the special case @@ -831,25 +915,31 @@ impl TenantManager { slot_guard.drop_old_value().expect("We just shut it down"); } - let tenant_path = self.conf.tenant_path(&tenant_id); + // TODO(sharding): make local paths sharding-aware + let tenant_path = self.conf.tenant_path(&tenant_shard_id.tenant_id); let new_slot = match &new_location_config.mode { LocationMode::Secondary(_) => { - let tenant_path = self.conf.tenant_path(&tenant_id); // Directory doesn't need to be fsync'd because if we crash it can // safely be recreated next time this tenant location is configured. unsafe_create_dir_all(&tenant_path) .await .with_context(|| format!("Creating {tenant_path}"))?; - Tenant::persist_tenant_config(self.conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; + // TODO(sharding): make local paths sharding-aware + Tenant::persist_tenant_config( + self.conf, + &tenant_shard_id.tenant_id, + &new_location_config, + ) + .await + .map_err(SetNewTenantConfigError::Persist)?; TenantSlot::Secondary } LocationMode::Attached(_attach_config) => { - let timelines_path = self.conf.timelines_path(&tenant_id); + // TODO(sharding): make local paths sharding-aware + let timelines_path = self.conf.timelines_path(&tenant_shard_id.tenant_id); // Directory doesn't need to be fsync'd because we do not depend on // it to exist after crashes: it may be recreated when tenant is @@ -858,13 +948,19 @@ impl TenantManager { .await .with_context(|| format!("Creating {timelines_path}"))?; - Tenant::persist_tenant_config(self.conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; + // TODO(sharding): make local paths sharding-aware + Tenant::persist_tenant_config( + self.conf, + &tenant_shard_id.tenant_id, + &new_location_config, + ) + .await + .map_err(SetNewTenantConfigError::Persist)?; + // TODO(sharding): make spawn sharding-aware let tenant = tenant_spawn( self.conf, - tenant_id, + tenant_shard_id.tenant_id, &tenant_path, self.resources.clone(), AttachedTenantConf::try_from(new_location_config)?, @@ -910,7 +1006,11 @@ pub(crate) fn get_tenant( active_only: bool, ) -> Result, GetTenantError> { let locked = TENANTS.read().unwrap(); - let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read)?; + + // TODO(sharding): make all callers of get_tenant shard-aware + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + let peek_slot = tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read)?; match peek_slot { Some(TenantSlot::Attached(tenant)) => match tenant.current_state() { @@ -970,12 +1070,16 @@ pub(crate) async fn get_active_tenant_with_timeout( Tenant(Arc), } + // TODO(sharding): make page service interface sharding-aware (page service should apply ShardIdentity to the key + // to decide which shard services the request) + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + let wait_start = Instant::now(); let deadline = wait_start + timeout; let wait_for = { let locked = TENANTS.read().unwrap(); - let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read) + let peek_slot = tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read) .map_err(GetTenantError::MapState)?; match peek_slot { Some(TenantSlot::Attached(tenant)) => { @@ -1019,8 +1123,9 @@ pub(crate) async fn get_active_tenant_with_timeout( })?; { let locked = TENANTS.read().unwrap(); - let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read) - .map_err(GetTenantError::MapState)?; + let peek_slot = + tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read) + .map_err(GetTenantError::MapState)?; match peek_slot { Some(TenantSlot::Attached(tenant)) => tenant.clone(), _ => { @@ -1062,7 +1167,7 @@ pub(crate) async fn get_active_tenant_with_timeout( pub(crate) async fn delete_tenant( conf: &'static PageServerConf, remote_storage: Option, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, ) -> Result<(), DeleteTenantError> { // We acquire a SlotGuard during this function to protect against concurrent // changes while the ::prepare phase of DeleteTenantFlow executes, but then @@ -1075,7 +1180,9 @@ pub(crate) async fn delete_tenant( // // See https://github.com/neondatabase/neon/issues/5080 - let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustExist)?; + // TODO(sharding): make delete API sharding-aware + let mut slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustExist)?; // unwrap is safe because we used MustExist mode when acquiring let tenant = match slot_guard.get_old_value().as_ref().unwrap() { @@ -1102,16 +1209,6 @@ pub(crate) enum DeleteTimelineError { Timeline(#[from] crate::tenant::DeleteTimelineError), } -pub(crate) async fn delete_timeline( - tenant_id: TenantId, - timeline_id: TimelineId, - _ctx: &RequestContext, -) -> Result<(), DeleteTimelineError> { - let tenant = get_tenant(tenant_id, true)?; - DeleteTimelineFlow::run(&tenant, timeline_id, false).await?; - Ok(()) -} - #[derive(Debug, thiserror::Error)] pub(crate) enum TenantStateError { #[error("Tenant {0} is stopping")] @@ -1126,14 +1223,14 @@ pub(crate) enum TenantStateError { pub(crate) async fn detach_tenant( conf: &'static PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, ) -> Result<(), TenantStateError> { let tmp_path = detach_tenant0( conf, &TENANTS, - tenant_id, + tenant_shard_id, detach_ignored, deletion_queue_client, ) @@ -1160,19 +1257,24 @@ pub(crate) async fn detach_tenant( async fn detach_tenant0( conf: &'static PageServerConf, tenants: &std::sync::RwLock, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, ) -> Result { - let tenant_dir_rename_operation = |tenant_id_to_clean| async move { - let local_tenant_directory = conf.tenant_path(&tenant_id_to_clean); + let tenant_dir_rename_operation = |tenant_id_to_clean: TenantShardId| async move { + // TODO(sharding): make local path helpers shard-aware + let local_tenant_directory = conf.tenant_path(&tenant_id_to_clean.tenant_id); safe_rename_tenant_dir(&local_tenant_directory) .await .with_context(|| format!("local tenant directory {local_tenant_directory:?} rename")) }; - let removal_result = - remove_tenant_from_memory(tenants, tenant_id, tenant_dir_rename_operation(tenant_id)).await; + let removal_result = remove_tenant_from_memory( + tenants, + tenant_shard_id, + tenant_dir_rename_operation(tenant_shard_id), + ) + .await; // Flush pending deletions, so that they have a good chance of passing validation // before this tenant is potentially re-attached elsewhere. @@ -1186,12 +1288,15 @@ async fn detach_tenant0( Err(TenantStateError::SlotError(TenantSlotError::NotFound(_))) ) { - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); + // TODO(sharding): make local paths sharding-aware + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id.tenant_id); if tenant_ignore_mark.exists() { info!("Detaching an ignored tenant"); - let tmp_path = tenant_dir_rename_operation(tenant_id) + let tmp_path = tenant_dir_rename_operation(tenant_shard_id) .await - .with_context(|| format!("Ignored tenant {tenant_id} local directory rename"))?; + .with_context(|| { + format!("Ignored tenant {tenant_shard_id} local directory rename") + })?; return Ok(tmp_path); } } @@ -1208,7 +1313,11 @@ pub(crate) async fn load_tenant( deletion_queue_client: DeletionQueueClient, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + // This is a legacy API (replaced by `/location_conf`). It does not support sharding + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + let slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; let tenant_path = conf.tenant_path(&tenant_id); let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); @@ -1261,7 +1370,10 @@ async fn ignore_tenant0( tenants: &std::sync::RwLock, tenant_id: TenantId, ) -> Result<(), TenantStateError> { - remove_tenant_from_memory(tenants, tenant_id, async { + // This is a legacy API (replaced by `/location_conf`). It does not support sharding + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + remove_tenant_from_memory(tenants, tenant_shard_id, async { let ignore_mark_file = conf.tenant_ignore_mark_file_path(&tenant_id); fs::File::create(&ignore_mark_file) .await @@ -1270,7 +1382,7 @@ async fn ignore_tenant0( crashsafe::fsync_file_and_parent(&ignore_mark_file) .context("Failed to fsync ignore mark file") }) - .with_context(|| format!("Failed to crate ignore mark for tenant {tenant_id}"))?; + .with_context(|| format!("Failed to crate ignore mark for tenant {tenant_shard_id}"))?; Ok(()) }) .await @@ -1293,10 +1405,12 @@ pub(crate) async fn list_tenants() -> Result, Tenan }; Ok(m.iter() .filter_map(|(id, tenant)| match tenant { - TenantSlot::Attached(tenant) => Some((*id, tenant.current_state())), + TenantSlot::Attached(tenant) => Some((id, tenant.current_state())), TenantSlot::Secondary => None, TenantSlot::InProgress(_) => None, }) + // TODO(sharding): make callers of this function shard-aware + .map(|(k, v)| (k.tenant_id, v)) .collect()) } @@ -1312,7 +1426,11 @@ pub(crate) async fn attach_tenant( resources: TenantSharedResources, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + // This is a legacy API (replaced by `/location_conf`). It does not support sharding + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + let slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; let location_conf = LocationConf::attached_single(tenant_conf, generation); let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; // TODO: tenant directory remains on disk if we bail out from here on. @@ -1359,14 +1477,14 @@ pub(crate) enum TenantMapInsertError { pub enum TenantSlotError { /// When acquiring a slot with the expectation that the tenant already exists. #[error("Tenant {0} not found")] - NotFound(TenantId), + NotFound(TenantShardId), /// When acquiring a slot with the expectation that the tenant does not already exist. #[error("tenant {0} already exists, state: {1:?}")] - AlreadyExists(TenantId, TenantState), + AlreadyExists(TenantShardId, TenantState), #[error("tenant {0} already exists in but is not attached")] - Conflict(TenantId), + Conflict(TenantShardId), // Tried to read a slot that is currently being mutated by another administrative // operation. @@ -1428,7 +1546,7 @@ pub enum TenantMapError { /// `drop_old_value`. It is an error to call this without shutting down /// the conents of `old_value`. pub struct SlotGuard { - tenant_id: TenantId, + tenant_shard_id: TenantShardId, old_value: Option, upserted: bool, @@ -1439,12 +1557,12 @@ pub struct SlotGuard { impl SlotGuard { fn new( - tenant_id: TenantId, + tenant_shard_id: TenantShardId, old_value: Option, completion: utils::completion::Completion, ) -> Self { Self { - tenant_id, + tenant_shard_id, old_value, upserted: false, _completion: completion, @@ -1487,7 +1605,7 @@ impl SlotGuard { TenantsMap::Open(m) => m, }; - let replaced = m.insert(self.tenant_id, new_value); + let replaced = m.insert(self.tenant_shard_id, new_value); self.upserted = true; METRICS.tenant_slots.set(m.len() as u64); @@ -1506,7 +1624,7 @@ impl SlotGuard { None => { METRICS.unexpected_errors.inc(); error!( - tenant_id = %self.tenant_id, + tenant_shard_id = %self.tenant_shard_id, "Missing InProgress marker during tenant upsert, this is a bug." ); Err(TenantSlotUpsertError::InternalError( @@ -1515,7 +1633,7 @@ impl SlotGuard { } Some(slot) => { METRICS.unexpected_errors.inc(); - error!(tenant_id=%self.tenant_id, "Unexpected contents of TenantSlot during upsert, this is a bug. Contents: {:?}", slot); + error!(tenant_shard_id=%self.tenant_shard_id, "Unexpected contents of TenantSlot during upsert, this is a bug. Contents: {:?}", slot); Err(TenantSlotUpsertError::InternalError( "Unexpected contents of TenantSlot".into(), )) @@ -1593,12 +1711,12 @@ impl Drop for SlotGuard { TenantsMap::Open(m) => m, }; - use std::collections::hash_map::Entry; - match m.entry(self.tenant_id) { + use std::collections::btree_map::Entry; + match m.entry(self.tenant_shard_id) { Entry::Occupied(mut entry) => { if !matches!(entry.get(), TenantSlot::InProgress(_)) { METRICS.unexpected_errors.inc(); - error!(tenant_id=%self.tenant_id, "Unexpected contents of TenantSlot during drop, this is a bug. Contents: {:?}", entry.get()); + error!(tenant_shard_id=%self.tenant_shard_id, "Unexpected contents of TenantSlot during drop, this is a bug. Contents: {:?}", entry.get()); } if self.old_value_is_shutdown() { @@ -1610,7 +1728,7 @@ impl Drop for SlotGuard { Entry::Vacant(_) => { METRICS.unexpected_errors.inc(); error!( - tenant_id = %self.tenant_id, + tenant_shard_id = %self.tenant_shard_id, "Missing InProgress marker during SlotGuard drop, this is a bug." ); } @@ -1629,7 +1747,7 @@ enum TenantSlotPeekMode { fn tenant_map_peek_slot<'a>( tenants: &'a std::sync::RwLockReadGuard<'a, TenantsMap>, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, mode: TenantSlotPeekMode, ) -> Result, TenantMapError> { let m = match tenants.deref() { @@ -1643,7 +1761,7 @@ fn tenant_map_peek_slot<'a>( TenantsMap::Open(m) => m, }; - Ok(m.get(tenant_id)) + Ok(m.get(tenant_shard_id)) } enum TenantSlotAcquireMode { @@ -1656,14 +1774,14 @@ enum TenantSlotAcquireMode { } fn tenant_map_acquire_slot( - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, mode: TenantSlotAcquireMode, ) -> Result { - tenant_map_acquire_slot_impl(tenant_id, &TENANTS, mode) + tenant_map_acquire_slot_impl(tenant_shard_id, &TENANTS, mode) } fn tenant_map_acquire_slot_impl( - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, tenants: &std::sync::RwLock, mode: TenantSlotAcquireMode, ) -> Result { @@ -1671,7 +1789,7 @@ fn tenant_map_acquire_slot_impl( METRICS.tenant_slot_writes.inc(); let mut locked = tenants.write().unwrap(); - let span = tracing::info_span!("acquire_slot", %tenant_id); + let span = tracing::info_span!("acquire_slot", tenant_id=%tenant_shard_id.tenant_id, shard=tenant_shard_id.shard_slug()); let _guard = span.enter(); let m = match &mut *locked { @@ -1680,19 +1798,21 @@ fn tenant_map_acquire_slot_impl( TenantsMap::Open(m) => m, }; - use std::collections::hash_map::Entry; - let entry = m.entry(*tenant_id); + use std::collections::btree_map::Entry; + + let entry = m.entry(*tenant_shard_id); + match entry { Entry::Vacant(v) => match mode { MustExist => { tracing::debug!("Vacant && MustExist: return NotFound"); - Err(TenantSlotError::NotFound(*tenant_id)) + Err(TenantSlotError::NotFound(*tenant_shard_id)) } _ => { let (completion, barrier) = utils::completion::channel(); v.insert(TenantSlot::InProgress(barrier)); tracing::debug!("Vacant, inserted InProgress"); - Ok(SlotGuard::new(*tenant_id, None, completion)) + Ok(SlotGuard::new(*tenant_shard_id, None, completion)) } }, Entry::Occupied(mut o) => { @@ -1706,7 +1826,7 @@ fn tenant_map_acquire_slot_impl( TenantSlot::Attached(tenant) => { tracing::debug!("Attached && MustNotExist, return AlreadyExists"); Err(TenantSlotError::AlreadyExists( - *tenant_id, + *tenant_shard_id, tenant.current_state(), )) } @@ -1715,7 +1835,7 @@ fn tenant_map_acquire_slot_impl( // to get the state from tracing::debug!("Occupied & MustNotExist, return AlreadyExists"); Err(TenantSlotError::AlreadyExists( - *tenant_id, + *tenant_shard_id, TenantState::Broken { reason: "Present but not attached".to_string(), backtrace: "".to_string(), @@ -1728,7 +1848,11 @@ fn tenant_map_acquire_slot_impl( let (completion, barrier) = utils::completion::channel(); let old_value = o.insert(TenantSlot::InProgress(barrier)); tracing::debug!("Occupied, replaced with InProgress"); - Ok(SlotGuard::new(*tenant_id, Some(old_value), completion)) + Ok(SlotGuard::new( + *tenant_shard_id, + Some(old_value), + completion, + )) } } } @@ -1741,7 +1865,7 @@ fn tenant_map_acquire_slot_impl( /// operation would be needed to remove it. async fn remove_tenant_from_memory( tenants: &std::sync::RwLock, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, tenant_cleanup: F, ) -> Result where @@ -1750,7 +1874,7 @@ where use utils::completion; let mut slot_guard = - tenant_map_acquire_slot_impl(&tenant_id, tenants, TenantSlotAcquireMode::MustExist)?; + tenant_map_acquire_slot_impl(&tenant_shard_id, tenants, TenantSlotAcquireMode::MustExist)?; // The SlotGuard allows us to manipulate the Tenant object without fear of some // concurrent API request doing something else for the same tenant ID. @@ -1777,7 +1901,7 @@ where // if pageserver shutdown or other detach/ignore is already ongoing, we don't want to // wait for it but return an error right away because these are distinct requests. slot_guard.revert(); - return Err(TenantStateError::IsStopping(tenant_id)); + return Err(TenantStateError::IsStopping(tenant_shard_id.tenant_id)); } } } @@ -1788,7 +1912,7 @@ where match tenant_cleanup .await - .with_context(|| format!("Failed to run cleanup for tenant {tenant_id}")) + .with_context(|| format!("Failed to run cleanup for tenant {tenant_shard_id}")) { Ok(hook_value) => { // Success: drop the old TenantSlot::Attached. @@ -1867,7 +1991,8 @@ pub(crate) async fn immediate_gc( #[cfg(test)] mod tests { - use std::collections::HashMap; + use pageserver_api::shard::TenantShardId; + use std::collections::BTreeMap; use std::sync::Arc; use tracing::{info_span, Instrument}; @@ -1887,12 +2012,12 @@ mod tests { // harness loads it to active, which is forced and nothing is running on the tenant - let id = t.tenant_id(); + let id = TenantShardId::unsharded(t.tenant_id()); // tenant harness configures the logging and we cannot escape it let _e = info_span!("testing", tenant_id = %id).entered(); - let tenants = HashMap::from([(id, TenantSlot::Attached(t.clone()))]); + let tenants = BTreeMap::from([(id, TenantSlot::Attached(t.clone()))]); let tenants = Arc::new(std::sync::RwLock::new(TenantsMap::Open(tenants))); // Invoke remove_tenant_from_memory with a cleanup hook that blocks until we manually From 6b82f22ada28c72797235df0cd6a8c2104e15df3 Mon Sep 17 00:00:00 2001 From: khanova <32508607+khanova@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:19:13 +0100 Subject: [PATCH 56/63] Collect number of connections by sni type (#5867) ## Problem We don't know the number of users with the different kind of authentication: ["sni", "endpoint in options" (A and B from [here](https://neon.tech/docs/connect/connection-errors)), "password_hack"] ## Summary of changes Collect metrics by sni kind. --- proxy/src/auth/credentials.rs | 20 +++++++++++++++++++- proxy/src/proxy.rs | 9 +++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index d7a8edca79..9fe9c26f0c 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -1,7 +1,9 @@ //! User credentials used in authentication. use crate::{ - auth::password_hack::parse_endpoint_param, error::UserFacingError, proxy::neon_options, + auth::password_hack::parse_endpoint_param, + error::UserFacingError, + proxy::{neon_options, NUM_CONNECTION_ACCEPTED_BY_SNI}, }; use itertools::Itertools; use pq_proto::StartupMessageParams; @@ -124,6 +126,22 @@ impl<'a> ClientCredentials<'a> { .transpose()?; info!(user, project = project.as_deref(), "credentials"); + if sni.is_some() { + info!("Connection with sni"); + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["sni"]) + .inc(); + } else if project.is_some() { + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["no_sni"]) + .inc(); + info!("Connection without sni"); + } else { + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["password_hack"]) + .inc(); + info!("Connection with password hack"); + } let cache_key = format!( "{}{}", diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 2d38acd05b..b03f95dc5f 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -129,6 +129,15 @@ pub static RATE_LIMITER_LIMIT: Lazy = Lazy::new(|| { .unwrap() }); +pub static NUM_CONNECTION_ACCEPTED_BY_SNI: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "proxy_accepted_connections_by_sni", + "Number of connections (per sni).", + &["kind"], + ) + .unwrap() +}); + pub struct LatencyTimer { // time since the stopwatch was started start: Option, From d0a842a5096d754852c424e25a4b3e1dc609fade Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 16 Nov 2023 18:17:42 +0100 Subject: [PATCH 57/63] Update vm-builder to v0.19.0 and move its customization here (#5783) ref neondatabase/autoscaling#600 for more --- .github/workflows/build_and_test.yml | 5 +- vm-image-spec.yaml | 126 +++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 vm-image-spec.yaml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ed7c0d3dfa..caa7ebee05 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -852,7 +852,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.18.5 + VM_BUILDER_VERSION: v0.19.0 steps: - name: Checkout @@ -874,8 +874,7 @@ jobs: - name: Build vm image run: | ./vm-builder \ - -enable-file-cache \ - -cgroup-uid=postgres \ + -spec=vm-image-spec.yaml \ -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} diff --git a/vm-image-spec.yaml b/vm-image-spec.yaml new file mode 100644 index 0000000000..2aa935fac6 --- /dev/null +++ b/vm-image-spec.yaml @@ -0,0 +1,126 @@ +# Supplemental file for neondatabase/autoscaling's vm-builder, for producing the VM compute image. +--- +commands: + - name: cgconfigparser + user: root + sysvInitAction: sysinit + shell: 'cgconfigparser -l /etc/cgconfig.conf -s 1664' + - name: pgbouncer + user: nobody + sysvInitAction: respawn + shell: '/usr/local/bin/pgbouncer /etc/pgbouncer.ini' + - name: postgres-exporter + user: nobody + sysvInitAction: respawn + shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres" /bin/postgres_exporter' +shutdownHook: | + su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10' +files: + - filename: pgbouncer.ini + content: | + [databases] + *=host=localhost port=5432 auth_user=cloud_admin + [pgbouncer] + listen_port=6432 + listen_addr=0.0.0.0 + auth_type=scram-sha-256 + auth_user=cloud_admin + auth_dbname=postgres + client_tls_sslmode=disable + server_tls_sslmode=disable + pool_mode=transaction + max_client_conn=10000 + default_pool_size=16 + max_prepared_statements=0 + - filename: cgconfig.conf + content: | + # Configuration for cgroups in VM compute nodes + group neon-postgres { + perm { + admin { + uid = postgres; + } + task { + gid = users; + } + } + memory {} + } +build: | + # Build cgroup-tools + # + # At time of writing (2023-03-14), debian bullseye has a version of cgroup-tools (technically + # libcgroup) that doesn't support cgroup v2 (version 0.41-11). Unfortunately, the vm-monitor + # requires cgroup v2, so we'll build cgroup-tools ourselves. + FROM debian:bullseye-slim as libcgroup-builder + ENV LIBCGROUP_VERSION v2.0.3 + + RUN set -exu \ + && apt update \ + && apt install --no-install-recommends -y \ + git \ + ca-certificates \ + automake \ + cmake \ + make \ + gcc \ + byacc \ + flex \ + libtool \ + libpam0g-dev \ + && git clone --depth 1 -b $LIBCGROUP_VERSION https://github.com/libcgroup/libcgroup \ + && INSTALL_DIR="/libcgroup-install" \ + && mkdir -p "$INSTALL_DIR/bin" "$INSTALL_DIR/include" \ + && cd libcgroup \ + # extracted from bootstrap.sh, with modified flags: + && (test -d m4 || mkdir m4) \ + && autoreconf -fi \ + && rm -rf autom4te.cache \ + && CFLAGS="-O3" ./configure --prefix="$INSTALL_DIR" --sysconfdir=/etc --localstatedir=/var --enable-opaque-hierarchy="name=systemd" \ + # actually build the thing... + && make install + + FROM quay.io/prometheuscommunity/postgres-exporter:v0.12.0 AS postgres-exporter + + # Build pgbouncer + # + FROM debian:bullseye-slim AS pgbouncer + RUN set -e \ + && apt-get update \ + && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libevent-dev \ + libssl-dev + + ENV PGBOUNCER_VERSION 1.21.0 + ENV PGBOUNCER_GITPATH 1_21_0 + RUN set -e \ + && curl -sfSL https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_${PGBOUNCER_GITPATH}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz -o pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ + && tar xzvf pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ + && cd pgbouncer-${PGBOUNCER_VERSION} \ + && LDFLAGS=-static ./configure --prefix=/usr/local/pgbouncer --without-openssl \ + && make -j $(nproc) \ + && make install +merge: | + # tweak nofile limits + RUN set -e \ + && echo 'fs.file-max = 1048576' >>/etc/sysctl.conf \ + && test ! -e /etc/security || ( \ + echo '* - nofile 1048576' >>/etc/security/limits.conf \ + && echo 'root - nofile 1048576' >>/etc/security/limits.conf \ + ) + + COPY cgconfig.conf /etc/cgconfig.conf + COPY pgbouncer.ini /etc/pgbouncer.ini + RUN set -e \ + && chown postgres:postgres /etc/pgbouncer.ini \ + && chmod 0644 /etc/pgbouncer.ini \ + && chmod 0644 /etc/cgconfig.conf + + COPY --from=libcgroup-builder /libcgroup-install/bin/* /usr/bin/ + COPY --from=libcgroup-builder /libcgroup-install/lib/* /usr/lib/ + COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/ + COPY --from=postgres-exporter /bin/postgres_exporter /bin/postgres_exporter + COPY --from=pgbouncer /usr/local/pgbouncer/bin/pgbouncer /usr/local/bin/pgbouncer From 0c243faf96c4ee9234bef5453c213b154b0781c9 Mon Sep 17 00:00:00 2001 From: khanova <32508607+khanova@users.noreply.github.com> Date: Thu, 16 Nov 2023 21:46:23 +0100 Subject: [PATCH 58/63] Proxy log pid hack (#5869) ## Problem Improve observability for the compute node. ## Summary of changes Log pid from the compute node. Doesn't work with pgbouncer. --- Cargo.lock | 10 +++---- Cargo.toml | 12 ++++---- proxy/src/compute.rs | 1 + proxy/src/proxy.rs | 2 +- proxy/src/serverless/conn_pool.rs | 40 +++++++++++++++++---------- proxy/src/serverless/sql_over_http.rs | 2 +- 6 files changed, 40 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 841c60c7e4..f08ed2d744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3221,7 +3221,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "bytes", "fallible-iterator", @@ -3234,7 +3234,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "native-tls", "tokio", @@ -3245,7 +3245,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "base64 0.20.0", "byteorder", @@ -3263,7 +3263,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "bytes", "fallible-iterator", @@ -4933,7 +4933,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "async-trait", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index bfdb0442ab..7d7d74993e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,11 +165,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -206,7 +206,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } ################# Binary contents sections diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 53eb0e3a76..0741ad0623 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -248,6 +248,7 @@ impl ConnCfg { // connect_raw() will not use TLS if sslmode is "disable" let (client, connection) = self.0.connect_raw(stream, tls).await?; + tracing::Span::current().record("pid", &tracing::field::display(client.get_process_id())); let stream = connection.stream.into_inner(); info!( diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index b03f95dc5f..b05b902303 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -514,7 +514,7 @@ pub fn invalidate_cache(node_info: console::CachedNodeInfo) -> compute::ConnCfg } /// Try to connect to the compute node once. -#[tracing::instrument(name = "connect_once", skip_all)] +#[tracing::instrument(name = "connect_once", fields(pid = tracing::field::Empty), skip_all)] async fn connect_to_compute_once( node_info: &console::CachedNodeInfo, timeout: time::Duration, diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index d09554a922..b753bc8918 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -208,14 +208,13 @@ impl GlobalConnPool { } else { info!("pool: reusing connection '{conn_info}'"); client.session.send(session_id)?; + tracing::Span::current().record( + "pid", + &tracing::field::display(client.inner.get_process_id()), + ); latency_timer.pool_hit(); latency_timer.success(); - return Ok(Client { - conn_id: client.conn_id, - inner: Some(client), - span: Span::current(), - pool, - }); + return Ok(Client::new(client, pool).await); } } else { let conn_id = uuid::Uuid::new_v4(); @@ -229,6 +228,12 @@ impl GlobalConnPool { ) .await }; + if let Ok(client) = &new_client { + tracing::Span::current().record( + "pid", + &tracing::field::display(client.inner.get_process_id()), + ); + } match &new_client { // clear the hash. it's no longer valid @@ -262,13 +267,8 @@ impl GlobalConnPool { } _ => {} } - - new_client.map(|inner| Client { - conn_id: inner.conn_id, - inner: Some(inner), - span: Span::current(), - pool, - }) + let new_client = new_client?; + Ok(Client::new(new_client, pool).await) } fn put(&self, conn_info: &ConnInfo, client: ClientInner) -> anyhow::Result<()> { @@ -394,7 +394,7 @@ impl ConnectMechanism for TokioMechanism<'_> { // Wake up the destination if needed. Code here is a bit involved because // we reuse the code from the usual proxy and we need to prepare few structures // that this code expects. -#[tracing::instrument(skip_all)] +#[tracing::instrument(fields(pid = tracing::field::Empty), skip_all)] async fn connect_to_compute( config: &config::ProxyConfig, conn_info: &ConnInfo, @@ -461,6 +461,7 @@ async fn connect_to_compute_once( .connect_timeout(timeout) .connect(tokio_postgres::NoTls) .await?; + tracing::Span::current().record("pid", &tracing::field::display(client.get_process_id())); let (tx, mut rx) = tokio::sync::watch::channel(session); @@ -547,6 +548,17 @@ pub struct Discard<'a> { } impl Client { + pub(self) async fn new( + inner: ClientInner, + pool: Option<(ConnInfo, Arc)>, + ) -> Self { + Self { + conn_id: inner.conn_id, + inner: Some(inner), + span: Span::current(), + pool, + } + } pub fn inner(&mut self) -> (&mut tokio_postgres::Client, Discard<'_>) { let Self { inner, diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 16736ac00d..4a9829e360 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -250,7 +250,7 @@ pub async fn handle( Ok(response) } -#[instrument(name = "sql-over-http", skip_all)] +#[instrument(name = "sql-over-http", fields(pid = tracing::field::Empty), skip_all)] async fn handle_inner( request: Request, sni_hostname: Option, From 5d13a2e4264ef12e1df23ed667a7ed0b31fc873d Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 16 Nov 2023 22:51:26 +0100 Subject: [PATCH 59/63] Improve error message when neon.max_cluster_size reached (#4173) Changes the error message encountered when the `neon.max_cluster_size` limit is reached. Reasoning is that this is user-visible, and so should *probably* use language that's closer to what users are familiar with. --- pgxn/neon/pagestore_smgr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index 3a841b04ec..84b26198a7 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -1687,9 +1687,9 @@ neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, if (current_size >= ((uint64) max_cluster_size) * 1024 * 1024) ereport(ERROR, (errcode(ERRCODE_DISK_FULL), - errmsg("could not extend file because cluster size limit (%d MB) has been exceeded", + errmsg("could not extend file because project size limit (%d MB) has been exceeded", max_cluster_size), - errhint("This limit is defined by neon.max_cluster_size GUC"))); + errhint("This limit is defined externally by the project size limit, and internally by neon.max_cluster_size GUC"))); } /* From cad0dca4b843759dbd8d5396c702ba4ec47de02f Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Sat, 18 Nov 2023 12:43:54 +0100 Subject: [PATCH 60/63] compute_ctl: Remove deprecated flag `--file-cache-on-disk` (#5622) See neondatabase/cloud#7516 for more. --- compute_tools/src/bin/compute_ctl.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 81d4320b14..7f22bda13e 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -479,13 +479,6 @@ fn cli() -> clap::Command { ) .value_name("FILECACHE_CONNSTR"), ) - .arg( - // DEPRECATED, NO LONGER DOES ANYTHING. - // See https://github.com/neondatabase/cloud/issues/7516 - Arg::new("file-cache-on-disk") - .long("file-cache-on-disk") - .action(clap::ArgAction::SetTrue), - ) } #[test] From 3b3f040be339efd89dd7057ebdded172f753d8ae Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Sun, 19 Nov 2023 15:16:31 +0100 Subject: [PATCH 61/63] fix(background_tasks): first backoff, compaction error stacktraces (#5881) First compaction/gc error backoff starts from 0 which is less than 2s what it was before #5672. This is now fixed to be the intended 2**n. Additionally noticed the `compaction_iteration` creating an `anyhow::Error` via `into()` always captures a stacktrace even if we had a stacktraceful anyhow error within the CompactionError because there is no stable api for querying that. --- pageserver/src/tenant.rs | 14 ++++---------- pageserver/src/tenant/tasks.rs | 4 ++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 758f8b15a1..4025e93f66 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1649,22 +1649,16 @@ impl Tenant { /// This function is periodically called by compactor task. /// Also it can be explicitly requested per timeline through page server /// api's 'compact' command. - pub async fn compaction_iteration( + async fn compaction_iteration( &self, cancel: &CancellationToken, ctx: &RequestContext, - ) -> anyhow::Result<()> { - // Don't start doing work during shutdown - if let TenantState::Stopping { .. } = self.current_state() { + ) -> anyhow::Result<(), timeline::CompactionError> { + // Don't start doing work during shutdown, or when broken, we do not need those in the logs + if !self.is_active() { return Ok(()); } - // We should only be called once the tenant has activated. - anyhow::ensure!( - self.is_active(), - "Cannot run compaction iteration on inactive tenant" - ); - { let conf = self.tenant_conf.read().unwrap(); if !conf.location.may_delete_layers_hint() || !conf.location.may_upload_layers_hint() { diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index eb77f7c83a..381d731b79 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -180,7 +180,7 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { // Run compaction if let Err(e) = tenant.compaction_iteration(&cancel, &ctx).await { let wait_duration = backoff::exponential_backoff_duration_seconds( - error_run_count, + error_run_count + 1, 1.0, MAX_BACKOFF_SECS, ); @@ -261,7 +261,7 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { .await; if let Err(e) = res { let wait_duration = backoff::exponential_backoff_duration_seconds( - error_run_count, + error_run_count + 1, 1.0, MAX_BACKOFF_SECS, ); From d22dce2e31093205d99cdc4aaff68499f8b6b749 Mon Sep 17 00:00:00 2001 From: John Spray Date: Sun, 19 Nov 2023 15:21:16 +0100 Subject: [PATCH 62/63] pageserver: shut down idle walredo processes (#5877) The longer a pageserver runs, the more walredo processes it accumulates from tenants that are touched intermittently (e.g. by availability checks). This can lead to getting OOM killed. Changes: - Add an Instant recording the last use of the walredo process for a tenant - After compaction iteration in the background task, check for idleness and stop the walredo process if idle for more than 10x compaction period. Cc: #3620 Co-authored-by: Joonas Koivunen Co-authored-by: Shany Pozin --- pageserver/src/tenant.rs | 10 ++++++++++ pageserver/src/tenant/tasks.rs | 4 ++++ pageserver/src/walredo.rs | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 4025e93f66..04fe9db76a 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -291,6 +291,16 @@ impl From for WalRedoManager { } impl WalRedoManager { + pub(crate) fn maybe_quiesce(&self, idle_timeout: Duration) { + match self { + Self::Prod(mgr) => mgr.maybe_quiesce(idle_timeout), + #[cfg(test)] + Self::Test(_) => { + // Not applicable to test redo manager + } + } + } + pub async fn request_redo( &self, key: crate::repository::Key, diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index 381d731b79..e59001297c 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -198,6 +198,10 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Compaction); + // Perhaps we did no work and the walredo process has been idle for some time: + // give it a chance to shut down to avoid leaving walredo process running indefinitely. + tenant.walredo_mgr.maybe_quiesce(period * 10); + // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) .await diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index ccdf621c30..9290940acf 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -91,6 +91,7 @@ struct ProcessOutput { pub struct PostgresRedoManager { tenant_id: TenantId, conf: &'static PageServerConf, + last_redo_at: std::sync::Mutex>, redo_process: RwLock>>, } @@ -187,10 +188,26 @@ impl PostgresRedoManager { PostgresRedoManager { tenant_id, conf, + last_redo_at: std::sync::Mutex::default(), redo_process: RwLock::new(None), } } + /// This type doesn't have its own background task to check for idleness: we + /// rely on our owner calling this function periodically in its own housekeeping + /// loops. + pub(crate) fn maybe_quiesce(&self, idle_timeout: Duration) { + if let Ok(g) = self.last_redo_at.try_lock() { + if let Some(last_redo_at) = *g { + if last_redo_at.elapsed() >= idle_timeout { + drop(g); + let mut guard = self.redo_process.write().unwrap(); + *guard = None; + } + } + } + } + /// /// Process one request for WAL redo using wal-redo postgres /// @@ -205,6 +222,8 @@ impl PostgresRedoManager { wal_redo_timeout: Duration, pg_version: u32, ) -> anyhow::Result { + *(self.last_redo_at.lock().unwrap()) = Some(Instant::now()); + let (rel, blknum) = key_to_rel_block(key).context("invalid record")?; const MAX_RETRY_ATTEMPTS: u32 = 1; let mut n_attempts = 0u32; From ac08072d2e9f4091358f47ab86af869bd8ee9231 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Sun, 19 Nov 2023 15:57:39 +0100 Subject: [PATCH 63/63] fix(layer): VirtualFile opening and read errors can be caused by contention (#5880) A very low number of layer loads have been marked wrongly as permanent, as I did not remember that `VirtualFile::open` or reading could fail transiently for contention. Return separate errors for transient and persistent errors from `{Delta,Image}LayerInner::load`. Includes drive-by comment changes. The implementation looks quite ugly because having the same type be both the inner (operation error) and outer (critical error), but with the alternatives I tried I did not find a better way. --- pageserver/src/metrics.rs | 2 +- .../src/tenant/storage_layer/delta_layer.rs | 30 +++++++++---- .../src/tenant/storage_layer/image_layer.rs | 33 +++++++++++---- pageserver/src/tenant/storage_layer/layer.rs | 42 ++++++++++++------- pageserver/src/walredo.rs | 7 ++-- 5 files changed, 77 insertions(+), 37 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 4b52f07326..d5915f4c98 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -638,7 +638,7 @@ const STORAGE_IO_TIME_BUCKETS: &[f64] = &[ /// /// Operations: /// - open ([`std::fs::OpenOptions::open`]) -/// - close (dropping [`std::fs::File`]) +/// - close (dropping [`crate::virtual_file::VirtualFile`]) /// - close-by-replace (close by replacement algorithm) /// - read (`read_at`) /// - write (`write_at`) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 4ccce3d6bd..79f37dcb2d 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -289,7 +289,9 @@ impl DeltaLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result> { let path = self.path(); - let loaded = DeltaLayerInner::load(&path, None, ctx).await?; + let loaded = DeltaLayerInner::load(&path, None, ctx) + .await + .and_then(|res| res)?; // not production code let actual_filename = path.file_name().unwrap().to_owned(); @@ -610,18 +612,28 @@ impl Drop for DeltaLayerWriter { } impl DeltaLayerInner { + /// Returns nested result following Result, Critical>: + /// - inner has the success or transient failure + /// - outer has the permanent failure pub(super) async fn load( path: &Utf8Path, summary: Option, ctx: &RequestContext, - ) -> anyhow::Result { - let file = VirtualFile::open(path) - .await - .with_context(|| format!("Failed to open file '{path}'"))?; + ) -> Result, anyhow::Error> { + let file = match VirtualFile::open(path).await { + Ok(file) => file, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("open layer file"))), + }; let file = FileBlockReader::new(file); - let summary_blk = file.read_blk(0, ctx).await?; - let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; + let summary_blk = match file.read_blk(0, ctx).await { + Ok(blk) => blk, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("read first block"))), + }; + + // TODO: this should be an assertion instead; see ImageLayerInner::load + let actual_summary = + Summary::des_prefix(summary_blk.as_ref()).context("deserialize first block")?; if let Some(mut expected_summary) = summary { // production code path @@ -636,11 +648,11 @@ impl DeltaLayerInner { } } - Ok(DeltaLayerInner { + Ok(Ok(DeltaLayerInner { file, index_start_blk: actual_summary.index_start_blk, index_root_blk: actual_summary.index_root_blk, - }) + })) } pub(super) async fn get_value_reconstruct_data( diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index b7b9ca69b0..c38a9f6883 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -249,7 +249,9 @@ impl ImageLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result { let path = self.path(); - let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, ctx).await?; + let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, ctx) + .await + .and_then(|res| res)?; // not production code let actual_filename = path.file_name().unwrap().to_owned(); @@ -295,18 +297,31 @@ impl ImageLayer { } impl ImageLayerInner { + /// Returns nested result following Result, Critical>: + /// - inner has the success or transient failure + /// - outer has the permanent failure pub(super) async fn load( path: &Utf8Path, lsn: Lsn, summary: Option, ctx: &RequestContext, - ) -> anyhow::Result { - let file = VirtualFile::open(path) - .await - .with_context(|| format!("Failed to open file '{}'", path))?; + ) -> Result, anyhow::Error> { + let file = match VirtualFile::open(path).await { + Ok(file) => file, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("open layer file"))), + }; let file = FileBlockReader::new(file); - let summary_blk = file.read_blk(0, ctx).await?; - let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; + let summary_blk = match file.read_blk(0, ctx).await { + Ok(blk) => blk, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("read first block"))), + }; + + // length is the only way how this could fail, so it's not actually likely at all unless + // read_blk returns wrong sized block. + // + // TODO: confirm and make this into assertion + let actual_summary = + Summary::des_prefix(summary_blk.as_ref()).context("deserialize first block")?; if let Some(mut expected_summary) = summary { // production code path @@ -322,12 +337,12 @@ impl ImageLayerInner { } } - Ok(ImageLayerInner { + Ok(Ok(ImageLayerInner { index_start_blk: actual_summary.index_start_blk, index_root_blk: actual_summary.index_root_blk, lsn, file, - }) + })) } pub(super) async fn get_value_reconstruct_data( diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 17df39733f..f28f1c9444 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -868,6 +868,9 @@ impl LayerInner { } Ok((Err(e), _permit)) => { // FIXME: this should be with the spawned task and be cancellation sensitive + // + // while we should not need this, this backoff has turned out to be useful with + // a bug of unexpectedly deleted remote layer file (#5787). let consecutive_failures = self.consecutive_failures.fetch_add(1, Ordering::Relaxed); tracing::error!(consecutive_failures, "layer file download failed: {e:#}"); @@ -1196,7 +1199,7 @@ impl DownloadedLayer { )); delta_layer::DeltaLayerInner::load(&owner.path, summary, ctx) .await - .map(LayerKind::Delta) + .map(|res| res.map(LayerKind::Delta)) } else { let lsn = owner.desc.image_layer_lsn(); let summary = Some(image_layer::Summary::expected( @@ -1207,23 +1210,32 @@ impl DownloadedLayer { )); image_layer::ImageLayerInner::load(&owner.path, lsn, summary, ctx) .await - .map(LayerKind::Image) - } - // this will be a permanent failure - .context("load layer"); + .map(|res| res.map(LayerKind::Image)) + }; - if let Err(e) = res.as_ref() { - LAYER_IMPL_METRICS.inc_permanent_loading_failures(); - // TODO(#5815): we are not logging all errors, so temporarily log them here as well - tracing::error!("layer loading failed permanently: {e:#}"); + match res { + Ok(Ok(layer)) => Ok(Ok(layer)), + Ok(Err(transient)) => Err(transient), + Err(permanent) => { + LAYER_IMPL_METRICS.inc_permanent_loading_failures(); + // TODO(#5815): we are not logging all errors, so temporarily log them **once** + // here as well + let permanent = permanent.context("load layer"); + tracing::error!("layer loading failed permanently: {permanent:#}"); + Ok(Err(permanent)) + } } - res }; - self.kind.get_or_init(init).await.as_ref().map_err(|e| { - // errors are not clonabled, cannot but stringify - // test_broken_timeline matches this string - anyhow::anyhow!("layer loading failed: {e:#}") - }) + self.kind + .get_or_try_init(init) + // return transient errors using `?` + .await? + .as_ref() + .map_err(|e| { + // errors are not clonabled, cannot but stringify + // test_broken_timeline matches this string + anyhow::anyhow!("layer loading failed: {e:#}") + }) } async fn get_value_reconstruct_data( diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 9290940acf..5d8cc0e181 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -367,12 +367,13 @@ impl PostgresRedoManager { self.apply_record_neon(key, &mut page, *record_lsn, record)?; } // Success! - let end_time = Instant::now(); - let duration = end_time.duration_since(start_time); + let duration = start_time.elapsed(); + // FIXME: using the same metric here creates a bimodal distribution by default, and because + // there could be multiple batch sizes this would be N+1 modal. WAL_REDO_TIME.observe(duration.as_secs_f64()); debug!( - "neon applied {} WAL records in {} ms to reconstruct page image at LSN {}", + "neon applied {} WAL records in {} us to reconstruct page image at LSN {}", records.len(), duration.as_micros(), lsn