From 7ebe9ca1aca6c6fb23507f984950cfd061e637f0 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 31 Oct 2023 09:32:58 +0100 Subject: [PATCH 01/29] pageserver: `/attach`: clarify semantics of 409 (#5698) context: https://app.incident.io/neondb/incidents/75 specifically: https://neondb.slack.com/archives/C0634NXQ6E7/p1698422852902959?thread_ts=1698419362.155059&cid=C0634NXQ6E7 --- pageserver/src/http/openapi_spec.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 2d954f6823..d31e4a1554 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -569,7 +569,17 @@ paths: schema: $ref: "#/components/schemas/NotFoundError" "409": - description: Tenant download is already in progress + description: | + The tenant is already known to Pageserver in some way, + and hence this `/attach` call has been rejected. + + Some examples of how this can happen: + - tenant was created on this pageserver + - tenant attachment was started by an earlier call to `/attach`. + + Callers should poll the tenant status's `attachment_status` field, + like for status 202. See the longer description for `POST /attach` + for details. content: application/json: schema: From e5c81fef86b47d319e98e24a8d3693d9659ada21 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 31 Oct 2023 11:44:35 +0000 Subject: [PATCH 02/29] tests: minor improvements (#5674) Minor changes from while I have been working on HA tests: - Manual pytest executions came with some warnings from `log.warn()` usage - When something fails in a generations-enabled test, it it useful to have a log from the attachment service of what attached when, and with which generation. --------- Co-authored-by: Joonas Koivunen --- control_plane/src/bin/attachment_service.rs | 26 +++++++++++++++++---- test_runner/fixtures/pageserver/utils.py | 4 ++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index 0ccd9e75fc..3205703c7d 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -12,6 +12,7 @@ use hyper::{Body, Request, Response}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use std::{collections::HashMap, sync::Arc}; +use utils::http::endpoint::request_span; use utils::logging::{self, LogFormat}; use utils::signals::{ShutdownSignals, Signal}; @@ -221,8 +222,25 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap generation: 0, }); - if attach_req.node_id.is_some() { + if let Some(attaching_pageserver) = attach_req.node_id.as_ref() { tenant_state.generation += 1; + tracing::info!( + tenant_id = %attach_req.tenant_id, + ps_id = %attaching_pageserver, + generation = %tenant_state.generation, + "issuing", + ); + } else if let Some(ps_id) = tenant_state.pageserver { + tracing::info!( + tenant_id = %attach_req.tenant_id, + %ps_id, + generation = %tenant_state.generation, + "dropping", + ); + } else { + tracing::info!( + tenant_id = %attach_req.tenant_id, + "no-op: tenant already has no pageserver"); } tenant_state.pageserver = attach_req.node_id; let generation = tenant_state.generation; @@ -240,9 +258,9 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap fn make_router(persistent_state: PersistentState) -> RouterBuilder { endpoint::make_router() .data(Arc::new(State::new(persistent_state))) - .post("/re-attach", handle_re_attach) - .post("/validate", handle_validate) - .post("/attach-hook", handle_attach_hook) + .post("/re-attach", |r| request_span(r, handle_re_attach)) + .post("/validate", |r| request_span(r, handle_validate)) + .post("/attach-hook", |r| request_span(r, handle_attach_hook)) } #[tokio::main] diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index e54b5408b4..007ff387f4 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -249,7 +249,7 @@ def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str # this has been seen in the wild by tests with the below contradicting logging # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-5322/6207777020/index.html#suites/3556ed71f2d69272a7014df6dcb02317/53b5c368b5a68865 # this seems like a mock_s3 issue - log.warn( + log.warning( f"contrading ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}, assuming this means KeyCount=0" ) keys = 0 @@ -257,7 +257,7 @@ def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str # this has been seen in one case with mock_s3: # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-4938/6000769714/index.html#suites/3556ed71f2d69272a7014df6dcb02317/ca01e4f4d8d9a11f # looking at moto impl, it might be there's a race with common prefix (sub directory) not going away with deletes - log.warn( + log.warning( f"contradicting ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}" ) From 896347f307653446cfd9b3c09f1ec94e283e566a Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 31 Oct 2023 18:40:08 +0200 Subject: [PATCH 03/29] refactor(layer): remove version checking with atomics (#5742) The `LayerInner::version` never needed to be read in more than one place. Clarified while fixing #5737 of which this is the first step. This decrements possible wrong atomics usage in Layer, but does not really fix anything. --- pageserver/src/tenant/storage_layer/layer.rs | 121 ++++++++++--------- 1 file changed, 66 insertions(+), 55 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 5a26421b5b..aaba9bd933 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -125,6 +125,7 @@ impl Layer { let inner = Arc::new(DownloadedLayer { owner: owner.clone(), kind: tokio::sync::OnceCell::default(), + version: 0, }); resident = Some(inner.clone()); @@ -163,6 +164,7 @@ impl Layer { let inner = Arc::new(DownloadedLayer { owner: owner.clone(), kind: tokio::sync::OnceCell::default(), + version: 0, }); resident = Some(inner.clone()); let access_stats = LayerAccessStats::empty_will_record_residence_event_later(); @@ -328,16 +330,17 @@ impl Layer { /// read with [`Layer::get_value_reconstruct_data`]. /// /// [`LayerMap::search`]: crate::tenant::layer_map::LayerMap::search +#[derive(Debug)] enum ResidentOrWantedEvicted { Resident(Arc), - WantedEvicted(Weak), + WantedEvicted(Weak, usize), } impl ResidentOrWantedEvicted { fn get(&self) -> Option> { match self { ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()), - ResidentOrWantedEvicted::WantedEvicted(weak) => match weak.upgrade() { + ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() { Some(strong) => { LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses(); Some(strong) @@ -349,21 +352,16 @@ impl ResidentOrWantedEvicted { /// When eviction is first requested, drop down to holding a [`Weak`]. /// /// Returns `true` if this was the first time eviction was requested. - fn downgrade(&mut self) -> &Weak { - let _was_first = match self { + fn downgrade(&mut self) -> bool { + match self { ResidentOrWantedEvicted::Resident(strong) => { let weak = Arc::downgrade(strong); - *self = ResidentOrWantedEvicted::WantedEvicted(weak); + *self = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version); // returning the weak is not useful, because the drop could had already ran with // the replacement above, and that will take care of cleaning the Option we are in true } - ResidentOrWantedEvicted::WantedEvicted(_) => false, - }; - - match self { - ResidentOrWantedEvicted::WantedEvicted(ref weak) => weak, - _ => unreachable!("just wrote wanted evicted"), + ResidentOrWantedEvicted::WantedEvicted(..) => false, } } } @@ -398,8 +396,10 @@ struct LayerInner { /// [`LayerInner::on_downloaded_layer_drop`]. wanted_evicted: AtomicBool, - /// Version is to make sure we will in fact only evict a file if no new download has been - /// started. + /// Version is to make sure we will only evict a specific download of a file. + /// + /// Incremented for each download, stored in `DownloadedLayer::version` or + /// `ResidentOrWantedEvicted::WantedEvicted`. version: AtomicUsize, /// Allow subscribing to when the layer actually gets evicted. @@ -515,6 +515,14 @@ impl LayerInner { .timeline_path(&timeline.tenant_id, &timeline.timeline_id) .join(desc.filename().to_string()); + let (inner, version) = if let Some(inner) = downloaded { + let version = inner.version; + let resident = ResidentOrWantedEvicted::Resident(inner); + (heavier_once_cell::OnceCell::new(resident), version) + } else { + (heavier_once_cell::OnceCell::default(), 0) + }; + LayerInner { conf, path, @@ -524,12 +532,8 @@ impl LayerInner { access_stats, wanted_garbage_collected: AtomicBool::new(false), wanted_evicted: AtomicBool::new(false), - inner: if let Some(inner) = downloaded { - heavier_once_cell::OnceCell::new(ResidentOrWantedEvicted::Resident(inner)) - } else { - heavier_once_cell::OnceCell::default() - }, - version: AtomicUsize::new(0), + inner, + version: AtomicUsize::new(version), status: tokio::sync::broadcast::channel(1).0, consecutive_failures: AtomicUsize::new(0), generation, @@ -604,7 +608,7 @@ impl LayerInner { loop { let download = move || async move { // disable any scheduled but not yet running eviction deletions for this - self.version.fetch_add(1, Ordering::Relaxed); + let next_version = 1 + self.version.fetch_add(1, Ordering::Relaxed); // no need to make the evict_and_wait wait for the actual download to complete drop(self.status.send(Status::Downloaded)); @@ -655,6 +659,7 @@ impl LayerInner { let res = Arc::new(DownloadedLayer { owner: Arc::downgrade(self), kind: tokio::sync::OnceCell::default(), + version: next_version, }); self.access_stats.record_residence_event( @@ -896,7 +901,7 @@ impl LayerInner { } /// `DownloadedLayer` is being dropped, so it calls this method. - fn on_downloaded_layer_drop(self: Arc) { + fn on_downloaded_layer_drop(self: Arc, version: usize) { let gc = self.wanted_garbage_collected.load(Ordering::Acquire); let evict = self.wanted_evicted.load(Ordering::Acquire); let can_evict = self.have_remote_client; @@ -904,15 +909,16 @@ impl LayerInner { if gc { // do nothing now, only in LayerInner::drop } else if can_evict && evict { - let version = self.version.load(Ordering::Relaxed); - - let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_id, timeline_id = %self.desc.timeline_id, layer=%self); + let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_id, timeline_id = %self.desc.timeline_id, layer=%self, %version); // downgrade for queueing, in case there's a tear down already ongoing we should not // hold it alive. let this = Arc::downgrade(&self); drop(self); + // NOTE: this scope *must* never call `self.inner.get` because evict_and_wait might + // drop while the `self.inner` is being locked, leading to a deadlock. + crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || { let _g = span.entered(); @@ -922,19 +928,15 @@ impl LayerInner { LAYER_IMPL_METRICS.inc_eviction_cancelled(EvictionCancelled::LayerGone); return; }; - this.evict_blocking(version); + match this.evict_blocking(version) { + Ok(()) => LAYER_IMPL_METRICS.inc_completed_evictions(), + Err(reason) => LAYER_IMPL_METRICS.inc_eviction_cancelled(reason), + } }); } } - fn evict_blocking(&self, version: usize) { - match self.evict_blocking0(version) { - Ok(()) => LAYER_IMPL_METRICS.inc_completed_evictions(), - Err(reason) => LAYER_IMPL_METRICS.inc_eviction_cancelled(reason), - } - } - - fn evict_blocking0(&self, version: usize) -> Result<(), EvictionCancelled> { + fn evict_blocking(&self, only_version: usize) -> Result<(), EvictionCancelled> { // deleted or detached timeline, don't do anything. let Some(timeline) = self.timeline.upgrade() else { return Err(EvictionCancelled::TimelineGone); @@ -945,32 +947,34 @@ impl LayerInner { let _permit = { let maybe_downloaded = self.inner.get(); - if version != self.version.load(Ordering::Relaxed) { - // downloadness-state has advanced, we might no longer be the latest eviction - // work; don't do anything. - // - // this is possible to get to by having: - // - // 1. wanted_evicted.store(true) - // 2. ResidentOrWantedEvicted::downgrade - // 3. DownloadedLayer::drop - // 4. LayerInner::get_or_maybe_download - // 5. LayerInner::evict_blocking - return Err(EvictionCancelled::VersionCheckFailed); - } - - // free the DownloadedLayer allocation - match maybe_downloaded.map(|mut g| g.take_and_deinit()) { - Some((taken, permit)) => { - assert!(matches!(taken, ResidentOrWantedEvicted::WantedEvicted(_))); - permit + let (_weak, permit) = match maybe_downloaded { + Some(mut guard) => { + if let ResidentOrWantedEvicted::WantedEvicted(_weak, version) = &*guard { + if *version == only_version { + guard.take_and_deinit() + } else { + // this was not for us; maybe there's another eviction job + // TODO: does it make any sense to stall here? unique versions do not + // matter, we only want to make sure not to evict a resident, which we + // are not doing. + return Err(EvictionCancelled::VersionCheckFailed); + } + } else { + return Err(EvictionCancelled::AlreadyReinitialized); + } } None => { - unreachable!("we do the version checking for this exact reason") + // already deinitialized, perhaps get_or_maybe_download did this and is + // currently waiting to reinitialize it + return Err(EvictionCancelled::LostToDownload); } - } + }; + + permit }; + // now accesses to inner.get_or_init wait on the semaphore or the `_permit` + self.access_stats.record_residence_event( LayerResidenceStatus::Evicted, LayerResidenceEventReason::ResidenceChange, @@ -1086,6 +1090,7 @@ impl std::fmt::Display for NeedsDownload { pub(crate) struct DownloadedLayer { owner: Weak, kind: tokio::sync::OnceCell>, + version: usize, } impl std::fmt::Debug for DownloadedLayer { @@ -1093,6 +1098,7 @@ impl std::fmt::Debug for DownloadedLayer { f.debug_struct("DownloadedLayer") // owner omitted because it is always "Weak" .field("kind", &self.kind) + .field("version", &self.version) .finish() } } @@ -1100,7 +1106,7 @@ impl std::fmt::Debug for DownloadedLayer { impl Drop for DownloadedLayer { fn drop(&mut self) { if let Some(owner) = self.owner.upgrade() { - owner.on_downloaded_layer_drop(); + owner.on_downloaded_layer_drop(self.version); } else { // no need to do anything, we are shutting down } @@ -1458,6 +1464,9 @@ enum EvictionCancelled { VersionCheckFailed, FileNotFound, RemoveFailed, + AlreadyReinitialized, + /// Not evicted because of a pending reinitialization + LostToDownload, } impl EvictionCancelled { @@ -1468,6 +1477,8 @@ impl EvictionCancelled { EvictionCancelled::VersionCheckFailed => "version_check_fail", EvictionCancelled::FileNotFound => "file_not_found", EvictionCancelled::RemoveFailed => "remove_failed", + EvictionCancelled::AlreadyReinitialized => "already_reinitialized", + EvictionCancelled::LostToDownload => "lost_to_download", } } } From f7067a38b7d8b8e7413325f15edf9735be416856 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Tue, 31 Oct 2023 10:00:23 -0700 Subject: [PATCH 04/29] compute_ctl: Assume --vm-monitor-addr arg is always present (#5611) It has a default value, so this should be sound. Treating its presence as semantically significant was leading to spurious warnings. --- compute_tools/src/bin/compute_ctl.rs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 04ac077c7b..263f71452d 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -278,8 +278,9 @@ fn main() -> Result<()> { if #[cfg(target_os = "linux")] { use std::env; use tokio_util::sync::CancellationToken; - use tracing::warn; - let vm_monitor_addr = matches.get_one::("vm-monitor-addr"); + let vm_monitor_addr = matches + .get_one::("vm-monitor-addr") + .expect("--vm-monitor-addr should always be set because it has a default arg"); let file_cache_connstr = matches.get_one::("filecache-connstr"); let cgroup = matches.get_one::("cgroup"); let file_cache_on_disk = matches.get_flag("file-cache-on-disk"); @@ -288,22 +289,16 @@ fn main() -> Result<()> { // Note: it seems like you can make a runtime in an inner scope and // if you start a task in it it won't be dropped. However, make it // in the outermost scope just to be safe. - let rt = match (env::var_os("AUTOSCALING"), vm_monitor_addr) { - (None, None) => None, - (None, Some(_)) => { - warn!("--vm-monitor-addr option set but AUTOSCALING env var not present"); - None - } - (Some(_), None) => { - panic!("AUTOSCALING env var present but --vm-monitor-addr option not set") - } - (Some(_), Some(_)) => Some( + let rt = if env::var_os("AUTOSCALING").is_some() { + Some( tokio::runtime::Builder::new_multi_thread() .worker_threads(4) .enable_all() .build() - .expect("failed to create tokio runtime for monitor"), - ), + .expect("failed to create tokio runtime for monitor") + ) + } else { + None }; // This token is used internally by the monitor to clean up all threads @@ -314,7 +309,7 @@ fn main() -> Result<()> { Box::leak(Box::new(vm_monitor::Args { cgroup: cgroup.cloned(), pgconnstr: file_cache_connstr.cloned(), - addr: vm_monitor_addr.cloned().unwrap(), + addr: vm_monitor_addr.clone(), file_cache_on_disk, })), token.clone(), From 726c8e673008c607788b9914013b3245b4eabba2 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 11 Aug 2023 12:26:38 -0500 Subject: [PATCH 05/29] Add docs for updating Postgres for new minor versions --- docs/updating-postgres.md | 108 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 docs/updating-postgres.md diff --git a/docs/updating-postgres.md b/docs/updating-postgres.md new file mode 100644 index 0000000000..1868bbf5f7 --- /dev/null +++ b/docs/updating-postgres.md @@ -0,0 +1,108 @@ +# Updating Postgres + +## Minor Versions + +When upgrading to a new minor version of Postgres, please follow these steps: + +_Example: 15.4 is the new minor version to upgrade to from 15.3._ + +1. Clone the Neon Postgres repository if you have not done so already. + + ```shell + git clone git@github.com:neondatabase/postgres.git + ``` + +1. Add the Postgres upstream remote. + + ```shell + git remote add upstream https://git.postgresql.org/git/postgresql.git + ``` + +1. Create a new branch based on the stable branch you are updating. + + ```shell + git checkout -b my-branch REL_15_STABLE_neon + ``` + +1. Tag the last commit on the stable branch you are updating. + + ```shell + git tag REL_15_3_neon + ``` + +1. Push the new tag to the Neon Postgres repository. + + ```shell + git push origin REL_15_3_neon + ``` + +1. Find the release tags you're looking for. They are of the form `REL_X_Y`. + +1. Rebase the branch you created on the tag and resolve any conflicts. + + ```shell + git fetch upstream REL_15_4 + git rebase REL_15_4 + ``` + +1. Run the Postgres test suite to make sure our commits have not affected +Postgres in a negative way. + + ```shell + make check + # OR + meson test -C builddir + ``` + +1. Push your branch to the Neon Postgres repository. + + ```shell + git push origin my-branch + ``` + +1. Clone the Neon repository if you have not done so already. + + ```shell + git clone git@github.com:neondatabase/neon.git + ``` + +1. Create a new branch. + +1. Change the `revisions.json` file to point at the HEAD of your Postgres +branch. + +1. Update the Git submodule. + + ```shell + git submodule set-branch --branch my-branch vendor/postgres-v15 + git submodule update --remote vendor/postgres-v15 + ``` + +1. Run the Neon test suite to make sure that Neon is still good to go on this +minor Postgres release. + + ```shell + ./scripts/poetry -k pg15 + ``` + +1. Commit your changes. + +1. Create a pull request, and wait for CI to go green. + +1. Force push the rebased Postgres branches into the Neon Postgres repository. + + ```shell + git push --force origin my-branch:REL_15_STABLE_neon + ``` + + It may require disabling various branch protections. + +1. Update your Neon PR to point at the branches. + + ```shell + git submodule set-branch --branch REL_15_STABLE_neon vendor/postgres-v15 + git commit --amend --no-edit + git push --force origin + ``` + +1. Merge the pull request after getting approval(s) and CI completion. From 5952f350cb0c627a70998da3e9f90158d22fe8bc Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 31 Oct 2023 20:57:03 +0200 Subject: [PATCH 06/29] Always handle POLLHUP in walredo error poll loop (#5716) ## Problem test_stderr hangs on MacOS. See https://neondb.slack.com/archives/C036U0GRMRB/p1698438997903919 ## Summary of changes Always handle POLLHUP to prevent infinite loop. ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist Co-authored-by: Konstantin Knizhnik --- pageserver/src/walredo.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index cffd912e16..e1d699fcba 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -857,7 +857,8 @@ impl WalRedoProcess { let in_revents = stdin_pollfds[0].revents().unwrap(); if in_revents & (PollFlags::POLLERR | PollFlags::POLLOUT) != PollFlags::empty() { nwrite += proc.stdin.write(&writebuf[nwrite..])?; - } else if in_revents.contains(PollFlags::POLLHUP) { + } + if in_revents.contains(PollFlags::POLLHUP) { // We still have more data to write, but the process closed the pipe. anyhow::bail!("WAL redo process closed its stdin unexpectedly"); } @@ -907,7 +908,8 @@ impl WalRedoProcess { let out_revents = stdout_pollfds[0].revents().unwrap(); if out_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { nresult += output.stdout.read(&mut resultbuf[nresult..])?; - } else if out_revents.contains(PollFlags::POLLHUP) { + } + if out_revents.contains(PollFlags::POLLHUP) { anyhow::bail!("WAL redo process closed its stdout unexpectedly"); } } From 4f0a8e92ad83cf0fad4436f3265d3eec2c601c96 Mon Sep 17 00:00:00 2001 From: Muhammet Yazici <44066377+mtyazici@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:41:48 +0300 Subject: [PATCH 07/29] fix: Add bearer prefix to Authorization header (#5740) ## Problem Some requests with `Authorization` header did not properly set the `Bearer ` prefix. Problem explained here https://github.com/neondatabase/cloud/issues/6390. ## Summary of changes Added `Bearer ` prefix to missing requests. --- compute_tools/src/spec.rs | 2 +- pageserver/src/control_plane_client.rs | 5 ++++- proxy/src/console/provider/neon.rs | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 6e4d4ccf49..3fa3cf055f 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -24,7 +24,7 @@ fn do_control_plane_request( ) -> Result { let resp = reqwest::blocking::Client::new() .get(uri) - .header("Authorization", jwt) + .header("Authorization", format!("Bearer {}", jwt)) .send() .map_err(|e| { ( diff --git a/pageserver/src/control_plane_client.rs b/pageserver/src/control_plane_client.rs index be4fcab75e..f50c19a629 100644 --- a/pageserver/src/control_plane_client.rs +++ b/pageserver/src/control_plane_client.rs @@ -57,7 +57,10 @@ impl ControlPlaneClient { if let Some(jwt) = &conf.control_plane_api_token { let mut headers = hyper::HeaderMap::new(); - headers.insert("Authorization", jwt.get_contents().parse().unwrap()); + headers.insert( + "Authorization", + format!("Bearer {}", jwt.get_contents()).parse().unwrap(), + ); client = client.default_headers(headers); } diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 163cdfffc0..927fea0a13 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -49,7 +49,7 @@ impl Api { .endpoint .get("proxy_get_role_secret") .header("X-Request-ID", &request_id) - .header("Authorization", &self.jwt) + .header("Authorization", format!("Bearer {}", &self.jwt)) .query(&[("session_id", extra.session_id)]) .query(&[ ("application_name", extra.application_name), @@ -94,7 +94,7 @@ impl Api { .endpoint .get("proxy_wake_compute") .header("X-Request-ID", &request_id) - .header("Authorization", &self.jwt) + .header("Authorization", format!("Bearer {}", &self.jwt)) .query(&[("session_id", extra.session_id)]) .query(&[ ("application_name", extra.application_name), From e82d1ad6b8f00544d20614788f0f09fb02fb096d Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Wed, 1 Nov 2023 17:38:32 +0200 Subject: [PATCH 08/29] fix(layer): reinit on access before eviction happens (#5743) Right before merging, I added a loop to `fn LayerInner::get_or_maybe_download`, which was always supposed to be there. However I had forgotten to restart initialization instead of waiting for the eviction to happen to support original design goal of "eviction should always lose to redownload (or init)". This was wrong. After this fix, if `spawn_blocking` queue is blocked on something, nothing bad will happen. Part of #5737. --- pageserver/src/tenant/storage_layer/layer.rs | 130 +++++++++++-------- 1 file changed, 75 insertions(+), 55 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index aaba9bd933..761fe311c6 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -337,31 +337,41 @@ enum ResidentOrWantedEvicted { } impl ResidentOrWantedEvicted { - fn get(&self) -> Option> { + /// If `Some` is returned, the ResidentOrWantedEvicted has been upgraded back from + /// `ResidentOrWantedEvicted::WantedEvicted` to `ResidentOrWantedEvicted::Resident`. + fn get_and_upgrade(&mut self) -> Option> { match self { ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()), ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() { Some(strong) => { LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses(); + + *self = ResidentOrWantedEvicted::Resident(strong.clone()); + Some(strong) } None => None, }, } } + /// When eviction is first requested, drop down to holding a [`Weak`]. /// - /// Returns `true` if this was the first time eviction was requested. - fn downgrade(&mut self) -> bool { + /// Returns `Some` if this was the first time eviction was requested. Care should be taken to + /// drop the possibly last strong reference outside of the mutex of + /// heavier_once_cell::OnceCell. + fn downgrade(&mut self) -> Option> { match self { ResidentOrWantedEvicted::Resident(strong) => { let weak = Arc::downgrade(strong); - *self = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version); - // returning the weak is not useful, because the drop could had already ran with - // the replacement above, and that will take care of cleaning the Option we are in - true + let mut temp = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version); + std::mem::swap(self, &mut temp); + match temp { + ResidentOrWantedEvicted::Resident(strong) => Some(strong), + ResidentOrWantedEvicted::WantedEvicted(..) => unreachable!("just swapped"), + } } - ResidentOrWantedEvicted::WantedEvicted(..) => false, + ResidentOrWantedEvicted::WantedEvicted(..) => None, } } } @@ -563,20 +573,22 @@ impl LayerInner { let mut rx = self.status.subscribe(); - let res = - self.wanted_evicted - .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed); + let strong = { + match self.inner.get() { + Some(mut either) => { + self.wanted_evicted.store(true, Ordering::Relaxed); + either.downgrade() + } + None => return Err(EvictionError::NotFound), + } + }; - if res.is_ok() { + if strong.is_some() { + // drop the DownloadedLayer outside of the holding the guard + drop(strong); LAYER_IMPL_METRICS.inc_started_evictions(); } - if self.get().is_none() { - // it was not evictable in the first place - // our store to the wanted_evicted does not matter; it will be reset by next download - return Err(EvictionError::NotFound); - } - match rx.recv().await { Ok(Status::Evicted) => Ok(()), Ok(Status::Downloaded) => Err(EvictionError::Downloaded), @@ -590,7 +602,8 @@ impl LayerInner { // // use however late (compared to the initial expressing of wanted) as the // "outcome" now - match self.get() { + LAYER_IMPL_METRICS.inc_broadcast_lagged(); + match self.inner.get() { Some(_) => Err(EvictionError::Downloaded), None => Ok(()), } @@ -605,6 +618,8 @@ impl LayerInner { allow_download: bool, ctx: Option<&RequestContext>, ) -> Result, DownloadError> { + let mut permit = None; + loop { let download = move || async move { // disable any scheduled but not yet running eviction deletions for this @@ -622,6 +637,8 @@ impl LayerInner { // check if we really need to be downloaded; could have been already downloaded by a // cancelled previous attempt. + // + // FIXME: what if it's a directory? that is currently needs_download == true let needs_download = self .needs_download() .await @@ -670,16 +687,37 @@ impl LayerInner { Ok(ResidentOrWantedEvicted::Resident(res)) }; - let locked = self.inner.get_or_init(download).await?; + let (weak, _permit) = { + // should we be able to give the permit to the `get_or_init`? would make sense. + drop(permit.take()); + let mut locked = self.inner.get_or_init(download).await?; - if let Some(strong) = Self::get_or_apply_evictedness(Some(locked), &self.wanted_evicted) - { - return Ok(strong); - } + if let Some(strong) = locked.get_and_upgrade() { + self.wanted_evicted.store(false, Ordering::Relaxed); + + // error out any `evict_and_wait` + drop(self.status.send(Status::Downloaded)); + + return Ok(strong); + } else { + // path to here: the evict_blocking is stuck on spawn_blocking queue. + // + // reset the contents, deactivating the eviction and causing a + // EvictionCancelled::LostToDownload or EvictionCancelled::VersionCheckFailed. + locked.take_and_deinit() + } + }; + + // unlock first, then drop the weak, but because upgrade failed, we + // know it cannot be a problem. + + assert!( + matches!(weak, ResidentOrWantedEvicted::WantedEvicted(..)), + "unexpected {weak:?}, ResidentOrWantedEvicted::get_and_upgrade has a bug" + ); + + permit = Some(_permit); - // the situation in which we might need to retry is that our init was ready - // immediatedly, but the DownloadedLayer had been dropped BUT failed to complete - // Self::evict_blocking LAYER_IMPL_METRICS.inc_retried_get_or_maybe_download(); } } @@ -812,33 +850,6 @@ impl LayerInner { } } - /// Access the current state without waiting for the file to be downloaded. - /// - /// Requires that we've initialized to state which is respective to the - /// actual residency state. - fn get(&self) -> Option> { - let locked = self.inner.get(); - Self::get_or_apply_evictedness(locked, &self.wanted_evicted) - } - - fn get_or_apply_evictedness( - guard: Option>, - wanted_evicted: &AtomicBool, - ) -> Option> { - if let Some(mut x) = guard { - if let Some(won) = x.get() { - // there are no guarantees that we will always get to observe a concurrent call - // to evict - if wanted_evicted.load(Ordering::Acquire) { - x.downgrade(); - } - return Some(won); - } - } - - None - } - async fn needs_download(&self) -> Result, std::io::Error> { match tokio::fs::metadata(&self.path).await { Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()), @@ -872,7 +883,9 @@ impl LayerInner { fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.desc.filename().file_name(); - let remote = self.get().is_none(); + // this is not accurate: we could have the file locally but there was a cancellation + // and now we are not in sync, or we are currently downloading it. + let remote = self.inner.get().is_none(); let access_stats = self.access_stats.as_api_model(reset); @@ -1456,6 +1469,13 @@ impl LayerImplMetrics { .unwrap() .inc(); } + + fn inc_broadcast_lagged(&self) { + self.rare_counters + .get_metric_with_label_values(&["broadcast_lagged"]) + .unwrap() + .inc(); + } } enum EvictionCancelled { From 0b790b6d0058dcc47e2e56160a7f2c3b595483d9 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Wed, 1 Nov 2023 17:02:58 -0400 Subject: [PATCH 09/29] Record wal size in import benchmark (#5755) --- test_runner/performance/test_bulk_insert.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py index 62301f3919..46acec0f63 100644 --- a/test_runner/performance/test_bulk_insert.py +++ b/test_runner/performance/test_bulk_insert.py @@ -1,8 +1,10 @@ from contextlib import closing +from fixtures.benchmark_fixture import MetricReport from fixtures.compare_fixtures import NeonCompare, PgCompare from fixtures.pageserver.utils import wait_tenant_status_404 from fixtures.pg_version import PgVersion +from fixtures.types import Lsn # @@ -18,6 +20,8 @@ from fixtures.pg_version import PgVersion def test_bulk_insert(neon_with_baseline: PgCompare): env = neon_with_baseline + start_lsn = Lsn(env.pg.safe_psql("SELECT pg_current_wal_lsn()")[0][0]) + with closing(env.pg.connect()) as conn: with conn.cursor() as cur: cur.execute("create table huge (i int, j int);") @@ -31,6 +35,13 @@ def test_bulk_insert(neon_with_baseline: PgCompare): env.report_peak_memory_use() env.report_size() + # Report amount of wal written. Useful for comparing vanilla wal format vs + # neon wal format, measuring neon write amplification, etc. + end_lsn = Lsn(env.pg.safe_psql("SELECT pg_current_wal_lsn()")[0][0]) + wal_written_bytes = end_lsn - start_lsn + wal_written_mb = round(wal_written_bytes / (1024 * 1024)) + env.zenbenchmark.record("wal_written", wal_written_mb, "MB", MetricReport.TEST_PARAM) + # When testing neon, also check how long it takes the pageserver to reingest the # wal from safekeepers. If this number is close to total runtime, then the pageserver # is the bottleneck. From 2dca4c03fc8d12f83398408504eee7c404870d95 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 10:06:32 +0200 Subject: [PATCH 10/29] feat(layer): cancellable get_or_maybe_download (#5744) With the layer implementation as was done in #4938, it is possible via cancellation to cause two concurrent downloads on the same path, due to how `RemoteTimelineClient::download_remote_layer` does tempfiles. Thread the init semaphore through the spawned task of downloading to make this impossible to happen. --- libs/utils/src/sync/heavier_once_cell.rs | 104 +++++++++++++------ pageserver/src/tenant/storage_layer/layer.rs | 48 ++++++--- 2 files changed, 105 insertions(+), 47 deletions(-) diff --git a/libs/utils/src/sync/heavier_once_cell.rs b/libs/utils/src/sync/heavier_once_cell.rs index 4d66a54c98..8a5aaf2ceb 100644 --- a/libs/utils/src/sync/heavier_once_cell.rs +++ b/libs/utils/src/sync/heavier_once_cell.rs @@ -60,8 +60,8 @@ impl OnceCell { /// Initialization is panic-safe and cancellation-safe. pub async fn get_or_init(&self, factory: F) -> Result, E> where - F: FnOnce() -> Fut, - Fut: std::future::Future>, + F: FnOnce(InitPermit) -> Fut, + Fut: std::future::Future>, { let sem = { let guard = self.inner.lock().unwrap(); @@ -72,28 +72,55 @@ impl OnceCell { }; let permit = sem.acquire_owned().await; - if permit.is_err() { - let guard = self.inner.lock().unwrap(); - assert!( - guard.value.is_some(), - "semaphore got closed, must be initialized" - ); - return Ok(Guard(guard)); - } else { - // now we try - let value = factory().await?; - let mut guard = self.inner.lock().unwrap(); - assert!( - guard.value.is_none(), - "we won permit, must not be initialized" - ); - guard.value = Some(value); - guard.init_semaphore.close(); - Ok(Guard(guard)) + match permit { + Ok(permit) => { + let permit = InitPermit(permit); + let (value, _permit) = factory(permit).await?; + + let guard = self.inner.lock().unwrap(); + + Ok(Self::set0(value, guard)) + } + Err(_closed) => { + let guard = self.inner.lock().unwrap(); + assert!( + guard.value.is_some(), + "semaphore got closed, must be initialized" + ); + return Ok(Guard(guard)); + } } } + /// Assuming a permit is held after previous call to [`Guard::take_and_deinit`], it can be used + /// to complete initializing the inner value. + /// + /// # Panics + /// + /// If the inner has already been initialized. + pub fn set(&self, value: T, _permit: InitPermit) -> Guard<'_, T> { + // cannot assert that this permit is for self.inner.semaphore + let guard = self.inner.lock().unwrap(); + + if guard.init_semaphore.try_acquire().is_ok() { + drop(guard); + panic!("semaphore is of wrong origin"); + } + + Self::set0(value, guard) + } + + fn set0(value: T, mut guard: std::sync::MutexGuard<'_, Inner>) -> Guard<'_, T> { + if guard.value.is_some() { + drop(guard); + unreachable!("we won permit, must not be initialized"); + } + guard.value = Some(value); + guard.init_semaphore.close(); + Guard(guard) + } + /// Returns a guard to an existing initialized value, if any. pub fn get(&self) -> Option> { let guard = self.inner.lock().unwrap(); @@ -135,7 +162,7 @@ impl<'a, T> Guard<'a, T> { /// /// The permit will be on a semaphore part of the new internal value, and any following /// [`OnceCell::get_or_init`] will wait on it to complete. - pub fn take_and_deinit(&mut self) -> (T, tokio::sync::OwnedSemaphorePermit) { + pub fn take_and_deinit(&mut self) -> (T, InitPermit) { let mut swapped = Inner::default(); let permit = swapped .init_semaphore @@ -145,11 +172,14 @@ impl<'a, T> Guard<'a, T> { std::mem::swap(&mut *self.0, &mut swapped); swapped .value - .map(|v| (v, permit)) + .map(|v| (v, InitPermit(permit))) .expect("guard is not created unless value has been initialized") } } +/// Type held by OnceCell (de)initializing task. +pub struct InitPermit(tokio::sync::OwnedSemaphorePermit); + #[cfg(test)] mod tests { use super::*; @@ -185,11 +215,11 @@ mod tests { barrier.wait().await; let won = { let g = cell - .get_or_init(|| { + .get_or_init(|permit| { counters.factory_got_to_run.fetch_add(1, Ordering::Relaxed); async { counters.future_polled.fetch_add(1, Ordering::Relaxed); - Ok::<_, Infallible>(i) + Ok::<_, Infallible>((i, permit)) } }) .await @@ -243,7 +273,7 @@ mod tests { deinitialization_started.wait().await; let started_at = tokio::time::Instant::now(); - cell.get_or_init(|| async { Ok::<_, Infallible>(reinit) }) + cell.get_or_init(|permit| async { Ok::<_, Infallible>((reinit, permit)) }) .await .unwrap(); @@ -258,18 +288,32 @@ mod tests { assert_eq!(*cell.get().unwrap(), reinit); } + #[test] + fn reinit_with_deinit_permit() { + let cell = Arc::new(OnceCell::new(42)); + + let (mol, permit) = cell.get().unwrap().take_and_deinit(); + cell.set(5, permit); + assert_eq!(*cell.get().unwrap(), 5); + + let (five, permit) = cell.get().unwrap().take_and_deinit(); + assert_eq!(5, five); + cell.set(mol, permit); + assert_eq!(*cell.get().unwrap(), 42); + } + #[tokio::test] async fn initialization_attemptable_until_ok() { let cell = OnceCell::default(); for _ in 0..10 { - cell.get_or_init(|| async { Err("whatever error") }) + cell.get_or_init(|_permit| async { Err("whatever error") }) .await .unwrap_err(); } let g = cell - .get_or_init(|| async { Ok::<_, Infallible>("finally success") }) + .get_or_init(|permit| async { Ok::<_, Infallible>(("finally success", permit)) }) .await .unwrap(); assert_eq!(*g, "finally success"); @@ -281,11 +325,11 @@ mod tests { let barrier = tokio::sync::Barrier::new(2); - let initializer = cell.get_or_init(|| async { + let initializer = cell.get_or_init(|permit| async { barrier.wait().await; futures::future::pending::<()>().await; - Ok::<_, Infallible>("never reached") + Ok::<_, Infallible>(("never reached", permit)) }); tokio::select! { @@ -298,7 +342,7 @@ mod tests { assert!(cell.get().is_none()); let g = cell - .get_or_init(|| async { Ok::<_, Infallible>("now initialized") }) + .get_or_init(|permit| async { Ok::<_, Infallible>(("now initialized", permit)) }) .await .unwrap(); assert_eq!(*g, "now initialized"); diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 761fe311c6..94edfa6fe0 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -618,10 +618,10 @@ impl LayerInner { allow_download: bool, ctx: Option<&RequestContext>, ) -> Result, DownloadError> { - let mut permit = None; + let mut init_permit = None; loop { - let download = move || async move { + let download = move |permit| async move { // disable any scheduled but not yet running eviction deletions for this let next_version = 1 + self.version.fetch_add(1, Ordering::Relaxed); @@ -644,7 +644,7 @@ impl LayerInner { .await .map_err(DownloadError::PreStatFailed)?; - if let Some(reason) = needs_download { + let permit = if let Some(reason) = needs_download { // only reset this after we've decided we really need to download. otherwise it'd // be impossible to mark cancelled downloads for eviction, like one could imagine // we would like to do for prefetching which was not needed. @@ -666,12 +666,14 @@ impl LayerInner { return Err(DownloadError::DownloadRequired); } - self.spawn_download_and_wait(timeline).await?; + self.spawn_download_and_wait(timeline, permit).await? } else { // the file is present locally, probably by a previous but cancelled call to // get_or_maybe_download. alternatively we might be running without remote storage. LAYER_IMPL_METRICS.inc_init_needed_no_download(); - } + + permit + }; let res = Arc::new(DownloadedLayer { owner: Arc::downgrade(self), @@ -684,12 +686,21 @@ impl LayerInner { LayerResidenceEventReason::ResidenceChange, ); - Ok(ResidentOrWantedEvicted::Resident(res)) + Ok((ResidentOrWantedEvicted::Resident(res), permit)) }; - let (weak, _permit) = { - // should we be able to give the permit to the `get_or_init`? would make sense. - drop(permit.take()); + if let Some(init_permit) = init_permit.take() { + // use the already held initialization permit because it is impossible to hit the + // below paths anymore essentially limiting the max loop iterations to 2. + let (value, init_permit) = download(init_permit).await?; + let mut guard = self.inner.set(value, init_permit); + let strong = guard + .get_and_upgrade() + .expect("init creates strong reference, we held the init permit"); + return Ok(strong); + } + + let (weak, permit) = { let mut locked = self.inner.get_or_init(download).await?; if let Some(strong) = locked.get_and_upgrade() { @@ -716,7 +727,7 @@ impl LayerInner { "unexpected {weak:?}, ResidentOrWantedEvicted::get_and_upgrade has a bug" ); - permit = Some(_permit); + init_permit = Some(permit); LAYER_IMPL_METRICS.inc_retried_get_or_maybe_download(); } @@ -752,10 +763,12 @@ impl LayerInner { async fn spawn_download_and_wait( self: &Arc, timeline: Arc, - ) -> Result<(), DownloadError> { + permit: heavier_once_cell::InitPermit, + ) -> Result { let task_name = format!("download layer {}", self); let (tx, rx) = tokio::sync::oneshot::channel(); + // this is sadly needed because of task_mgr::shutdown_tasks, otherwise we cannot // block tenant::mgr::remove_tenant_from_memory. @@ -789,9 +802,9 @@ impl LayerInner { } }; - if let Err(res) = tx.send(result) { + if let Err(res) = tx.send((result, permit)) { match res { - Ok(()) => { + (Ok(()), _) => { // our caller is cancellation safe so this is fine; if someone // else requests the layer, they'll find it already downloaded // or redownload. @@ -802,7 +815,7 @@ impl LayerInner { tracing::info!("layer file download completed after requester had cancelled"); LAYER_IMPL_METRICS.inc_download_completed_without_requester(); }, - Err(e) => { + (Err(e), _) => { // our caller is cancellation safe, but we might be racing with // another attempt to initialize. before we have cancellation // token support: these attempts should converge regardless of @@ -818,7 +831,7 @@ impl LayerInner { .in_current_span(), ); match rx.await { - Ok(Ok(())) => { + Ok((Ok(()), permit)) => { if let Some(reason) = self .needs_download() .await @@ -830,9 +843,10 @@ impl LayerInner { self.consecutive_failures.store(0, Ordering::Relaxed); - Ok(()) + Ok(permit) } - Ok(Err(e)) => { + Ok((Err(e), _permit)) => { + // FIXME: this should be with the spawned task and be cancellation sensitive let consecutive_failures = self.consecutive_failures.fetch_add(1, Ordering::Relaxed); tracing::error!(consecutive_failures, "layer file download failed: {e:#}"); From 5650138532e5e3ca4a06f15f3ed79d6a3545f5a0 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 2 Nov 2023 09:14:26 +0000 Subject: [PATCH 11/29] pageserver: helpers for explicitly dying on fatal I/O errors (#5651) Following from discussion on https://github.com/neondatabase/neon/pull/5436 where hacking an implicit die-on-fatal-io behavior into an Error type was a source of disagreement -- in this PR, dying on fatal I/O errors is explicit, with `fatal_err` and `maybe_fatal_err` helpers in the `MaybeFatalIo` trait, which is implemented for std::io::Result. To enable this approach with `crashsafe_overwrite`, the return type of that function is changed to std::io::Result -- the previous error enum for this function was not used for any logic, and the utility of saying exactly which step in the function failed is outweighed by the hygiene of having an I/O funciton return an io::Result. The initial use case for these helpers is the deletion queue. --- pageserver/src/deletion_queue.rs | 6 +- pageserver/src/deletion_queue/list_writer.rs | 31 ++--- pageserver/src/deletion_queue/validator.rs | 14 +- pageserver/src/virtual_file.rs | 135 +++++++++++-------- 4 files changed, 104 insertions(+), 82 deletions(-) diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 22efa23f10..7b2db929fa 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -10,6 +10,7 @@ use crate::control_plane_client::ControlPlaneGenerationsApi; use crate::metrics; use crate::tenant::remote_timeline_client::remote_layer_path; use crate::tenant::remote_timeline_client::remote_timeline_path; +use crate::virtual_file::MaybeFatalIo; use crate::virtual_file::VirtualFile; use anyhow::Context; use camino::Utf8PathBuf; @@ -271,7 +272,9 @@ impl DeletionHeader { let temp_path = path_with_suffix_extension(&header_path, TEMP_SUFFIX); VirtualFile::crashsafe_overwrite(&header_path, &temp_path, &header_bytes) .await - .map_err(Into::into) + .maybe_fatal_err("save deletion header")?; + + Ok(()) } } @@ -360,6 +363,7 @@ impl DeletionList { let bytes = serde_json::to_vec(self).expect("Failed to serialize deletion list"); VirtualFile::crashsafe_overwrite(&path, &temp_path, &bytes) .await + .maybe_fatal_err("save deletion list") .map_err(Into::into) } } diff --git a/pageserver/src/deletion_queue/list_writer.rs b/pageserver/src/deletion_queue/list_writer.rs index 6727957b2a..28daae2da5 100644 --- a/pageserver/src/deletion_queue/list_writer.rs +++ b/pageserver/src/deletion_queue/list_writer.rs @@ -34,6 +34,8 @@ use crate::deletion_queue::TEMP_SUFFIX; use crate::metrics; use crate::tenant::remote_timeline_client::remote_layer_path; use crate::tenant::storage_layer::LayerFileName; +use crate::virtual_file::on_fatal_io_error; +use crate::virtual_file::MaybeFatalIo; // The number of keys in a DeletionList before we will proactively persist it // (without reaching a flush deadline). This aims to deliver objects of the order @@ -195,7 +197,7 @@ impl ListWriter { debug!("Deletion header {header_path} not found, first start?"); Ok(None) } else { - Err(anyhow::anyhow!(e)) + on_fatal_io_error(&e, "reading deletion header"); } } } @@ -216,16 +218,9 @@ impl ListWriter { self.pending.sequence = validated_sequence + 1; let deletion_directory = self.conf.deletion_prefix(); - let mut dir = match tokio::fs::read_dir(&deletion_directory).await { - Ok(d) => d, - Err(e) => { - warn!("Failed to open deletion list directory {deletion_directory}: {e:#}"); - - // Give up: if we can't read the deletion list directory, we probably can't - // write lists into it later, so the queue won't work. - return Err(e.into()); - } - }; + let mut dir = tokio::fs::read_dir(&deletion_directory) + .await + .fatal_err("read deletion directory"); let list_name_pattern = Regex::new("(?[a-zA-Z0-9]{16})-(?[a-zA-Z0-9]{2}).list").unwrap(); @@ -233,7 +228,7 @@ impl ListWriter { let temp_extension = format!(".{TEMP_SUFFIX}"); let header_path = self.conf.deletion_header_path(); let mut seqs: Vec = Vec::new(); - while let Some(dentry) = dir.next_entry().await? { + while let Some(dentry) = dir.next_entry().await.fatal_err("read deletion dentry") { let file_name = dentry.file_name(); let dentry_str = file_name.to_string_lossy(); @@ -246,11 +241,9 @@ impl ListWriter { info!("Cleaning up temporary file {dentry_str}"); let absolute_path = deletion_directory.join(dentry.file_name().to_str().expect("non-Unicode path")); - if let Err(e) = tokio::fs::remove_file(&absolute_path).await { - // Non-fatal error: we will just leave the file behind but not - // try and load it. - warn!("Failed to clean up temporary file {absolute_path}: {e:#}"); - } + tokio::fs::remove_file(&absolute_path) + .await + .fatal_err("delete temp file"); continue; } @@ -290,7 +283,9 @@ impl ListWriter { for s in seqs { let list_path = self.conf.deletion_list_path(s); - let list_bytes = tokio::fs::read(&list_path).await?; + let list_bytes = tokio::fs::read(&list_path) + .await + .fatal_err("read deletion list"); let mut deletion_list = match serde_json::from_slice::(&list_bytes) { Ok(l) => l, diff --git a/pageserver/src/deletion_queue/validator.rs b/pageserver/src/deletion_queue/validator.rs index a2cbfb9dc7..72bdbdefd6 100644 --- a/pageserver/src/deletion_queue/validator.rs +++ b/pageserver/src/deletion_queue/validator.rs @@ -28,6 +28,7 @@ use crate::config::PageServerConf; use crate::control_plane_client::ControlPlaneGenerationsApi; use crate::control_plane_client::RetryForeverError; use crate::metrics; +use crate::virtual_file::MaybeFatalIo; use super::deleter::DeleterMessage; use super::DeletionHeader; @@ -287,16 +288,9 @@ where async fn cleanup_lists(&mut self, list_paths: Vec) { for list_path in list_paths { debug!("Removing deletion list {list_path}"); - - if let Err(e) = tokio::fs::remove_file(&list_path).await { - // Unexpected: we should have permissions and nothing else should - // be touching these files. We will leave the file behind. Subsequent - // pageservers will try and load it again: hopefully whatever storage - // issue (probably permissions) has been fixed by then. - tracing::error!("Failed to delete {list_path}: {e:#}"); - metrics::DELETION_QUEUE.unexpected_errors.inc(); - break; - } + tokio::fs::remove_file(&list_path) + .await + .fatal_err("remove deletion list"); } } diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index a2e8f30e15..b58b883ab6 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -19,6 +19,7 @@ use std::io::{Error, ErrorKind, Seek, SeekFrom}; use std::os::unix::fs::FileExt; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{RwLock, RwLockWriteGuard}; +use utils::fs_ext; /// /// A virtual file descriptor. You can use this just like std::fs::File, but internally @@ -173,37 +174,78 @@ impl OpenFiles { } } -#[derive(Debug, thiserror::Error)] -pub enum CrashsafeOverwriteError { - #[error("final path has no parent dir")] - FinalPathHasNoParentDir, - #[error("remove tempfile")] - RemovePreviousTempfile(#[source] std::io::Error), - #[error("create tempfile")] - CreateTempfile(#[source] std::io::Error), - #[error("write tempfile")] - WriteContents(#[source] std::io::Error), - #[error("sync tempfile")] - SyncTempfile(#[source] std::io::Error), - #[error("rename tempfile to final path")] - RenameTempfileToFinalPath(#[source] std::io::Error), - #[error("open final path parent dir")] - OpenFinalPathParentDir(#[source] std::io::Error), - #[error("sync final path parent dir")] - SyncFinalPathParentDir(#[source] std::io::Error), +/// Identify error types that should alwways terminate the process. Other +/// error types may be elegible for retry. +pub(crate) fn is_fatal_io_error(e: &std::io::Error) -> bool { + use nix::errno::Errno::*; + match e.raw_os_error().map(nix::errno::from_i32) { + Some(EIO) => { + // Terminate on EIO because we no longer trust the device to store + // data safely, or to uphold persistence guarantees on fsync. + true + } + Some(EROFS) => { + // Terminate on EROFS because a filesystem is usually remounted + // readonly when it has experienced some critical issue, so the same + // logic as EIO applies. + true + } + Some(EACCES) => { + // Terminate on EACCESS because we should always have permissions + // for our own data dir: if we don't, then we can't do our job and + // need administrative intervention to fix permissions. Terminating + // is the best way to make sure we stop cleanly rather than going + // into infinite retry loops, and will make it clear to the outside + // world that we need help. + true + } + _ => { + // Treat all other local file I/O errors are retryable. This includes: + // - ENOSPC: we stay up and wait for eviction to free some space + // - EINVAL, EBADF, EBADFD: this is a code bug, not a filesystem/hardware issue + // - WriteZero, Interrupted: these are used internally VirtualFile + false + } + } } -impl CrashsafeOverwriteError { - /// Returns true iff the new contents are durably stored. - pub fn are_new_contents_durable(&self) -> bool { + +/// Call this when the local filesystem gives us an error with an external +/// cause: this includes EIO, EROFS, and EACCESS: all these indicate either +/// bad storage or bad configuration, and we can't fix that from inside +/// a running process. +pub(crate) fn on_fatal_io_error(e: &std::io::Error, context: &str) -> ! { + tracing::error!("Fatal I/O error: {e}: {context})"); + std::process::abort(); +} + +pub(crate) trait MaybeFatalIo { + fn maybe_fatal_err(self, context: &str) -> std::io::Result; + fn fatal_err(self, context: &str) -> T; +} + +impl MaybeFatalIo for std::io::Result { + /// Terminate the process if the result is an error of a fatal type, else pass it through + /// + /// This is appropriate for writes, where we typically want to die on EIO/ACCES etc, but + /// not on ENOSPC. + fn maybe_fatal_err(self, context: &str) -> std::io::Result { + if let Err(e) = &self { + if is_fatal_io_error(e) { + on_fatal_io_error(e, context); + } + } + self + } + + /// Terminate the process on any I/O error. + /// + /// This is appropriate for reads on files that we know exist: they should always work. + fn fatal_err(self, context: &str) -> T { match self { - Self::FinalPathHasNoParentDir => false, - Self::RemovePreviousTempfile(_) => false, - Self::CreateTempfile(_) => false, - Self::WriteContents(_) => false, - Self::SyncTempfile(_) => false, - Self::RenameTempfileToFinalPath(_) => false, - Self::OpenFinalPathParentDir(_) => false, - Self::SyncFinalPathParentDir(_) => true, + Ok(v) => v, + Err(e) => { + on_fatal_io_error(&e, context); + } } } } @@ -284,15 +326,13 @@ impl VirtualFile { final_path: &Utf8Path, tmp_path: &Utf8Path, content: &[u8], - ) -> Result<(), CrashsafeOverwriteError> { + ) -> std::io::Result<()> { let Some(final_path_parent) = final_path.parent() else { - return Err(CrashsafeOverwriteError::FinalPathHasNoParentDir); + return Err(std::io::Error::from_raw_os_error( + nix::errno::Errno::EINVAL as i32, + )); }; - match std::fs::remove_file(tmp_path) { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(CrashsafeOverwriteError::RemovePreviousTempfile(e)), - } + std::fs::remove_file(tmp_path).or_else(fs_ext::ignore_not_found)?; let mut file = Self::open_with_options( tmp_path, OpenOptions::new() @@ -301,31 +341,20 @@ impl VirtualFile { // we bail out instead of causing damage. .create_new(true), ) - .await - .map_err(CrashsafeOverwriteError::CreateTempfile)?; - file.write_all(content) - .await - .map_err(CrashsafeOverwriteError::WriteContents)?; - file.sync_all() - .await - .map_err(CrashsafeOverwriteError::SyncTempfile)?; + .await?; + file.write_all(content).await?; + file.sync_all().await?; drop(file); // before the rename, that's important! // renames are atomic - std::fs::rename(tmp_path, final_path) - .map_err(CrashsafeOverwriteError::RenameTempfileToFinalPath)?; + std::fs::rename(tmp_path, final_path)?; // Only open final path parent dirfd now, so that this operation only // ever holds one VirtualFile fd at a time. That's important because // the current `find_victim_slot` impl might pick the same slot for both // VirtualFile., and it eventually does a blocking write lock instead of // try_lock. let final_parent_dirfd = - Self::open_with_options(final_path_parent, OpenOptions::new().read(true)) - .await - .map_err(CrashsafeOverwriteError::OpenFinalPathParentDir)?; - final_parent_dirfd - .sync_all() - .await - .map_err(CrashsafeOverwriteError::SyncFinalPathParentDir)?; + Self::open_with_options(final_path_parent, OpenOptions::new().read(true)).await?; + final_parent_dirfd.sync_all().await?; Ok(()) } From 3737fe3a4bf50d8e24aa6eea400e6464aa73bcda Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 13:03:38 +0200 Subject: [PATCH 12/29] fix(layer): error out early if layer path is non-file (#5756) In an earlier PR https://github.com/neondatabase/neon/pull/5743#discussion_r1378625244 I added a FIXME and there's a simple solution suggested by @jcsp, so implement it. Wondering why I did not implement this originally, there is no concept of a permanent failure, so this failure will happen quite often. I don't think the frequency is a problem however. Sadly for std::fs::FileType there is only decimal and hex formatting, no octal. --- pageserver/src/tenant/storage_layer/layer.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 94edfa6fe0..fc4ba75dfc 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -637,14 +637,16 @@ impl LayerInner { // check if we really need to be downloaded; could have been already downloaded by a // cancelled previous attempt. - // - // FIXME: what if it's a directory? that is currently needs_download == true let needs_download = self .needs_download() .await .map_err(DownloadError::PreStatFailed)?; let permit = if let Some(reason) = needs_download { + if let NeedsDownload::NotFile(ft) = reason { + return Err(DownloadError::NotFile(ft)); + } + // only reset this after we've decided we really need to download. otherwise it'd // be impossible to mark cancelled downloads for eviction, like one could imagine // we would like to do for prefetching which was not needed. @@ -883,7 +885,7 @@ impl LayerInner { fn is_file_present_and_good_size(&self, m: &std::fs::Metadata) -> Result<(), NeedsDownload> { // in future, this should include sha2-256 validation of the file. if !m.is_file() { - Err(NeedsDownload::NotFile) + Err(NeedsDownload::NotFile(m.file_type())) } else if m.len() != self.desc.file_size { Err(NeedsDownload::WrongSize { actual: m.len(), @@ -1082,6 +1084,8 @@ enum DownloadError { ContextAndConfigReallyDeniesDownloads, #[error("downloading is really required but not allowed by this method")] DownloadRequired, + #[error("layer path exists, but it is not a file: {0:?}")] + NotFile(std::fs::FileType), /// Why no error here? Because it will be reported by page_service. We should had also done /// retries already. #[error("downloading evicted layer file failed")] @@ -1097,7 +1101,7 @@ enum DownloadError { #[derive(Debug, PartialEq)] pub(crate) enum NeedsDownload { NotFound, - NotFile, + NotFile(std::fs::FileType), WrongSize { actual: u64, expected: u64 }, } @@ -1105,7 +1109,7 @@ impl std::fmt::Display for NeedsDownload { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { NeedsDownload::NotFound => write!(f, "file was not found"), - NeedsDownload::NotFile => write!(f, "path is not a file"), + NeedsDownload::NotFile(ft) => write!(f, "path is not a file; {ft:?}"), NeedsDownload::WrongSize { actual, expected } => { write!(f, "file size mismatch {actual} vs. {expected}") } From 098d3111a5c4fb057571246d637edfa54d95b334 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 15:06:14 +0200 Subject: [PATCH 13/29] fix(layer): get_and_upgrade and metrics (#5767) when introducing `get_and_upgrade` I forgot that an `evict_and_wait` would had already incremented the counter for started evictions, but an upgrade would just "silently" cancel the eviction as no drop would ever run. these metrics are likely sources for alerts with the next release, so it's important to keep them correct. --- pageserver/src/tenant/storage_layer/layer.rs | 27 ++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index fc4ba75dfc..b320c02f9b 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -337,18 +337,16 @@ enum ResidentOrWantedEvicted { } impl ResidentOrWantedEvicted { - /// If `Some` is returned, the ResidentOrWantedEvicted has been upgraded back from - /// `ResidentOrWantedEvicted::WantedEvicted` to `ResidentOrWantedEvicted::Resident`. - fn get_and_upgrade(&mut self) -> Option> { + fn get_and_upgrade(&mut self) -> Option<(Arc, bool)> { match self { - ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()), + ResidentOrWantedEvicted::Resident(strong) => Some((strong.clone(), false)), ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() { Some(strong) => { LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses(); *self = ResidentOrWantedEvicted::Resident(strong.clone()); - Some(strong) + Some((strong, true)) } None => None, }, @@ -696,7 +694,7 @@ impl LayerInner { // below paths anymore essentially limiting the max loop iterations to 2. let (value, init_permit) = download(init_permit).await?; let mut guard = self.inner.set(value, init_permit); - let strong = guard + let (strong, _upgraded) = guard .get_and_upgrade() .expect("init creates strong reference, we held the init permit"); return Ok(strong); @@ -705,11 +703,17 @@ impl LayerInner { let (weak, permit) = { let mut locked = self.inner.get_or_init(download).await?; - if let Some(strong) = locked.get_and_upgrade() { - self.wanted_evicted.store(false, Ordering::Relaxed); + if let Some((strong, upgraded)) = locked.get_and_upgrade() { + if upgraded { + // when upgraded back, the Arc is still available, but + // previously a `evict_and_wait` was received. + self.wanted_evicted.store(false, Ordering::Relaxed); - // error out any `evict_and_wait` - drop(self.status.send(Status::Downloaded)); + // error out any `evict_and_wait` + drop(self.status.send(Status::Downloaded)); + LAYER_IMPL_METRICS + .inc_eviction_cancelled(EvictionCancelled::UpgradedBackOnAccess); + } return Ok(strong); } else { @@ -1505,6 +1509,8 @@ enum EvictionCancelled { AlreadyReinitialized, /// Not evicted because of a pending reinitialization LostToDownload, + /// After eviction, there was a new layer access which cancelled the eviction. + UpgradedBackOnAccess, } impl EvictionCancelled { @@ -1517,6 +1523,7 @@ impl EvictionCancelled { EvictionCancelled::RemoveFailed => "remove_failed", EvictionCancelled::AlreadyReinitialized => "already_reinitialized", EvictionCancelled::LostToDownload => "lost_to_download", + EvictionCancelled::UpgradedBackOnAccess => "upgraded_back_on_access", } } } From 51570114ea5dc5c5c73edc5460b3cf2ce76e2970 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 2 Nov 2023 10:43:59 -0400 Subject: [PATCH 14/29] Remove outdated and flaky perf test (#5762) --- scripts/benchmark_durations.py | 1 - test_runner/performance/test_startup.py | 49 ------------------------- 2 files changed, 50 deletions(-) diff --git a/scripts/benchmark_durations.py b/scripts/benchmark_durations.py index 781d0d2d1e..7f05d72a03 100755 --- a/scripts/benchmark_durations.py +++ b/scripts/benchmark_durations.py @@ -81,7 +81,6 @@ FALLBACK_DURATION = { "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-100000-100-0]": 0.55, "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-0]": 12.189, "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-4]": 13.899, - "test_runner/performance/test_startup.py::test_startup": 890.114, "test_runner/performance/test_startup.py::test_startup_simple": 2.51, "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_off-10-5-5]": 527.245, "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_on-10-5-5]": 583.46, diff --git a/test_runner/performance/test_startup.py b/test_runner/performance/test_startup.py index 366b9b0e68..301078d984 100644 --- a/test_runner/performance/test_startup.py +++ b/test_runner/performance/test_startup.py @@ -1,6 +1,3 @@ -from contextlib import closing - -import pytest import requests from fixtures.benchmark_fixture import MetricReport, NeonBenchmarker from fixtures.neon_fixtures import NeonEnvBuilder @@ -81,49 +78,3 @@ def test_startup_simple(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenc # Imitate optimizations that console would do for the second start endpoint.respec(skip_pg_catalog_updates=True) - - -# This test sometimes runs for longer than the global 5 minute timeout. -@pytest.mark.timeout(900) -def test_startup(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchmarker): - neon_env_builder.num_safekeepers = 3 - env = neon_env_builder.init_start() - - # Start - env.neon_cli.create_branch("test_startup") - with zenbenchmark.record_duration("startup_time"): - endpoint = env.endpoints.create_start("test_startup") - endpoint.safe_psql("select 1;") - - # Restart - endpoint.stop_and_destroy() - with zenbenchmark.record_duration("restart_time"): - endpoint.create_start("test_startup") - endpoint.safe_psql("select 1;") - - # Fill up - num_rows = 1000000 # 30 MB - num_tables = 100 - with closing(endpoint.connect()) as conn: - with conn.cursor() as cur: - for i in range(num_tables): - cur.execute(f"create table t_{i} (i integer);") - cur.execute(f"insert into t_{i} values (generate_series(1,{num_rows}));") - - # Read - with zenbenchmark.record_duration("read_time"): - endpoint.safe_psql("select * from t_0;") - - # Read again - with zenbenchmark.record_duration("second_read_time"): - endpoint.safe_psql("select * from t_0;") - - # Restart - endpoint.stop_and_destroy() - with zenbenchmark.record_duration("restart_with_data"): - endpoint.create_start("test_startup") - endpoint.safe_psql("select 1;") - - # Read - with zenbenchmark.record_duration("read_after_restart"): - endpoint.safe_psql("select * from t_0;") From 367971a0e9671abd2bcb45fbfa168ae6de585bb5 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 2 Nov 2023 09:06:16 -0700 Subject: [PATCH 15/29] vm-monitor: Remove support for file cache in tmpfs (#5617) ref neondatabase/cloud#7516. We switched everything over to file cache on disk, now time to remove support for having it in tmpfs. --- compute_tools/src/bin/compute_ctl.rs | 4 ++-- libs/vm_monitor/src/filecache.rs | 24 ++++--------------- libs/vm_monitor/src/lib.rs | 10 -------- libs/vm_monitor/src/runner.rs | 35 +++++++--------------------- 4 files changed, 15 insertions(+), 58 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 263f71452d..81d4320b14 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -283,7 +283,6 @@ fn main() -> Result<()> { .expect("--vm-monitor-addr should always be set because it has a default arg"); let file_cache_connstr = matches.get_one::("filecache-connstr"); let cgroup = matches.get_one::("cgroup"); - let file_cache_on_disk = matches.get_flag("file-cache-on-disk"); // Only make a runtime if we need to. // Note: it seems like you can make a runtime in an inner scope and @@ -310,7 +309,6 @@ fn main() -> Result<()> { cgroup: cgroup.cloned(), pgconnstr: file_cache_connstr.cloned(), addr: vm_monitor_addr.clone(), - file_cache_on_disk, })), token.clone(), )) @@ -482,6 +480,8 @@ fn cli() -> clap::Command { .value_name("FILECACHE_CONNSTR"), ) .arg( + // DEPRECATED, NO LONGER DOES ANYTHING. + // See https://github.com/neondatabase/cloud/issues/7516 Arg::new("file-cache-on-disk") .long("file-cache-on-disk") .action(clap::ArgAction::SetTrue), diff --git a/libs/vm_monitor/src/filecache.rs b/libs/vm_monitor/src/filecache.rs index 3a860500f1..fe71e11197 100644 --- a/libs/vm_monitor/src/filecache.rs +++ b/libs/vm_monitor/src/filecache.rs @@ -21,11 +21,6 @@ pub struct FileCacheState { #[derive(Debug)] pub struct FileCacheConfig { - /// Whether the file cache is *actually* stored in memory (e.g. by writing to - /// a tmpfs or shmem file). If true, the size of the file cache will be counted against the - /// memory available for the cgroup. - pub(crate) in_memory: bool, - /// The size of the file cache, in terms of the size of the resource it consumes /// (currently: only memory) /// @@ -59,22 +54,9 @@ pub struct FileCacheConfig { spread_factor: f64, } -impl FileCacheConfig { - pub fn default_in_memory() -> Self { +impl Default for FileCacheConfig { + fn default() -> Self { Self { - in_memory: true, - // 75 % - resource_multiplier: 0.75, - // 640 MiB; (512 + 128) - min_remaining_after_cache: NonZeroU64::new(640 * MiB).unwrap(), - // ensure any increase in file cache size is split 90-10 with 10% to other memory - spread_factor: 0.1, - } - } - - pub fn default_on_disk() -> Self { - Self { - in_memory: false, resource_multiplier: 0.75, // 256 MiB - lower than when in memory because overcommitting is safe; if we don't have // memory, the kernel will just evict from its page cache, rather than e.g. killing @@ -83,7 +65,9 @@ impl FileCacheConfig { spread_factor: 0.1, } } +} +impl FileCacheConfig { /// Make sure fields of the config are consistent. pub fn validate(&self) -> anyhow::Result<()> { // Single field validity diff --git a/libs/vm_monitor/src/lib.rs b/libs/vm_monitor/src/lib.rs index 1cbfdc6ba6..a844f78bd6 100644 --- a/libs/vm_monitor/src/lib.rs +++ b/libs/vm_monitor/src/lib.rs @@ -39,16 +39,6 @@ pub struct Args { #[arg(short, long)] pub pgconnstr: Option, - /// Flag to signal that the Postgres file cache is on disk (i.e. not in memory aside from the - /// kernel's page cache), and therefore should not count against available memory. - // - // NB: Ideally this flag would directly refer to whether the file cache is in memory (rather - // than a roundabout way, via whether it's on disk), but in order to be backwards compatible - // during the switch away from an in-memory file cache, we had to default to the previous - // behavior. - #[arg(long)] - pub file_cache_on_disk: bool, - /// The address we should listen on for connection requests. For the /// agent, this is 0.0.0.0:10301. For the informant, this is 127.0.0.1:10369. #[arg(short, long)] diff --git a/libs/vm_monitor/src/runner.rs b/libs/vm_monitor/src/runner.rs index 99bea6d7ca..f162f53d24 100644 --- a/libs/vm_monitor/src/runner.rs +++ b/libs/vm_monitor/src/runner.rs @@ -156,10 +156,7 @@ impl Runner { // memory limits. if let Some(connstr) = &args.pgconnstr { info!("initializing file cache"); - let config = match args.file_cache_on_disk { - true => FileCacheConfig::default_on_disk(), - false => FileCacheConfig::default_in_memory(), - }; + let config = FileCacheConfig::default(); let mut file_cache = FileCacheState::new(connstr, config, token.clone()) .await @@ -187,10 +184,7 @@ impl Runner { info!("file cache size actually got set to {actual_size}") } - if args.file_cache_on_disk { - file_cache_disk_size = actual_size; - } - + file_cache_disk_size = actual_size; state.filecache = Some(file_cache); } @@ -239,17 +233,11 @@ impl Runner { let requested_mem = target.mem; let usable_system_memory = requested_mem.saturating_sub(self.config.sys_buffer_bytes); - let (expected_file_cache_size, expected_file_cache_disk_size) = self + let expected_file_cache_size = self .filecache .as_ref() - .map(|file_cache| { - let size = file_cache.config.calculate_cache_size(usable_system_memory); - match file_cache.config.in_memory { - true => (size, 0), - false => (size, size), - } - }) - .unwrap_or((0, 0)); + .map(|file_cache| file_cache.config.calculate_cache_size(usable_system_memory)) + .unwrap_or(0); if let Some(cgroup) = &self.cgroup { let (last_time, last_history) = *cgroup.watcher.borrow(); @@ -273,7 +261,7 @@ impl Runner { let new_threshold = self .config - .cgroup_threshold(usable_system_memory, expected_file_cache_disk_size); + .cgroup_threshold(usable_system_memory, expected_file_cache_size); let current = last_history.avg_non_reclaimable; @@ -300,13 +288,10 @@ impl Runner { .set_file_cache_size(expected_file_cache_size) .await .context("failed to set file cache size")?; - if !file_cache.config.in_memory { - file_cache_disk_size = actual_usage; - } + file_cache_disk_size = actual_usage; let message = format!( - "set file cache size to {} MiB (in memory = {})", + "set file cache size to {} MiB", bytes_to_mebibytes(actual_usage), - file_cache.config.in_memory, ); info!("downscale: {message}"); status.push(message); @@ -357,9 +342,7 @@ impl Runner { .set_file_cache_size(expected_usage) .await .context("failed to set file cache size")?; - if !file_cache.config.in_memory { - file_cache_disk_size = actual_usage; - } + file_cache_disk_size = actual_usage; if actual_usage != expected_usage { warn!( From 4c7fa12a2aa1967329c29aafe1341e01460ed032 Mon Sep 17 00:00:00 2001 From: khanova <32508607+khanova@users.noreply.github.com> Date: Thu, 2 Nov 2023 17:26:15 +0100 Subject: [PATCH 16/29] Proxy introduce allowed ips (#5729) ## Problem Proxy doesn't accept wake_compute responses with the allowed IPs. ## Summary of changes Extend wake_compute api to be able to return allowed_ips. --- proxy/src/console/messages.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/proxy/src/console/messages.rs b/proxy/src/console/messages.rs index 0d321c077a..e5f1615b14 100644 --- a/proxy/src/console/messages.rs +++ b/proxy/src/console/messages.rs @@ -13,6 +13,7 @@ pub struct ConsoleError { #[derive(Deserialize)] pub struct GetRoleSecret { pub role_secret: Box, + pub allowed_ips: Option>>, } // Manually implement debug to omit sensitive info. @@ -187,4 +188,31 @@ mod tests { Ok(()) } + + #[test] + fn parse_wake_compute() -> anyhow::Result<()> { + let json = json!({ + "address": "0.0.0.0", + "aux": dummy_aux(), + }); + let _: WakeCompute = serde_json::from_str(&json.to_string())?; + Ok(()) + } + + #[test] + fn parse_get_role_secret() -> anyhow::Result<()> { + // Empty `allowed_ips` field. + let json = json!({ + "role_secret": "secret", + }); + let _: GetRoleSecret = serde_json::from_str(&json.to_string())?; + // Empty `allowed_ips` field. + let json = json!({ + "role_secret": "secret", + "allowed_ips": ["8.8.8.8"], + }); + let _: GetRoleSecret = serde_json::from_str(&json.to_string())?; + + Ok(()) + } } From 27bdbf5e365bc77d5a7406e9fefa76e9a31ec12b Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 21:05:33 +0200 Subject: [PATCH 17/29] chore(layer): restore logging, doc changes (#5766) Some of the log messages were lost with the #4938. This PR adds some of them back, most notably: - starting to on-demand download - successful completion of on-demand download - ability to see when there were many waiters for the layer download - "unexpectedly on-demand downloading ..." is now `info!` Additionally some rare events are logged as error, which should never happen. --- libs/utils/src/sync/heavier_once_cell.rs | 41 ++++++++++++++++++-- pageserver/src/tenant/storage_layer/layer.rs | 40 ++++++++++++++----- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/libs/utils/src/sync/heavier_once_cell.rs b/libs/utils/src/sync/heavier_once_cell.rs index 8a5aaf2ceb..0ccaf4e716 100644 --- a/libs/utils/src/sync/heavier_once_cell.rs +++ b/libs/utils/src/sync/heavier_once_cell.rs @@ -1,4 +1,7 @@ -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, MutexGuard, +}; use tokio::sync::Semaphore; /// Custom design like [`tokio::sync::OnceCell`] but using [`OwnedSemaphorePermit`] instead of @@ -10,6 +13,7 @@ use tokio::sync::Semaphore; /// [`OwnedSemaphorePermit`]: tokio::sync::OwnedSemaphorePermit pub struct OnceCell { inner: Mutex>, + initializers: AtomicUsize, } impl Default for OnceCell { @@ -17,6 +21,7 @@ impl Default for OnceCell { fn default() -> Self { Self { inner: Default::default(), + initializers: AtomicUsize::new(0), } } } @@ -49,6 +54,7 @@ impl OnceCell { init_semaphore: Arc::new(sem), value: Some(value), }), + initializers: AtomicUsize::new(0), } } @@ -71,7 +77,11 @@ impl OnceCell { guard.init_semaphore.clone() }; - let permit = sem.acquire_owned().await; + let permit = { + // increment the count for the duration of queued + let _guard = CountWaitingInitializers::start(self); + sem.acquire_owned().await + }; match permit { Ok(permit) => { @@ -100,12 +110,13 @@ impl OnceCell { /// /// If the inner has already been initialized. pub fn set(&self, value: T, _permit: InitPermit) -> Guard<'_, T> { - // cannot assert that this permit is for self.inner.semaphore let guard = self.inner.lock().unwrap(); + // cannot assert that this permit is for self.inner.semaphore, but we can assert it cannot + // give more permits right now. if guard.init_semaphore.try_acquire().is_ok() { drop(guard); - panic!("semaphore is of wrong origin"); + panic!("permit is of wrong origin"); } Self::set0(value, guard) @@ -130,6 +141,28 @@ impl OnceCell { None } } + + /// Return the number of [`Self::get_or_init`] calls waiting for initialization to complete. + pub fn initializer_count(&self) -> usize { + self.initializers.load(Ordering::Relaxed) + } +} + +/// DropGuard counter for queued tasks waiting to initialize, mainly accessible for the +/// initializing task for example at the end of initialization. +struct CountWaitingInitializers<'a, T>(&'a OnceCell); + +impl<'a, T> CountWaitingInitializers<'a, T> { + fn start(target: &'a OnceCell) -> Self { + target.initializers.fetch_add(1, Ordering::Relaxed); + CountWaitingInitializers(target) + } +} + +impl<'a, T> Drop for CountWaitingInitializers<'a, T> { + fn drop(&mut self) { + self.0.initializers.fetch_sub(1, Ordering::Relaxed); + } } /// Uninteresting guard object to allow short-lived access to inspect or clone the held, diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index b320c02f9b..d72982a9a0 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -411,6 +411,10 @@ struct LayerInner { version: AtomicUsize, /// Allow subscribing to when the layer actually gets evicted. + /// + /// If in future we need to implement "wait until layer instances are gone and done", carrying + /// this over to the gc spawn_blocking from LayerInner::drop will do the trick, and adding a + /// method for "wait_gc" which will wait to this being closed. status: tokio::sync::broadcast::Sender, /// Counter for exponential backoff with the download @@ -561,6 +565,8 @@ impl LayerInner { } } + /// Cancellation safe, however dropping the future and calling this method again might result + /// in a new attempt to evict OR join the previously started attempt. pub(crate) async fn evict_and_wait( &self, _: &RemoteTimelineClient, @@ -609,8 +615,8 @@ impl LayerInner { } } - /// Should be cancellation safe, but cancellation is troublesome together with the spawned - /// download. + /// Cancellation safe. + #[tracing::instrument(skip_all, fields(layer=%self))] async fn get_or_maybe_download( self: &Arc, allow_download: bool, @@ -654,8 +660,6 @@ impl LayerInner { return Err(DownloadError::NoRemoteStorage); } - tracing::debug!(%reason, "downloading layer"); - if let Some(ctx) = ctx { self.check_expected_download(ctx)?; } @@ -666,6 +670,8 @@ impl LayerInner { return Err(DownloadError::DownloadRequired); } + tracing::info!(%reason, "downloading on-demand"); + self.spawn_download_and_wait(timeline, permit).await? } else { // the file is present locally, probably by a previous but cancelled call to @@ -686,6 +692,11 @@ impl LayerInner { LayerResidenceEventReason::ResidenceChange, ); + let waiters = self.inner.initializer_count(); + if waiters > 0 { + tracing::info!(waiters, "completing the on-demand download for other tasks"); + } + Ok((ResidentOrWantedEvicted::Resident(res), permit)) }; @@ -746,8 +757,8 @@ impl LayerInner { match b { Download => Ok(()), Warn | Error => { - tracing::warn!( - "unexpectedly on-demand downloading remote layer {self} for task kind {:?}", + tracing::info!( + "unexpectedly on-demand downloading for task kind {:?}", ctx.task_kind() ); crate::metrics::UNEXPECTED_ONDEMAND_DOWNLOADS.inc(); @@ -779,6 +790,7 @@ impl LayerInner { // block tenant::mgr::remove_tenant_from_memory. let this: Arc = self.clone(); + crate::task_mgr::spawn( &tokio::runtime::Handle::current(), crate::task_mgr::TaskKind::RemoteDownloadTask, @@ -787,6 +799,7 @@ impl LayerInner { &task_name, false, async move { + let client = timeline .remote_client .as_ref() @@ -848,6 +861,7 @@ impl LayerInner { } self.consecutive_failures.store(0, Ordering::Relaxed); + tracing::info!("on-demand download successful"); Ok(permit) } @@ -1040,11 +1054,14 @@ impl LayerInner { Ok(()) } Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - tracing::info!("failed to evict file from disk, it was already gone"); + tracing::error!( + layer_size = %self.desc.file_size, + "failed to evict layer from disk, it was already gone (metrics will be inaccurate)" + ); Err(EvictionCancelled::FileNotFound) } Err(e) => { - tracing::warn!("failed to evict file from disk: {e:#}"); + tracing::error!("failed to evict file from disk: {e:#}"); Err(EvictionCancelled::RemoveFailed) } }; @@ -1124,6 +1141,8 @@ impl std::fmt::Display for NeedsDownload { /// Existence of `DownloadedLayer` means that we have the file locally, and can later evict it. pub(crate) struct DownloadedLayer { owner: Weak, + // Use tokio OnceCell as we do not need to deinitialize this, it'll just get dropped with the + // DownloadedLayer kind: tokio::sync::OnceCell>, version: usize, } @@ -1167,7 +1186,6 @@ impl DownloadedLayer { "these are the same, just avoiding the upgrade" ); - // there is nothing async here, but it should be async let res = if owner.desc.is_delta { let summary = Some(delta_layer::Summary::expected( owner.desc.tenant_id, @@ -1266,6 +1284,8 @@ impl std::fmt::Debug for ResidentLayer { impl ResidentLayer { /// Release the eviction guard, converting back into a plain [`Layer`]. + /// + /// You can access the [`Layer`] also by using `as_ref`. pub(crate) fn drop_eviction_guard(self) -> Layer { self.into() } @@ -1321,7 +1341,7 @@ impl AsRef for ResidentLayer { } } -/// Allow slimming down if we don't want the `2*usize` with eviction candidates? +/// Drop the eviction guard. impl From for Layer { fn from(value: ResidentLayer) -> Self { value.owner From cdcaa329bfb9dd045a20b5012b3f4e97f591ce7c Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 3 Nov 2023 08:30:58 +0000 Subject: [PATCH 18/29] proxy: no more statements (#5747) ## Problem my prepared statements change in tokio-postgres landed in the latest release. it didn't work as we intended ## Summary of changes https://github.com/neondatabase/rust-postgres/pull/24 --- Cargo.lock | 10 ++-- Cargo.toml | 12 ++--- proxy/src/serverless/sql_over_http.rs | 68 ++++++++++++++------------- test_runner/regress/test_proxy.py | 44 +++++++++++++++++ 4 files changed, 90 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39c33304ae..4d9b20dae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3550,7 +3550,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "bytes", "fallible-iterator", @@ -3563,7 +3563,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "native-tls", "tokio", @@ -3574,7 +3574,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "base64 0.20.0", "byteorder", @@ -3592,7 +3592,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "bytes", "fallible-iterator", @@ -5396,7 +5396,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "async-trait", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index a0be7bb9ac..6f538941f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,11 +161,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -202,7 +202,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } ################# Binary contents sections diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 93df86cfc4..8f7cf7fbaf 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -470,30 +470,26 @@ async fn query_to_json( } .and_then(|s| s.parse::().ok()); - let fields = if !rows.is_empty() { - rows[0] - .columns() - .iter() - .map(|c| { - json!({ - "name": Value::String(c.name().to_owned()), - "dataTypeID": Value::Number(c.type_().oid().into()), - "tableID": c.table_oid(), - "columnID": c.column_id(), - "dataTypeSize": c.type_size(), - "dataTypeModifier": c.type_modifier(), - "format": "text", - }) - }) - .collect::>() - } else { - Vec::new() - }; + let mut fields = vec![]; + let mut columns = vec![]; + + for c in row_stream.columns() { + fields.push(json!({ + "name": Value::String(c.name().to_owned()), + "dataTypeID": Value::Number(c.type_().oid().into()), + "tableID": c.table_oid(), + "columnID": c.column_id(), + "dataTypeSize": c.type_size(), + "dataTypeModifier": c.type_modifier(), + "format": "text", + })); + columns.push(client.get_type(c.type_oid()).await?); + } // convert rows to JSON let rows = rows .iter() - .map(|row| pg_text_row_to_json(row, raw_output, array_mode)) + .map(|row| pg_text_row_to_json(row, &columns, raw_output, array_mode)) .collect::, _>>()?; // resulting JSON format is based on the format of node-postgres result @@ -514,22 +510,28 @@ async fn query_to_json( // pub fn pg_text_row_to_json( row: &Row, + columns: &[Type], raw_output: bool, array_mode: bool, ) -> Result { - let iter = row.columns().iter().enumerate().map(|(i, column)| { - let name = column.name(); - let pg_value = row.as_text(i)?; - let json_value = if raw_output { - match pg_value { - Some(v) => Value::String(v.to_string()), - None => Value::Null, - } - } else { - pg_text_to_json(pg_value, column.type_())? - }; - Ok((name.to_string(), json_value)) - }); + let iter = row + .columns() + .iter() + .zip(columns) + .enumerate() + .map(|(i, (column, typ))| { + let name = column.name(); + let pg_value = row.as_text(i)?; + let json_value = if raw_output { + match pg_value { + Some(v) => Value::String(v.to_string()), + None => Value::Null, + } + } else { + pg_text_to_json(pg_value, typ)? + }; + Ok((name.to_string(), json_value)) + }); if array_mode { // drop keys and aggregate into array diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index 6388e9f365..c93cdf637a 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -432,3 +432,47 @@ def test_sql_over_http_pool_idle(static_proxy: NeonProxy): query(200, "BEGIN") pid2 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] assert pid1 != pid2 + + +@pytest.mark.timeout(60) +def test_sql_over_http_pool_dos(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth with password 'http' superuser") + + static_proxy.safe_psql("CREATE TYPE foo AS ENUM ('foo')") + + def query(status: int, query: str) -> Any: + return static_proxy.http_query( + query, + [], + user="http_auth", + password="http", + expected_code=status, + ) + + # query generates a million rows - should hit the 10MB reponse limit quickly + response = query( + 400, + "select * from generate_series(1, 5000) a cross join generate_series(1, 5000) b cross join (select 'foo'::foo) c;", + ) + assert "response is too large (max is 10485760 bytes)" in response["message"] + + +def test_sql_over_http_pool_custom_types(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth with password 'http' superuser") + + static_proxy.safe_psql("CREATE TYPE foo AS ENUM ('foo','bar','baz')") + + def query(status: int, query: str) -> Any: + return static_proxy.http_query( + query, + [], + user="http_auth", + password="http", + expected_code=status, + ) + + response = query( + 200, + "select array['foo'::foo, 'bar'::foo, 'baz'::foo] as data", + ) + assert response["rows"][0]["data"] == ["foo", "bar", "baz"] From 5ceccdc7def3d4bdd33b32d5b10a8f4e148ad9af Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 3 Nov 2023 18:40:27 +0200 Subject: [PATCH 19/29] Logical replication startup fixes (#5750) ## Problem See https://neondb.slack.com/archives/C04DGM6SMTM/p1698226491736459 ## Summary of changes Update WAL affected buffers when restoring WAL from safekeeper ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik Co-authored-by: Arseny Sher --- pgxn/neon/walproposer_pg.c | 19 +++- .../regress/test_logical_replication.py | 89 +++++++++++++++++++ vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/revisions.json | 6 +- 6 files changed, 111 insertions(+), 9 deletions(-) diff --git a/pgxn/neon/walproposer_pg.c b/pgxn/neon/walproposer_pg.c index 865f91165b..f83a08d407 100644 --- a/pgxn/neon/walproposer_pg.c +++ b/pgxn/neon/walproposer_pg.c @@ -88,7 +88,7 @@ static void StartProposerReplication(WalProposer *wp, StartReplicationCmd *cmd); static void WalSndLoop(WalProposer *wp); static void XLogBroadcastWalProposer(WalProposer *wp); -static void XLogWalPropWrite(char *buf, Size nbytes, XLogRecPtr recptr); +static void XLogWalPropWrite(WalProposer *wp, char *buf, Size nbytes, XLogRecPtr recptr); static void XLogWalPropClose(XLogRecPtr recptr); static void @@ -1241,7 +1241,7 @@ WalProposerRecovery(Safekeeper *sk, TimeLineID timeline, XLogRecPtr startpos, XL rec_end_lsn = rec_start_lsn + len - XLOG_HDR_SIZE; /* write WAL to disk */ - XLogWalPropWrite(&buf[XLOG_HDR_SIZE], len - XLOG_HDR_SIZE, rec_start_lsn); + XLogWalPropWrite(sk->wp, &buf[XLOG_HDR_SIZE], len - XLOG_HDR_SIZE, rec_start_lsn); ereport(DEBUG1, (errmsg("Recover message %X/%X length %d", @@ -1283,11 +1283,24 @@ static XLogSegNo walpropSegNo = 0; * Write XLOG data to disk. */ static void -XLogWalPropWrite(char *buf, Size nbytes, XLogRecPtr recptr) +XLogWalPropWrite(WalProposer *wp, char *buf, Size nbytes, XLogRecPtr recptr) { int startoff; int byteswritten; + /* + * Apart from walproposer, basebackup LSN page is also written out by + * postgres itself which writes WAL only in pages, and in basebackup it is + * inherently dummy (only safekeepers have historic WAL). Update WAL buffers + * here to avoid dummy page overwriting correct one we download here. Ugly, + * but alternatives are about the same ugly. We won't need that if we switch + * to on-demand WAL download from safekeepers, without writing to disk. + * + * https://github.com/neondatabase/neon/issues/5749 + */ + if (!wp->config->syncSafekeepers) + XLogUpdateWalBuffers(buf, recptr, nbytes); + while (nbytes > 0) { int segbytes; diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index 726e5e5def..d2d8d71e3f 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -1,11 +1,14 @@ import time +import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnv, logical_replication_sync, wait_for_last_flush_lsn, ) +from fixtures.types import Lsn +from fixtures.utils import query_scalar def test_logical_replication(neon_simple_env: NeonEnv, vanilla_pg): @@ -147,3 +150,89 @@ COMMIT; endpoint.start() # it must be gone (but walproposer slot still exists, hence 1) assert endpoint.safe_psql("select count(*) from pg_replication_slots")[0][0] == 1 + + +# Test compute start at LSN page of which starts with contrecord +# https://github.com/neondatabase/neon/issues/5749 +def test_wal_page_boundary_start(neon_simple_env: NeonEnv, vanilla_pg): + env = neon_simple_env + + env.neon_cli.create_branch("init") + endpoint = env.endpoints.create_start("init") + tenant_id = endpoint.safe_psql("show neon.tenant_id")[0][0] + timeline_id = endpoint.safe_psql("show neon.timeline_id")[0][0] + + cur = endpoint.connect().cursor() + cur.execute("create table t(key int, value text)") + cur.execute("CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int);") + cur.execute("insert into replication_example values (1, 2)") + cur.execute("create publication pub1 for table replication_example") + + # now start subscriber + vanilla_pg.start() + vanilla_pg.safe_psql("create table t(pk integer primary key, value text)") + vanilla_pg.safe_psql("CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int);") + + log.info(f"ep connstr is {endpoint.connstr()}, subscriber connstr {vanilla_pg.connstr()}") + connstr = endpoint.connstr().replace("'", "''") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + logical_replication_sync(vanilla_pg, endpoint) + vanilla_pg.stop() + + with endpoint.cursor() as cur: + # measure how much space logical message takes. Sometimes first attempt + # creates huge message and then it stabilizes, have no idea why. + for _ in range(3): + lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"current_lsn={lsn_before}") + # Non-transactional logical message doesn't write WAL, only XLogInsert's + # it, so use transactional. Which is a bit problematic as transactional + # necessitates commit record. Alternatively we can do smth like + # select neon_xlogflush(pg_current_wal_insert_lsn()); + # but isn't much better + that particular call complains on 'xlog flush + # request 0/282C018 is not satisfied' as pg_current_wal_insert_lsn skips + # page headers. + payload = "blahblah" + cur.execute(f"select pg_logical_emit_message(true, 'pref', '{payload}')") + lsn_after_by_curr_wal_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + lsn_diff = lsn_after_by_curr_wal_lsn - lsn_before + logical_message_base = lsn_after_by_curr_wal_lsn - lsn_before - len(payload) + log.info( + f"before {lsn_before}, after {lsn_after_by_curr_wal_lsn}, lsn diff is {lsn_diff}, base {logical_message_base}" + ) + + # and write logical message spanning exactly as we want + lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"current_lsn={lsn_before}") + curr_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + offs = int(curr_lsn) % 8192 + till_page = 8192 - offs + payload_len = ( + till_page - logical_message_base - 8 + ) # not sure why 8 is here, it is deduced from experiments + log.info(f"current_lsn={curr_lsn}, offs {offs}, till_page {till_page}") + + # payload_len above would go exactly till the page boundary; but we want contrecord, so make it slightly longer + payload_len += 8 + + cur.execute(f"select pg_logical_emit_message(true, 'pref', 'f{'a' * payload_len}')") + supposedly_contrecord_end = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"supposedly_page_boundary={supposedly_contrecord_end}") + # The calculations to hit the page boundary are very fuzzy, so just + # ignore test if we fail to reach it. + if not (int(supposedly_contrecord_end) % 8192 == 32): + pytest.skip("missed page boundary, bad luck") + + cur.execute("insert into replication_example values (2, 3)") + + wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop().start() + + cur = endpoint.connect().cursor() + # this should flush current wal page + cur.execute("insert into replication_example values (3, 4)") + vanilla_pg.start() + logical_replication_sync(vanilla_pg, endpoint) + assert vanilla_pg.safe_psql( + "select sum(somedata) from replication_example" + ) == endpoint.safe_psql("select sum(somedata) from replication_example") diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 6669a672ee..dd067cf656 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 6669a672ee14ab2c09d44c4552f9a13fad3afc10 +Subproject commit dd067cf656f6810a25aca6025633d32d02c5085a diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index ab67ab9635..bc88f53931 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit ab67ab96355d61e9d0218630be4aa7db53bf83e7 +Subproject commit bc88f539312fcc4bb292ce94ae9db09ab6656e8a diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 550ffa6495..763000f1d0 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 550ffa6495a5dc62fccc3a8b449386633758680b +Subproject commit 763000f1d0873b827829c41f2f6f799ffc0de55c diff --git a/vendor/revisions.json b/vendor/revisions.json index 012fb14035..377357e131 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "postgres-v16": "550ffa6495a5dc62fccc3a8b449386633758680b", - "postgres-v15": "ab67ab96355d61e9d0218630be4aa7db53bf83e7", - "postgres-v14": "6669a672ee14ab2c09d44c4552f9a13fad3afc10" + "postgres-v16": "763000f1d0873b827829c41f2f6f799ffc0de55c", + "postgres-v15": "bc88f539312fcc4bb292ce94ae9db09ab6656e8a", + "postgres-v14": "dd067cf656f6810a25aca6025633d32d02c5085a" } From 306c4f99678d5d256c9ad237dd502e9dbe4bb63e Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 3 Nov 2023 17:36:02 +0000 Subject: [PATCH 20/29] s3_scrubber: prepare for scrubbing buckets with generation-aware content (#5700) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The scrubber didn't know how to find the latest index_part when generations were in use. ## Summary of changes - Teach the scrubber to do the same dance that pageserver does when finding the latest index_part.json - Teach the scrubber how to understand layer files with generation suffixes. - General improvement to testability: scan_metadata has a machine readable output that the testing `S3Scrubber` wrapper can read. - Existing test coverage of scrubber was false-passing because it just didn't see any data due to prefixing of data in the bucket. Fix that. This is incremental improvement: the more confidence we can have in the scrubber, the more we can use it in integration tests to validate the state of remote storage. --------- Co-authored-by: Arpad Müller --- Cargo.lock | 1 + libs/utils/src/generation.rs | 2 +- .../src/tenant/remote_timeline_client.rs | 2 +- .../tenant/remote_timeline_client/index.rs | 2 +- s3_scrubber/Cargo.toml | 1 + s3_scrubber/src/checks.rs | 173 ++++++++++++------ s3_scrubber/src/lib.rs | 39 ++-- s3_scrubber/src/main.rs | 13 +- s3_scrubber/src/metadata_stream.rs | 4 +- s3_scrubber/src/scan_metadata.rs | 5 + test_runner/fixtures/neon_fixtures.py | 31 +++- test_runner/fixtures/utils.py | 3 +- .../regress/test_pageserver_generations.py | 15 ++ 13 files changed, 209 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d9b20dae4..3e9a7198a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4419,6 +4419,7 @@ dependencies = [ "itertools", "pageserver", "rand 0.8.5", + "remote_storage", "reqwest", "serde", "serde_json", diff --git a/libs/utils/src/generation.rs b/libs/utils/src/generation.rs index 88d50905c6..49e290dab8 100644 --- a/libs/utils/src/generation.rs +++ b/libs/utils/src/generation.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; /// /// See docs/rfcs/025-generation-numbers.md for detail on how generation /// numbers are used. -#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)] pub enum Generation { // Generations with this magic value will not add a suffix to S3 keys, and will not // be included in persisted index_part.json. This value is only to be used diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 76e75253fb..bbf6a0c5c5 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1542,7 +1542,7 @@ pub fn remote_index_path( } /// Given the key of an index, parse out the generation part of the name -pub(crate) fn parse_remote_index_path(path: RemotePath) -> Option { +pub fn parse_remote_index_path(path: RemotePath) -> Option { let file_name = match path.get_path().file_name() { Some(f) => f, None => { diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 7cf963ca9d..fdab74a8be 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -155,7 +155,7 @@ pub struct IndexLayerMetadata { #[serde(default = "Generation::none")] #[serde(skip_serializing_if = "Generation::is_none")] - pub(super) generation: Generation, + pub generation: Generation, } impl From for IndexLayerMetadata { diff --git a/s3_scrubber/Cargo.toml b/s3_scrubber/Cargo.toml index f3ea6e222c..0f3e5630e8 100644 --- a/s3_scrubber/Cargo.toml +++ b/s3_scrubber/Cargo.toml @@ -33,6 +33,7 @@ reqwest = { workspace = true, default-features = false, features = ["rustls-tls" aws-config = { workspace = true, default-features = false, features = ["rustls", "credentials-sso"] } pageserver = { path = "../pageserver" } +remote_storage = { path = "../libs/remote_storage" } tracing.workspace = true tracing-subscriber.workspace = true diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs index c11f1f9779..64702fca3d 100644 --- a/s3_scrubber/src/checks.rs +++ b/s3_scrubber/src/checks.rs @@ -1,13 +1,18 @@ use std::collections::HashSet; use anyhow::Context; -use aws_sdk_s3::Client; +use aws_sdk_s3::{types::ObjectIdentifier, Client}; use tracing::{error, info, warn}; +use utils::generation::Generation; use crate::cloud_admin_api::BranchData; -use crate::{download_object_with_retries, list_objects_with_retries, RootTarget}; +use crate::metadata_stream::stream_listing; +use crate::{download_object_with_retries, RootTarget}; +use futures_util::{pin_mut, StreamExt}; +use pageserver::tenant::remote_timeline_client::parse_remote_index_path; use pageserver::tenant::storage_layer::LayerFileName; use pageserver::tenant::IndexPart; +use remote_storage::RemotePath; use utils::id::TenantTimelineId; pub(crate) struct TimelineAnalysis { @@ -68,6 +73,7 @@ pub(crate) async fn branch_cleanup_and_check_errors( match s3_data.blob_data { BlobDataParseResult::Parsed { index_part, + index_part_generation, mut s3_layers, } => { if !IndexPart::KNOWN_VERSIONS.contains(&index_part.get_version()) { @@ -107,33 +113,62 @@ pub(crate) async fn branch_cleanup_and_check_errors( )) } - if !s3_layers.remove(&layer) { + let layer_map_key = (layer, metadata.generation); + if !s3_layers.remove(&layer_map_key) { + // FIXME: this will emit false positives if an index was + // uploaded concurrently with our scan. To make this check + // correct, we need to try sending a HEAD request for the + // layer we think is missing. result.errors.push(format!( - "index_part.json contains a layer {} that is not present in S3", - layer.file_name(), + "index_part.json contains a layer {}{} that is not present in remote storage", + layer_map_key.0.file_name(), + layer_map_key.1.get_suffix() )) } } - if !s3_layers.is_empty() { + let orphan_layers: Vec<(LayerFileName, Generation)> = s3_layers + .into_iter() + .filter(|(_layer_name, gen)| + // A layer is only considered orphaned if it has a generation below + // the index. If the generation is >= the index, then the layer may + // be an upload from a running pageserver, or even an upload from + // a new generation that didn't upload an index yet. + // + // Even so, a layer that is not referenced by the index could just + // be something enqueued for deletion, so while this check is valid + // for indicating that a layer is garbage, it is not an indicator + // of a problem. + gen < &index_part_generation) + .collect(); + + if !orphan_layers.is_empty() { result.errors.push(format!( "index_part.json does not contain layers from S3: {:?}", - s3_layers + orphan_layers .iter() - .map(|layer_name| layer_name.file_name()) + .map(|(layer_name, gen)| format!( + "{}{}", + layer_name.file_name(), + gen.get_suffix() + )) .collect::>(), )); - result - .garbage_keys - .extend(s3_layers.iter().map(|layer_name| { + result.garbage_keys.extend(orphan_layers.iter().map( + |(layer_name, layer_gen)| { let mut key = s3_root.timeline_root(id).prefix_in_bucket; let delimiter = s3_root.delimiter(); if !key.ends_with(delimiter) { key.push_str(delimiter); } - key.push_str(&layer_name.file_name()); + key.push_str(&format!( + "{}{}", + &layer_name.file_name(), + layer_gen.get_suffix() + )); key - })); + }, + )); } } BlobDataParseResult::Incorrect(parse_errors) => result.errors.extend( @@ -178,70 +213,97 @@ pub(crate) struct S3TimelineBlobData { pub(crate) enum BlobDataParseResult { Parsed { index_part: IndexPart, - s3_layers: HashSet, + index_part_generation: Generation, + s3_layers: HashSet<(LayerFileName, Generation)>, }, Incorrect(Vec), } +fn parse_layer_object_name(name: &str) -> Result<(LayerFileName, Generation), String> { + match name.rsplit_once('-') { + // FIXME: this is gross, just use a regex? + Some((layer_filename, gen)) if gen.len() == 8 => { + let layer = layer_filename.parse::()?; + let gen = + Generation::parse_suffix(gen).ok_or("Malformed generation suffix".to_string())?; + Ok((layer, gen)) + } + _ => Ok((name.parse::()?, Generation::none())), + } +} + pub(crate) async fn list_timeline_blobs( s3_client: &Client, id: TenantTimelineId, s3_root: &RootTarget, ) -> anyhow::Result { let mut s3_layers = HashSet::new(); - let mut index_part_object = None; - - let timeline_dir_target = s3_root.timeline_root(&id); - let mut continuation_token = None; let mut errors = Vec::new(); let mut keys_to_remove = Vec::new(); - loop { - let fetch_response = - list_objects_with_retries(s3_client, &timeline_dir_target, continuation_token.clone()) - .await?; + let mut timeline_dir_target = s3_root.timeline_root(&id); + timeline_dir_target.delimiter = String::new(); - let subdirectories = fetch_response.common_prefixes().unwrap_or_default(); - if !subdirectories.is_empty() { - errors.push(format!( - "S3 list response should not contain any subdirectories, but got {subdirectories:?}" - )); - } + let mut index_parts: Vec = Vec::new(); - for (object, key) in fetch_response - .contents() - .unwrap_or_default() - .iter() - .filter_map(|object| Some((object, object.key()?))) - { - let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); - match blob_name { - Some("index_part.json") => index_part_object = Some(object.clone()), - Some(maybe_layer_name) => match maybe_layer_name.parse::() { - Ok(new_layer) => { - s3_layers.insert(new_layer); - } - Err(e) => { - errors.push( - format!("S3 list response got an object with key {key} that is not a layer name: {e}"), - ); - keys_to_remove.push(key.to_string()); - } - }, - None => { - errors.push(format!("S3 list response got an object with odd key {key}")); + let stream = stream_listing(s3_client, &timeline_dir_target); + pin_mut!(stream); + while let Some(obj) = stream.next().await { + let obj = obj?; + let key = match obj.key() { + Some(k) => k, + None => continue, + }; + + let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); + match blob_name { + Some(name) if name.starts_with("index_part.json") => { + tracing::info!("Index key {key}"); + index_parts.push(obj) + } + Some(maybe_layer_name) => match parse_layer_object_name(maybe_layer_name) { + Ok((new_layer, gen)) => { + tracing::info!("Parsed layer key: {} {:?}", new_layer, gen); + s3_layers.insert((new_layer, gen)); + } + Err(e) => { + tracing::info!("Error parsing key {maybe_layer_name}"); + errors.push( + format!("S3 list response got an object with key {key} that is not a layer name: {e}"), + ); keys_to_remove.push(key.to_string()); } + }, + None => { + tracing::info!("Peculiar key {}", key); + errors.push(format!("S3 list response got an object with odd key {key}")); + keys_to_remove.push(key.to_string()); } } - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } } + // Choose the index_part with the highest generation + let (index_part_object, index_part_generation) = match index_parts + .iter() + .filter_map(|k| { + let key = k.key().unwrap(); + // Stripping the index key to the last part, because RemotePath doesn't + // like absolute paths, and depending on prefix_in_bucket it's possible + // for the keys we read back to start with a slash. + let basename = key.rsplit_once('/').unwrap().1; + parse_remote_index_path(RemotePath::from_string(basename).unwrap()).map(|g| (k, g)) + }) + .max_by_key(|i| i.1) + .map(|(k, g)| (k.clone(), g)) + { + Some((key, gen)) => (Some(key), gen), + None => { + // Legacy/missing case: one or zero index parts, which did not have a generation + (index_parts.pop(), Generation::none()) + } + }; + if index_part_object.is_none() { errors.push("S3 list response got no index_part.json file".to_string()); } @@ -261,6 +323,7 @@ pub(crate) async fn list_timeline_blobs( return Ok(S3TimelineBlobData { blob_data: BlobDataParseResult::Parsed { index_part, + index_part_generation, s3_layers, }, keys_to_remove, diff --git a/s3_scrubber/src/lib.rs b/s3_scrubber/src/lib.rs index 98e7ed5334..d892438633 100644 --- a/s3_scrubber/src/lib.rs +++ b/s3_scrubber/src/lib.rs @@ -34,6 +34,9 @@ const CLOUD_ADMIN_API_TOKEN_ENV_VAR: &str = "CLOUD_ADMIN_API_TOKEN"; #[derive(Debug, Clone)] pub struct S3Target { pub bucket_name: String, + /// This `prefix_in_bucket` is only equal to the PS/SK config of the same + /// name for the RootTarget: other instances of S3Target will have prefix_in_bucket + /// with extra parts. pub prefix_in_bucket: String, pub delimiter: String, } @@ -77,9 +80,13 @@ impl Display for NodeKind { impl S3Target { pub fn with_sub_segment(&self, new_segment: &str) -> Self { let mut new_self = self.clone(); - let _ = new_self.prefix_in_bucket.pop(); - new_self.prefix_in_bucket = - [&new_self.prefix_in_bucket, new_segment, ""].join(&new_self.delimiter); + if new_self.prefix_in_bucket.is_empty() { + new_self.prefix_in_bucket = format!("/{}/", new_segment); + } else { + let _ = new_self.prefix_in_bucket.pop(); + new_self.prefix_in_bucket = + [&new_self.prefix_in_bucket, new_segment, ""].join(&new_self.delimiter); + } new_self } } @@ -91,10 +98,10 @@ pub enum RootTarget { } impl RootTarget { - pub fn tenants_root(&self) -> &S3Target { + pub fn tenants_root(&self) -> S3Target { match self { - Self::Pageserver(root) => root, - Self::Safekeeper(root) => root, + Self::Pageserver(root) => root.with_sub_segment(TENANTS_SEGMENT_NAME), + Self::Safekeeper(root) => root.with_sub_segment("wal"), } } @@ -133,6 +140,7 @@ impl RootTarget { pub struct BucketConfig { pub region: String, pub bucket: String, + pub prefix_in_bucket: Option, /// Use SSO if this is set, else rely on AWS_* environment vars pub sso_account_id: Option, @@ -155,10 +163,12 @@ impl BucketConfig { let sso_account_id = env::var("SSO_ACCOUNT_ID").ok(); let region = env::var("REGION").context("'REGION' param retrieval")?; let bucket = env::var("BUCKET").context("'BUCKET' param retrieval")?; + let prefix_in_bucket = env::var("BUCKET_PREFIX").ok(); Ok(Self { region, bucket, + prefix_in_bucket, sso_account_id, }) } @@ -191,14 +201,14 @@ pub fn init_logging(file_name: &str) -> WorkerGuard { .with_target(false) .with_ansi(false) .with_writer(file_writer); - let stdout_logs = fmt::Layer::new() - .with_ansi(std::io::stdout().is_terminal()) + let stderr_logs = fmt::Layer::new() + .with_ansi(std::io::stderr().is_terminal()) .with_target(false) - .with_writer(std::io::stdout); + .with_writer(std::io::stderr); tracing_subscriber::registry() .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) .with(file_logs) - .with(stdout_logs) + .with(stderr_logs) .init(); guard @@ -250,15 +260,20 @@ fn init_remote( let bucket_region = Region::new(bucket_config.region); let delimiter = "/".to_string(); let s3_client = Arc::new(init_s3_client(bucket_config.sso_account_id, bucket_region)); + let s3_root = match node_kind { NodeKind::Pageserver => RootTarget::Pageserver(S3Target { bucket_name: bucket_config.bucket, - prefix_in_bucket: ["pageserver", "v1", TENANTS_SEGMENT_NAME, ""].join(&delimiter), + prefix_in_bucket: bucket_config + .prefix_in_bucket + .unwrap_or("pageserver/v1".to_string()), delimiter, }), NodeKind::Safekeeper => RootTarget::Safekeeper(S3Target { bucket_name: bucket_config.bucket, - prefix_in_bucket: ["safekeeper", "v1", "wal", ""].join(&delimiter), + prefix_in_bucket: bucket_config + .prefix_in_bucket + .unwrap_or("safekeeper/v1".to_string()), delimiter, }), }; diff --git a/s3_scrubber/src/main.rs b/s3_scrubber/src/main.rs index 9a0d6c9ae8..1f0ceebdaf 100644 --- a/s3_scrubber/src/main.rs +++ b/s3_scrubber/src/main.rs @@ -31,7 +31,10 @@ enum Command { #[arg(short, long, default_value_t = PurgeMode::DeletedOnly)] mode: PurgeMode, }, - ScanMetadata {}, + ScanMetadata { + #[arg(short, long, default_value_t = false)] + json: bool, + }, } #[tokio::main] @@ -54,13 +57,17 @@ async fn main() -> anyhow::Result<()> { )); match cli.command { - Command::ScanMetadata {} => match scan_metadata(bucket_config).await { + Command::ScanMetadata { json } => match scan_metadata(bucket_config).await { Err(e) => { tracing::error!("Failed: {e}"); Err(e) } Ok(summary) => { - println!("{}", summary.summary_string()); + if json { + println!("{}", serde_json::to_string(&summary).unwrap()) + } else { + println!("{}", summary.summary_string()); + } if summary.is_fatal() { Err(anyhow::anyhow!("Fatal scrub errors detected")) } else { diff --git a/s3_scrubber/src/metadata_stream.rs b/s3_scrubber/src/metadata_stream.rs index 125a370e90..8095071c1f 100644 --- a/s3_scrubber/src/metadata_stream.rs +++ b/s3_scrubber/src/metadata_stream.rs @@ -13,10 +13,10 @@ pub fn stream_tenants<'a>( ) -> impl Stream> + 'a { try_stream! { let mut continuation_token = None; + let tenants_target = target.tenants_root(); loop { - let tenants_target = target.tenants_root(); let fetch_response = - list_objects_with_retries(s3_client, tenants_target, continuation_token.clone()).await?; + list_objects_with_retries(s3_client, &tenants_target, continuation_token.clone()).await?; let new_entry_ids = fetch_response .common_prefixes() diff --git a/s3_scrubber/src/scan_metadata.rs b/s3_scrubber/src/scan_metadata.rs index 33acd41a5b..ad82db1e76 100644 --- a/s3_scrubber/src/scan_metadata.rs +++ b/s3_scrubber/src/scan_metadata.rs @@ -10,8 +10,10 @@ use aws_sdk_s3::Client; use futures_util::{pin_mut, StreamExt, TryStreamExt}; use histogram::Histogram; use pageserver::tenant::IndexPart; +use serde::Serialize; use utils::id::TenantTimelineId; +#[derive(Serialize)] pub struct MetadataSummary { count: usize, with_errors: HashSet, @@ -25,7 +27,9 @@ pub struct MetadataSummary { } /// A histogram plus minimum and maximum tracking +#[derive(Serialize)] struct MinMaxHisto { + #[serde(skip)] histo: Histogram, min: u64, max: u64, @@ -109,6 +113,7 @@ impl MetadataSummary { self.count += 1; if let BlobDataParseResult::Parsed { index_part, + index_part_generation: _, s3_layers: _, } = &data.blob_data { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 81a7b0750d..02faf715da 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2968,24 +2968,33 @@ class S3Scrubber: self.env = env self.log_dir = log_dir - def scrubber_cli(self, args, timeout): + def scrubber_cli(self, args: list[str], timeout) -> str: assert isinstance(self.env.pageserver_remote_storage, S3Storage) s3_storage = self.env.pageserver_remote_storage env = { "REGION": s3_storage.bucket_region, "BUCKET": s3_storage.bucket_name, + "BUCKET_PREFIX": s3_storage.prefix_in_bucket, + "RUST_LOG": "DEBUG", } env.update(s3_storage.access_env_vars()) if s3_storage.endpoint is not None: env.update({"AWS_ENDPOINT_URL": s3_storage.endpoint}) - base_args = [self.env.neon_binpath / "s3_scrubber"] + base_args = [str(self.env.neon_binpath / "s3_scrubber")] args = base_args + args - (output_path, _, status_code) = subprocess_capture( - self.log_dir, args, echo_stderr=True, echo_stdout=True, env=env, check=False + (output_path, stdout, status_code) = subprocess_capture( + self.log_dir, + args, + echo_stderr=True, + echo_stdout=True, + env=env, + check=False, + capture_stdout=True, + timeout=timeout, ) if status_code: log.warning(f"Scrub command {args} failed") @@ -2994,8 +3003,18 @@ class S3Scrubber: raise RuntimeError("Remote storage scrub failed") - def scan_metadata(self): - self.scrubber_cli(["scan-metadata"], timeout=30) + assert stdout is not None + return stdout + + def scan_metadata(self) -> Any: + stdout = self.scrubber_cli(["scan-metadata", "--json"], timeout=30) + + try: + return json.loads(stdout) + except: + log.error("Failed to decode JSON output from `scan-metadata`. Dumping stdout:") + log.error(stdout) + raise def get_test_output_dir(request: FixtureRequest, top_output_dir: Path) -> Path: diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index e54b82dfb4..ba8d70d5a9 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -35,6 +35,7 @@ def subprocess_capture( echo_stderr=False, echo_stdout=False, capture_stdout=False, + timeout=None, **kwargs: Any, ) -> Tuple[str, Optional[str], int]: """Run a process and bifurcate its output to files and the `log` logger @@ -104,7 +105,7 @@ def subprocess_capture( stderr_handler = OutputHandler(p.stderr, stderr_f, echo=echo_stderr, capture=False) stderr_handler.start() - r = p.wait() + r = p.wait(timeout=timeout) stdout_handler.join() stderr_handler.join() diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 78ef6b9165..3e5021ae06 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -21,6 +21,7 @@ from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, PgBin, + S3Scrubber, last_flush_lsn_upload, wait_for_last_flush_lsn, ) @@ -234,8 +235,22 @@ def test_generations_upgrade(neon_env_builder: NeonEnvBuilder): assert len(suffixed_objects) > 0 assert len(legacy_objects) > 0 + # Flush through deletions to get a clean state for scrub: we are implicitly validating + # that our generations-enabled pageserver was able to do deletions of layers + # from earlier which don't have a generation. + env.pageserver.http_client().deletion_queue_flush(execute=True) + assert get_deletion_queue_unexpected_errors(env.pageserver.http_client()) == 0 + # Having written a mixture of generation-aware and legacy index_part.json, + # ensure the scrubber handles the situation as expected. + metadata_summary = S3Scrubber( + neon_env_builder.test_output_dir, neon_env_builder + ).scan_metadata() + assert metadata_summary["count"] == 1 # Scrubber should have seen our timeline + assert not metadata_summary["with_errors"] + assert not metadata_summary["with_warnings"] + def test_deferred_deletion(neon_env_builder: NeonEnvBuilder): neon_env_builder.enable_generations = True From 09b5954526d5192fc54f181c719b129f8db08249 Mon Sep 17 00:00:00 2001 From: duguorong009 <80258679+duguorong009@users.noreply.github.com> Date: Sun, 5 Nov 2023 05:16:54 -0500 Subject: [PATCH 21/29] refactor: use streaming in safekeeper `/v1/debug_dump` http response (#5731) - Update the handler for `/v1/debug_dump` http response in safekeeper - Update the `debug_dump::build()` to use the streaming in JSON build process --- Cargo.lock | 1 + libs/utils/src/http/endpoint.rs | 170 +++++++++++++------------- safekeeper/Cargo.toml | 1 + safekeeper/src/debug_dump.rs | 111 +++++++++++------ safekeeper/src/http/routes.rs | 55 ++++++++- safekeeper/src/pull_timeline.rs | 13 +- test_runner/fixtures/neon_fixtures.py | 2 +- 7 files changed, 227 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e9a7198a9..2203ad462c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4478,6 +4478,7 @@ dependencies = [ "tokio", "tokio-io-timeout", "tokio-postgres", + "tokio-stream", "toml_edit", "tracing", "url", diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index f3f5e95d0b..a9ee2425a4 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -14,6 +14,11 @@ use tracing::{self, debug, info, info_span, warn, Instrument}; use std::future::Future; use std::str::FromStr; +use bytes::{Bytes, BytesMut}; +use std::io::Write as _; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + static SERVE_METRICS_COUNT: Lazy = Lazy::new(|| { register_int_counter!( "libmetrics_metric_handler_requests_total", @@ -146,94 +151,89 @@ impl Drop for RequestCancelled { } } +/// An [`std::io::Write`] implementation on top of a channel sending [`bytes::Bytes`] chunks. +pub struct ChannelWriter { + buffer: BytesMut, + pub tx: mpsc::Sender>, + written: usize, +} + +impl ChannelWriter { + pub fn new(buf_len: usize, tx: mpsc::Sender>) -> Self { + assert_ne!(buf_len, 0); + ChannelWriter { + // split about half off the buffer from the start, because we flush depending on + // capacity. first flush will come sooner than without this, but now resizes will + // have better chance of picking up the "other" half. not guaranteed of course. + buffer: BytesMut::with_capacity(buf_len).split_off(buf_len / 2), + tx, + written: 0, + } + } + + pub fn flush0(&mut self) -> std::io::Result { + let n = self.buffer.len(); + if n == 0 { + return Ok(0); + } + + tracing::trace!(n, "flushing"); + let ready = self.buffer.split().freeze(); + + // not ideal to call from blocking code to block_on, but we are sure that this + // operation does not spawn_blocking other tasks + let res: Result<(), ()> = tokio::runtime::Handle::current().block_on(async { + self.tx.send(Ok(ready)).await.map_err(|_| ())?; + + // throttle sending to allow reuse of our buffer in `write`. + self.tx.reserve().await.map_err(|_| ())?; + + // now the response task has picked up the buffer and hopefully started + // sending it to the client. + Ok(()) + }); + if res.is_err() { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + self.written += n; + Ok(n) + } + + pub fn flushed_bytes(&self) -> usize { + self.written + } +} + +impl std::io::Write for ChannelWriter { + fn write(&mut self, mut buf: &[u8]) -> std::io::Result { + let remaining = self.buffer.capacity() - self.buffer.len(); + + let out_of_space = remaining < buf.len(); + + let original_len = buf.len(); + + if out_of_space { + let can_still_fit = buf.len() - remaining; + self.buffer.extend_from_slice(&buf[..can_still_fit]); + buf = &buf[can_still_fit..]; + self.flush0()?; + } + + // assume that this will often under normal operation just move the pointer back to the + // beginning of allocation, because previous split off parts are already sent and + // dropped. + self.buffer.extend_from_slice(buf); + Ok(original_len) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.flush0().map(|_| ()) + } +} + async fn prometheus_metrics_handler(_req: Request) -> Result, ApiError> { - use bytes::{Bytes, BytesMut}; - use std::io::Write as _; - use tokio::sync::mpsc; - use tokio_stream::wrappers::ReceiverStream; - SERVE_METRICS_COUNT.inc(); - /// An [`std::io::Write`] implementation on top of a channel sending [`bytes::Bytes`] chunks. - struct ChannelWriter { - buffer: BytesMut, - tx: mpsc::Sender>, - written: usize, - } - - impl ChannelWriter { - fn new(buf_len: usize, tx: mpsc::Sender>) -> Self { - assert_ne!(buf_len, 0); - ChannelWriter { - // split about half off the buffer from the start, because we flush depending on - // capacity. first flush will come sooner than without this, but now resizes will - // have better chance of picking up the "other" half. not guaranteed of course. - buffer: BytesMut::with_capacity(buf_len).split_off(buf_len / 2), - tx, - written: 0, - } - } - - fn flush0(&mut self) -> std::io::Result { - let n = self.buffer.len(); - if n == 0 { - return Ok(0); - } - - tracing::trace!(n, "flushing"); - let ready = self.buffer.split().freeze(); - - // not ideal to call from blocking code to block_on, but we are sure that this - // operation does not spawn_blocking other tasks - let res: Result<(), ()> = tokio::runtime::Handle::current().block_on(async { - self.tx.send(Ok(ready)).await.map_err(|_| ())?; - - // throttle sending to allow reuse of our buffer in `write`. - self.tx.reserve().await.map_err(|_| ())?; - - // now the response task has picked up the buffer and hopefully started - // sending it to the client. - Ok(()) - }); - if res.is_err() { - return Err(std::io::ErrorKind::BrokenPipe.into()); - } - self.written += n; - Ok(n) - } - - fn flushed_bytes(&self) -> usize { - self.written - } - } - - impl std::io::Write for ChannelWriter { - fn write(&mut self, mut buf: &[u8]) -> std::io::Result { - let remaining = self.buffer.capacity() - self.buffer.len(); - - let out_of_space = remaining < buf.len(); - - let original_len = buf.len(); - - if out_of_space { - let can_still_fit = buf.len() - remaining; - self.buffer.extend_from_slice(&buf[..can_still_fit]); - buf = &buf[can_still_fit..]; - self.flush0()?; - } - - // assume that this will often under normal operation just move the pointer back to the - // beginning of allocation, because previous split off parts are already sent and - // dropped. - self.buffer.extend_from_slice(buf); - Ok(original_len) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.flush0().map(|_| ()) - } - } - let started_at = std::time::Instant::now(); let (tx, rx) = mpsc::channel(1); diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index 64ef9f6997..30516c0763 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -47,6 +47,7 @@ pq_proto.workspace = true remote_storage.workspace = true safekeeper_api.workspace = true storage_broker.workspace = true +tokio-stream.workspace = true utils.workspace = true workspace_hack.workspace = true diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index ee9d7118c6..8cbc0aa47f 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -5,6 +5,7 @@ use std::fs::DirEntry; use std::io::BufReader; use std::io::Read; use std::path::PathBuf; +use std::sync::Arc; use anyhow::Result; use camino::Utf8Path; @@ -28,7 +29,7 @@ use crate::send_wal::WalSenderState; use crate::GlobalTimelines; /// Various filters that influence the resulting JSON output. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct Args { /// Dump all available safekeeper state. False by default. pub dump_all: bool, @@ -53,15 +54,76 @@ pub struct Args { } /// Response for debug dump request. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize)] pub struct Response { pub start_time: DateTime, pub finish_time: DateTime, - pub timelines: Vec, + pub timelines: Vec, pub timelines_count: usize, pub config: Config, } +pub struct TimelineDumpSer { + pub tli: Arc, + pub args: Args, + pub runtime: Arc, +} + +impl std::fmt::Debug for TimelineDumpSer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TimelineDumpSer") + .field("tli", &self.tli.ttid) + .field("args", &self.args) + .finish() + } +} + +impl Serialize for TimelineDumpSer { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let dump = self + .runtime + .block_on(build_from_tli_dump(self.tli.clone(), self.args.clone())); + dump.serialize(serializer) + } +} + +async fn build_from_tli_dump(timeline: Arc, args: Args) -> Timeline { + let control_file = if args.dump_control_file { + let mut state = timeline.get_state().await.1; + if !args.dump_term_history { + state.acceptor_state.term_history = TermHistory(vec![]); + } + Some(state) + } else { + None + }; + + let memory = if args.dump_memory { + Some(timeline.memory_dump().await) + } else { + None + }; + + let disk_content = if args.dump_disk_content { + // build_disk_content can fail, but we don't want to fail the whole + // request because of that. + build_disk_content(&timeline.timeline_dir).ok() + } else { + None + }; + + Timeline { + tenant_id: timeline.ttid.tenant_id, + timeline_id: timeline.ttid.timeline_id, + control_file, + memory, + disk_content, + } +} + /// Safekeeper configuration. #[derive(Debug, Serialize, Deserialize)] pub struct Config { @@ -140,8 +202,12 @@ pub async fn build(args: Args) -> Result { GlobalTimelines::get_all() }; - // TODO: return Stream instead of Vec let mut timelines = Vec::new(); + let runtime = Arc::new( + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(), + ); for tli in ptrs_snapshot { let ttid = tli.ttid; if let Some(tenant_id) = args.tenant_id { @@ -155,38 +221,11 @@ pub async fn build(args: Args) -> Result { } } - let control_file = if args.dump_control_file { - let mut state = tli.get_state().await.1; - if !args.dump_term_history { - state.acceptor_state.term_history = TermHistory(vec![]); - } - Some(state) - } else { - None - }; - - let memory = if args.dump_memory { - Some(tli.memory_dump().await) - } else { - None - }; - - let disk_content = if args.dump_disk_content { - // build_disk_content can fail, but we don't want to fail the whole - // request because of that. - build_disk_content(&tli.timeline_dir).ok() - } else { - None - }; - - let timeline = Timeline { - tenant_id: ttid.tenant_id, - timeline_id: ttid.timeline_id, - control_file, - memory, - disk_content, - }; - timelines.push(timeline); + timelines.push(TimelineDumpSer { + tli, + args: args.clone(), + runtime: runtime.clone(), + }); } let config = GlobalTimelines::get_global_config(); diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 940ac82df6..474d636441 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -13,7 +13,12 @@ use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use tokio::fs::File; use tokio::io::AsyncReadExt; -use utils::http::endpoint::request_span; + +use std::io::Write as _; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tracing::info_span; +use utils::http::endpoint::{request_span, ChannelWriter}; use crate::receive_wal::WalReceiverState; use crate::safekeeper::Term; @@ -373,8 +378,52 @@ async fn dump_debug_handler(mut request: Request) -> Result .await .map_err(ApiError::InternalServerError)?; - // TODO: use streaming response - json_response(StatusCode::OK, resp) + let started_at = std::time::Instant::now(); + + let (tx, rx) = mpsc::channel(1); + + let body = Body::wrap_stream(ReceiverStream::new(rx)); + + let mut writer = ChannelWriter::new(128 * 1024, tx); + + let response = Response::builder() + .status(200) + .header(hyper::header::CONTENT_TYPE, "application/octet-stream") + .body(body) + .unwrap(); + + let span = info_span!("blocking"); + tokio::task::spawn_blocking(move || { + let _span = span.entered(); + + let res = serde_json::to_writer(&mut writer, &resp) + .map_err(std::io::Error::from) + .and_then(|_| writer.flush()); + + match res { + Ok(()) => { + tracing::info!( + bytes = writer.flushed_bytes(), + elapsed_ms = started_at.elapsed().as_millis(), + "responded /v1/debug_dump" + ); + } + Err(e) => { + tracing::warn!("failed to write out /v1/debug_dump response: {e:#}"); + // semantics of this error are quite... unclear. we want to error the stream out to + // abort the response to somehow notify the client that we failed. + // + // though, most likely the reason for failure is that the receiver is already gone. + drop( + writer + .tx + .blocking_send(Err(std::io::ErrorKind::BrokenPipe.into())), + ); + } + } + }); + + Ok(response) } /// Safekeeper http router. diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 1343bba5cc..e2f1b9fcff 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use anyhow::{bail, Context, Result}; @@ -32,6 +33,16 @@ pub struct Response { // TODO: add more fields? } +/// Response for debug dump request. +#[derive(Debug, Serialize, Deserialize)] +pub struct DebugDumpResponse { + pub start_time: DateTime, + pub finish_time: DateTime, + pub timelines: Vec, + pub timelines_count: usize, + pub config: debug_dump::Config, +} + /// Find the most advanced safekeeper and pull timeline from it. pub async fn handle_request(request: Request) -> Result { let existing_tli = GlobalTimelines::get(TenantTimelineId::new( @@ -103,7 +114,7 @@ async fn pull_timeline(status: TimelineStatus, host: String) -> Result // Implementing our own scp over HTTP. // At first, we need to fetch list of files from safekeeper. - let dump: debug_dump::Response = client + let dump: DebugDumpResponse = client .get(format!( "{}/v1/debug_dump?dump_all=true&tenant_id={}&timeline_id={}", host, status.tenant_id, status.timeline_id diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 02faf715da..740c05dcea 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2868,7 +2868,7 @@ class SafekeeperHttpClient(requests.Session): params = params or {} res = self.get(f"http://localhost:{self.port}/v1/debug_dump", params=params) res.raise_for_status() - res_json = res.json() + res_json = json.loads(res.text) assert isinstance(res_json, dict) return res_json From b85fc39bdb49cd19fea4e6bab6d3cec52d9ca471 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 6 Nov 2023 09:26:09 +0200 Subject: [PATCH 22/29] Update control plane API path for getting compute spec. (#5357) We changed the path in the control plane. The old path is still accepted for compatibility with existing computes, but we'd like to phase it out. --- compute_tools/src/spec.rs | 2 +- test_runner/regress/test_ddl_forwarding.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 3fa3cf055f..a85d6287b1 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -68,7 +68,7 @@ pub fn get_spec_from_control_plane( base_uri: &str, compute_id: &str, ) -> Result> { - let cp_uri = format!("{base_uri}/management/api/v2/computes/{compute_id}/spec"); + let cp_uri = format!("{base_uri}/compute/api/v2/computes/{compute_id}/spec"); let jwt: String = match std::env::var("NEON_CONTROL_PLANE_TOKEN") { Ok(v) => v, Err(_) => "".to_string(), diff --git a/test_runner/regress/test_ddl_forwarding.py b/test_runner/regress/test_ddl_forwarding.py index 6bd09c7030..07ee8b405e 100644 --- a/test_runner/regress/test_ddl_forwarding.py +++ b/test_runner/regress/test_ddl_forwarding.py @@ -72,7 +72,7 @@ class DdlForwardingContext: self.dbs: Dict[str, str] = {} self.roles: Dict[str, str] = {} self.fail = False - endpoint = "/management/api/v2/roles_and_databases" + endpoint = "/test/roles_and_databases" ddl_url = f"http://{host}:{port}{endpoint}" self.pg.configure( [ From b3d3a2587d80bc541403dc47d9b0729ab6e61b5d Mon Sep 17 00:00:00 2001 From: duguorong009 <80258679+duguorong009@users.noreply.github.com> Date: Mon, 6 Nov 2023 04:40:03 -0500 Subject: [PATCH 23/29] feat: improve the serde impl for several types(`Lsn`, `TenantId`, `TimelineId` ...) (#5335) Improve the serde impl for several types (`Lsn`, `TenantId`, `TimelineId`) by making them sensitive to `Serializer::is_human_readadable` (true for json, false for bincode). Fixes #3511 by: - Implement the custom serde for `Lsn` - Implement the custom serde for `Id` - Add the helper module `serde_as_u64` in `libs/utils/src/lsn.rs` - Remove the unnecessary attr `#[serde_as(as = "DisplayFromStr")]` in all possible structs Additionally some safekeeper types gained serde tests. --------- Co-authored-by: Joonas Koivunen --- Cargo.lock | 11 + Cargo.toml | 1 + control_plane/src/attachment_service.rs | 3 - control_plane/src/endpoint.rs | 4 - control_plane/src/local_env.rs | 4 - libs/compute_api/src/spec.rs | 11 +- libs/pageserver_api/src/control_api.rs | 7 - libs/pageserver_api/src/models.rs | 47 +--- libs/safekeeper_api/src/models.rs | 12 - libs/utils/Cargo.toml | 1 + libs/utils/src/auth.rs | 3 - libs/utils/src/hex.rs | 41 +++ libs/utils/src/id.rs | 180 ++++++++++++- libs/utils/src/lib.rs | 4 + libs/utils/src/lsn.rs | 206 +++++++++++++- libs/utils/src/pageserver_feedback.rs | 5 - pageserver/src/consumption_metrics/metrics.rs | 4 - pageserver/src/consumption_metrics/upload.rs | 4 - pageserver/src/deletion_queue.rs | 3 - pageserver/src/http/routes.rs | 5 - pageserver/src/tenant/metadata.rs | 119 +++++++++ .../tenant/remote_timeline_client/index.rs | 3 - pageserver/src/tenant/size.rs | 13 - pageserver/src/tenant/timeline.rs | 3 - s3_scrubber/src/cloud_admin_api.rs | 21 +- safekeeper/src/control_file_upgrade.rs | 252 +++++++++++++++++- safekeeper/src/debug_dump.rs | 4 - safekeeper/src/http/routes.rs | 13 - safekeeper/src/json_ctrl.rs | 3 + safekeeper/src/pull_timeline.rs | 5 - safekeeper/src/safekeeper.rs | 114 +++++++- safekeeper/src/send_wal.rs | 3 - safekeeper/src/timeline.rs | 6 - 33 files changed, 930 insertions(+), 185 deletions(-) create mode 100644 libs/utils/src/hex.rs diff --git a/Cargo.lock b/Cargo.lock index 2203ad462c..a7e88688ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4681,6 +4681,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_assert" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda563240c1288b044209be1f0d38bb4d15044fb3e00dc354fbc922ab4733e80" +dependencies = [ + "hashbrown 0.13.2", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.183" @@ -5967,6 +5977,7 @@ dependencies = [ "routerify", "sentry", "serde", + "serde_assert", "serde_json", "serde_with", "signal-hook", diff --git a/Cargo.toml b/Cargo.toml index 6f538941f9..d992db4858 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ sentry = { version = "0.31", default-features = false, features = ["backtrace", serde = { version = "1.0", features = ["derive"] } serde_json = "1" serde_with = "2.0" +serde_assert = "0.5.0" sha2 = "0.10.2" signal-hook = "0.3" smallvec = "1.11" diff --git a/control_plane/src/attachment_service.rs b/control_plane/src/attachment_service.rs index ca632c5eb6..fcefe0e431 100644 --- a/control_plane/src/attachment_service.rs +++ b/control_plane/src/attachment_service.rs @@ -2,7 +2,6 @@ use crate::{background_process, local_env::LocalEnv}; use anyhow::anyhow; use camino::Utf8PathBuf; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::{path::PathBuf, process::Child}; use utils::id::{NodeId, TenantId}; @@ -14,10 +13,8 @@ pub struct AttachmentService { const COMMAND: &str = "attachment_service"; -#[serde_as] #[derive(Serialize, Deserialize)] pub struct AttachHookRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, pub node_id: Option, } diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index cb16f48829..4443fd8704 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -46,7 +46,6 @@ use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{NodeId, TenantId, TimelineId}; use crate::local_env::LocalEnv; @@ -57,13 +56,10 @@ use compute_api::responses::{ComputeState, ComputeStatus}; use compute_api::spec::{Cluster, ComputeMode, ComputeSpec}; // contents of a endpoint.json file -#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct EndpointConf { endpoint_id: String, - #[serde_as(as = "DisplayFromStr")] tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] timeline_id: TimelineId, mode: ComputeMode, pg_port: u16, diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 45a7469787..b9c8aeddcb 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -8,7 +8,6 @@ use anyhow::{bail, ensure, Context}; use postgres_backend::AuthType; use reqwest::Url; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::collections::HashMap; use std::env; use std::fs; @@ -33,7 +32,6 @@ pub const DEFAULT_PG_VERSION: u32 = 15; // to 'neon_local init --config=' option. See control_plane/simple.conf for // an example. // -#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct LocalEnv { // Base directory for all the nodes (the pageserver, safekeepers and @@ -59,7 +57,6 @@ pub struct LocalEnv { // Default tenant ID to use with the 'neon_local' command line utility, when // --tenant_id is not explicitly specified. #[serde(default)] - #[serde_as(as = "Option")] pub default_tenant_id: Option, // used to issue tokens during e.g pg start @@ -84,7 +81,6 @@ pub struct LocalEnv { // A `HashMap>` would be more appropriate here, // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table". - #[serde_as(as = "HashMap<_, Vec<(DisplayFromStr, DisplayFromStr)>>")] branch_name_mappings: HashMap>, } diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index c16deceebb..175b4461ac 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{TenantId, TimelineId}; use utils::lsn::Lsn; @@ -19,7 +18,6 @@ pub type PgIdent = String; /// Cluster spec or configuration represented as an optional number of /// delta operations + final cluster state description. -#[serde_as] #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ComputeSpec { pub format_version: f32, @@ -50,12 +48,12 @@ pub struct ComputeSpec { // these, and instead set the "neon.tenant_id", "neon.timeline_id", // etc. GUCs in cluster.settings. TODO: Once the control plane has been // updated to fill these fields, we can make these non optional. - #[serde_as(as = "Option")] pub tenant_id: Option, - #[serde_as(as = "Option")] + pub timeline_id: Option, - #[serde_as(as = "Option")] + pub pageserver_connstring: Option, + #[serde(default)] pub safekeeper_connstrings: Vec, @@ -140,14 +138,13 @@ impl RemoteExtSpec { } } -#[serde_as] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] pub enum ComputeMode { /// A read-write node #[default] Primary, /// A read-only node, pinned at a particular LSN - Static(#[serde_as(as = "DisplayFromStr")] Lsn), + Static(Lsn), /// A read-only node that follows the tip of the branch in hot standby mode /// /// Future versions may want to distinguish between replicas with hot standby diff --git a/libs/pageserver_api/src/control_api.rs b/libs/pageserver_api/src/control_api.rs index 3819111a5b..8232e81b98 100644 --- a/libs/pageserver_api/src/control_api.rs +++ b/libs/pageserver_api/src/control_api.rs @@ -4,7 +4,6 @@ //! See docs/rfcs/025-generation-numbers.md use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{NodeId, TenantId}; #[derive(Serialize, Deserialize)] @@ -12,10 +11,8 @@ pub struct ReAttachRequest { pub node_id: NodeId, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ReAttachResponseTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub gen: u32, } @@ -25,10 +22,8 @@ pub struct ReAttachResponse { pub tenants: Vec, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ValidateRequestTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub gen: u32, } @@ -43,10 +38,8 @@ pub struct ValidateResponse { pub tenants: Vec, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ValidateResponseTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub valid: bool, } diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index e667645c0a..cb99dc0a55 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -6,7 +6,7 @@ use std::{ use byteorder::{BigEndian, ReadBytesExt}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; +use serde_with::serde_as; use strum_macros; use utils::{ completion, @@ -174,25 +174,19 @@ pub enum TimelineState { Broken { reason: String, backtrace: String }, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub new_timeline_id: TimelineId, #[serde(default)] - #[serde_as(as = "Option")] pub ancestor_timeline_id: Option, #[serde(default)] - #[serde_as(as = "Option")] pub ancestor_start_lsn: Option, pub pg_version: Option, } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub new_tenant_id: TenantId, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] @@ -201,7 +195,6 @@ pub struct TenantCreateRequest { pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it } -#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantLoadRequest { @@ -278,31 +271,26 @@ pub struct LocationConfig { pub tenant_conf: TenantConfig, } -#[serde_as] #[derive(Serialize, Deserialize)] #[serde(transparent)] -pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub TenantId); +pub struct TenantCreateResponse(pub TenantId); #[derive(Serialize)] pub struct StatusResponse { pub id: NodeId, } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantLocationConfigRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, #[serde(flatten)] pub config: LocationConfig, // as we have a flattened field, we should reject all unknown fields in it } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantConfigRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, #[serde(flatten)] pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it @@ -374,10 +362,8 @@ pub enum TenantAttachmentStatus { Failed { reason: String }, } -#[serde_as] #[derive(Serialize, Deserialize, Clone)] pub struct TenantInfo { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, // NB: intentionally not part of OpenAPI, we don't want to commit to a specific set of TenantState's pub state: TenantState, @@ -388,33 +374,22 @@ pub struct TenantInfo { } /// This represents the output of the "timeline_detail" and "timeline_list" API calls. -#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimelineInfo { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, - #[serde_as(as = "Option")] pub ancestor_timeline_id: Option, - #[serde_as(as = "Option")] pub ancestor_lsn: Option, - #[serde_as(as = "DisplayFromStr")] pub last_record_lsn: Lsn, - #[serde_as(as = "Option")] pub prev_record_lsn: Option, - #[serde_as(as = "DisplayFromStr")] pub latest_gc_cutoff_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, /// The LSN that we have succesfully uploaded to remote storage - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, /// The LSN that we are advertizing to safekeepers - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn_visible: Lsn, pub current_logical_size: Option, // is None when timeline is Unloaded @@ -426,7 +401,6 @@ pub struct TimelineInfo { pub timeline_dir_layer_file_size_sum: Option, pub wal_source_connstr: Option, - #[serde_as(as = "Option")] pub last_received_msg_lsn: Option, /// the timestamp (in microseconds) of the last received message pub last_received_msg_ts: Option, @@ -523,23 +497,13 @@ pub struct LayerAccessStats { pub residence_events_history: HistoryBufferWithDropCounter, } -#[serde_as] #[derive(Debug, Clone, Serialize)] #[serde(tag = "kind")] pub enum InMemoryLayerInfo { - Open { - #[serde_as(as = "DisplayFromStr")] - lsn_start: Lsn, - }, - Frozen { - #[serde_as(as = "DisplayFromStr")] - lsn_start: Lsn, - #[serde_as(as = "DisplayFromStr")] - lsn_end: Lsn, - }, + Open { lsn_start: Lsn }, + Frozen { lsn_start: Lsn, lsn_end: Lsn }, } -#[serde_as] #[derive(Debug, Clone, Serialize)] #[serde(tag = "kind")] pub enum HistoricLayerInfo { @@ -547,9 +511,7 @@ pub enum HistoricLayerInfo { layer_file_name: String, layer_file_size: u64, - #[serde_as(as = "DisplayFromStr")] lsn_start: Lsn, - #[serde_as(as = "DisplayFromStr")] lsn_end: Lsn, remote: bool, access_stats: LayerAccessStats, @@ -558,7 +520,6 @@ pub enum HistoricLayerInfo { layer_file_name: String, layer_file_size: u64, - #[serde_as(as = "DisplayFromStr")] lsn_start: Lsn, remote: bool, access_stats: LayerAccessStats, diff --git a/libs/safekeeper_api/src/models.rs b/libs/safekeeper_api/src/models.rs index eaea266f32..786712deb1 100644 --- a/libs/safekeeper_api/src/models.rs +++ b/libs/safekeeper_api/src/models.rs @@ -1,23 +1,18 @@ use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::{ id::{NodeId, TenantId, TimelineId}, lsn::Lsn, }; -#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub peer_ids: Option>, pub pg_version: u32, pub system_id: Option, pub wal_seg_size: Option, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, // If not passed, it is assigned to the beginning of commit_lsn segment. pub local_start_lsn: Option, @@ -28,7 +23,6 @@ fn lsn_invalid() -> Lsn { } /// Data about safekeeper's timeline, mirrors broker.proto. -#[serde_as] #[derive(Debug, Clone, Deserialize, Serialize)] pub struct SkTimelineInfo { /// Term. @@ -36,25 +30,19 @@ pub struct SkTimelineInfo { /// Term of the last entry. pub last_log_term: Option, /// LSN of the last record. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub flush_lsn: Lsn, /// Up to which LSN safekeeper regards its WAL as committed. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub commit_lsn: Lsn, /// LSN up to which safekeeper has backed WAL. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub backup_lsn: Lsn, /// LSN of last checkpoint uploaded by pageserver. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub remote_consistent_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub peer_horizon_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub local_start_lsn: Lsn, /// A connection string to use for WAL receiving. diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 27df0265b4..3f4ef2abeb 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -55,6 +55,7 @@ bytes.workspace = true criterion.workspace = true hex-literal.workspace = true camino-tempfile.workspace = true +serde_assert.workspace = true [[bench]] name = "benchmarks" diff --git a/libs/utils/src/auth.rs b/libs/utils/src/auth.rs index 54b90fa070..37299e4e7f 100644 --- a/libs/utils/src/auth.rs +++ b/libs/utils/src/auth.rs @@ -9,7 +9,6 @@ use jsonwebtoken::{ decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, }; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use crate::id::TenantId; @@ -32,11 +31,9 @@ pub enum Scope { } /// JWT payload. See docs/authentication.md for the format -#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct Claims { #[serde(default)] - #[serde_as(as = "Option")] pub tenant_id: Option, pub scope: Scope, } diff --git a/libs/utils/src/hex.rs b/libs/utils/src/hex.rs new file mode 100644 index 0000000000..fc0bb7e4a2 --- /dev/null +++ b/libs/utils/src/hex.rs @@ -0,0 +1,41 @@ +/// Useful type for asserting that expected bytes match reporting the bytes more readable +/// array-syntax compatible hex bytes. +/// +/// # Usage +/// +/// ``` +/// use utils::Hex; +/// +/// let actual = serialize_something(); +/// let expected = [0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64]; +/// +/// // the type implements PartialEq and on mismatch, both sides are printed in 16 wide multiline +/// // output suffixed with an array style length for easier comparisons. +/// assert_eq!(Hex(&actual), Hex(&expected)); +/// +/// // with `let expected = [0x68];` the error would had been: +/// // assertion `left == right` failed +/// // left: [0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64; 11] +/// // right: [0x68; 1] +/// # fn serialize_something() -> Vec { "hello world".as_bytes().to_vec() } +/// ``` +#[derive(PartialEq)] +pub struct Hex<'a>(pub &'a [u8]); + +impl std::fmt::Debug for Hex<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for (i, c) in self.0.chunks(16).enumerate() { + if i > 0 && !c.is_empty() { + writeln!(f, ", ")?; + } + for (j, b) in c.iter().enumerate() { + if j > 0 { + write!(f, ", ")?; + } + write!(f, "0x{b:02x}")?; + } + } + write!(f, "; {}]", self.0.len()) + } +} diff --git a/libs/utils/src/id.rs b/libs/utils/src/id.rs index ec13c2f96f..eef6a358e6 100644 --- a/libs/utils/src/id.rs +++ b/libs/utils/src/id.rs @@ -3,6 +3,7 @@ use std::{fmt, str::FromStr}; use anyhow::Context; use hex::FromHex; use rand::Rng; +use serde::de::Visitor; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -17,12 +18,74 @@ pub enum IdError { /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// -/// Use `#[serde_as(as = "DisplayFromStr")]` to (de)serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. -/// Check the `serde_with::serde_as` documentation for options for more complex types. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] struct Id([u8; 16]); +impl Serialize for Id { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + self.0.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for Id { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> Visitor<'de> for IdVisitor { + type Value = Id; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 16])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 16] = Deserialize::deserialize(s)?; + Ok(Id::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Id::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 16, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} + impl Id { pub fn get_from_buf(buf: &mut impl bytes::Buf) -> Id { let mut arr = [0u8; 16]; @@ -308,3 +371,112 @@ impl fmt::Display for NodeId { write!(f, "{}", self.0) } } + +#[cfg(test)] +mod tests { + use serde_assert::{Deserializer, Serializer, Token, Tokens}; + + use crate::bin_ser::BeSer; + + use super::*; + + #[test] + fn test_id_serde_non_human_readable() { + let original_id = Id([ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]); + let expected_tokens = Tokens(vec![ + Token::Tuple { len: 16 }, + Token::U8(173), + Token::U8(80), + Token::U8(132), + Token::U8(115), + Token::U8(129), + Token::U8(226), + Token::U8(72), + Token::U8(254), + Token::U8(170), + Token::U8(201), + Token::U8(135), + Token::U8(108), + Token::U8(199), + Token::U8(26), + Token::U8(228), + Token::U8(24), + Token::TupleEnd, + ]); + + let serializer = Serializer::builder().is_human_readable(false).build(); + let serialized_tokens = original_id.serialize(&serializer).unwrap(); + assert_eq!(serialized_tokens, expected_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(serialized_tokens) + .build(); + let deserialized_id = Id::deserialize(&mut deserializer).unwrap(); + assert_eq!(deserialized_id, original_id); + } + + #[test] + fn test_id_serde_human_readable() { + let original_id = Id([ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]); + let expected_tokens = Tokens(vec![Token::Str(String::from( + "ad50847381e248feaac9876cc71ae418", + ))]); + + let serializer = Serializer::builder().is_human_readable(true).build(); + let serialized_tokens = original_id.serialize(&serializer).unwrap(); + assert_eq!(serialized_tokens, expected_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(Tokens(vec![Token::Str(String::from( + "ad50847381e248feaac9876cc71ae418", + ))])) + .build(); + assert_eq!(Id::deserialize(&mut deserializer).unwrap(), original_id); + } + + macro_rules! roundtrip_type { + ($type:ty, $expected_bytes:expr) => {{ + let expected_bytes: [u8; 16] = $expected_bytes; + let original_id = <$type>::from(expected_bytes); + + let ser_bytes = original_id.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_id = <$type>::des(&ser_bytes).unwrap(); + assert_eq!(des_id, original_id); + }}; + } + + #[test] + fn test_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(Id, expected_bytes); + } + + #[test] + fn test_tenant_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(TenantId, expected_bytes); + } + + #[test] + fn test_timeline_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(TimelineId, expected_bytes); + } +} diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 88cefd516d..1e34034023 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -24,6 +24,10 @@ pub mod auth; // utility functions and helper traits for unified unique id generation/serialization etc. pub mod id; + +mod hex; +pub use hex::Hex; + // http endpoint utils pub mod http; diff --git a/libs/utils/src/lsn.rs b/libs/utils/src/lsn.rs index 7d9baf7d49..262dcb8a8a 100644 --- a/libs/utils/src/lsn.rs +++ b/libs/utils/src/lsn.rs @@ -1,7 +1,7 @@ #![warn(missing_docs)] use camino::Utf8Path; -use serde::{Deserialize, Serialize}; +use serde::{de::Visitor, Deserialize, Serialize}; use std::fmt; use std::ops::{Add, AddAssign}; use std::str::FromStr; @@ -13,10 +13,114 @@ use crate::seqwait::MonotonicCounter; pub const XLOG_BLCKSZ: u32 = 8192; /// A Postgres LSN (Log Sequence Number), also known as an XLogRecPtr -#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, Serialize, Deserialize)] -#[serde(transparent)] +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash)] pub struct Lsn(pub u64); +impl Serialize for Lsn { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + self.0.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for Lsn { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct LsnVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> Visitor<'de> for LsnVisitor { + type Value = Lsn; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str( + "value in form of hex string({upper_u32_hex}/{lower_u32_hex}) representing u64 integer", + ) + } else { + formatter.write_str("value in form of integer(u64)") + } + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + Ok(Lsn(v)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Lsn::from_str(v).map_err(|e| E::custom(e)) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(LsnVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_u64(LsnVisitor { + is_human_readable_deserializer: false, + }) + } + } +} + +/// Allows (de)serialization of an `Lsn` always as `u64`. +/// +/// ### Example +/// +/// ```rust +/// # use serde::{Serialize, Deserialize}; +/// use utils::lsn::Lsn; +/// +/// #[derive(PartialEq, Serialize, Deserialize, Debug)] +/// struct Foo { +/// #[serde(with = "utils::lsn::serde_as_u64")] +/// always_u64: Lsn, +/// } +/// +/// let orig = Foo { always_u64: Lsn(1234) }; +/// +/// let res = serde_json::to_string(&orig).unwrap(); +/// assert_eq!(res, r#"{"always_u64":1234}"#); +/// +/// let foo = serde_json::from_str::(&res).unwrap(); +/// assert_eq!(foo, orig); +/// ``` +/// +pub mod serde_as_u64 { + use super::Lsn; + + /// Serializes the Lsn as u64 disregarding the human readability of the format. + /// + /// Meant to be used via `#[serde(with = "...")]` or `#[serde(serialize_with = "...")]`. + pub fn serialize(lsn: &Lsn, serializer: S) -> Result { + use serde::Serialize; + lsn.0.serialize(serializer) + } + + /// Deserializes the Lsn as u64 disregarding the human readability of the format. + /// + /// Meant to be used via `#[serde(with = "...")]` or `#[serde(deserialize_with = "...")]`. + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + use serde::Deserialize; + u64::deserialize(deserializer).map(Lsn) + } +} + /// We tried to parse an LSN from a string, but failed #[derive(Debug, PartialEq, Eq, thiserror::Error)] #[error("LsnParseError")] @@ -264,8 +368,13 @@ impl MonotonicCounter for RecordLsn { #[cfg(test)] mod tests { + use crate::bin_ser::BeSer; + use super::*; + use serde::ser::Serialize; + use serde_assert::{Deserializer, Serializer, Token, Tokens}; + #[test] fn test_lsn_strings() { assert_eq!("12345678/AAAA5555".parse(), Ok(Lsn(0x12345678AAAA5555))); @@ -341,4 +450,95 @@ mod tests { assert_eq!(lsn.fetch_max(Lsn(6000)), Lsn(5678)); assert_eq!(lsn.fetch_max(Lsn(5000)), Lsn(6000)); } + + #[test] + fn test_lsn_serde() { + let original_lsn = Lsn(0x0123456789abcdef); + let expected_readable_tokens = Tokens(vec![Token::U64(0x0123456789abcdef)]); + let expected_non_readable_tokens = + Tokens(vec![Token::Str(String::from("1234567/89ABCDEF"))]); + + // Testing human_readable ser/de + let serializer = Serializer::builder().is_human_readable(false).build(); + let readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + assert_eq!(readable_ser_tokens, expected_readable_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(readable_ser_tokens) + .build(); + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + + // Testing NON human_readable ser/de + let serializer = Serializer::builder().is_human_readable(true).build(); + let non_readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + assert_eq!(non_readable_ser_tokens, expected_non_readable_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(non_readable_ser_tokens) + .build(); + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + + // Testing mismatching ser/de + let serializer = Serializer::builder().is_human_readable(false).build(); + let non_readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(non_readable_ser_tokens) + .build(); + Lsn::deserialize(&mut deserializer).unwrap_err(); + + let serializer = Serializer::builder().is_human_readable(true).build(); + let readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(readable_ser_tokens) + .build(); + Lsn::deserialize(&mut deserializer).unwrap_err(); + } + + #[test] + fn test_lsn_ensure_roundtrip() { + let original_lsn = Lsn(0xaaaabbbb); + + let serializer = Serializer::builder().is_human_readable(false).build(); + let ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(ser_tokens) + .build(); + + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + } + + #[test] + fn test_lsn_bincode_serde() { + let lsn = Lsn(0x0123456789abcdef); + let expected_bytes = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + + let ser_bytes = lsn.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_lsn = Lsn::des(&ser_bytes).unwrap(); + assert_eq!(des_lsn, lsn); + } + + #[test] + fn test_lsn_bincode_ensure_roundtrip() { + let original_lsn = Lsn(0x01_02_03_04_05_06_07_08); + let expected_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + + let ser_bytes = original_lsn.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_lsn = Lsn::des(&ser_bytes).unwrap(); + assert_eq!(des_lsn, original_lsn); + } } diff --git a/libs/utils/src/pageserver_feedback.rs b/libs/utils/src/pageserver_feedback.rs index a3b53201d3..c9fbdde928 100644 --- a/libs/utils/src/pageserver_feedback.rs +++ b/libs/utils/src/pageserver_feedback.rs @@ -3,7 +3,6 @@ use std::time::{Duration, SystemTime}; use bytes::{Buf, BufMut, Bytes, BytesMut}; use pq_proto::{read_cstr, PG_EPOCH}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use tracing::{trace, warn}; use crate::lsn::Lsn; @@ -15,21 +14,17 @@ use crate::lsn::Lsn; /// /// serde Serialize is used only for human readable dump to json (e.g. in /// safekeepers debug_dump). -#[serde_as] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct PageserverFeedback { /// Last known size of the timeline. Used to enforce timeline size limit. pub current_timeline_size: u64, /// LSN last received and ingested by the pageserver. Controls backpressure. - #[serde_as(as = "DisplayFromStr")] pub last_received_lsn: Lsn, /// LSN up to which data is persisted by the pageserver to its local disc. /// Controls backpressure. - #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, /// LSN up to which data is persisted by the pageserver on s3; safekeepers /// consider WAL before it can be removed. - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, // Serialize with RFC3339 format. #[serde(with = "serde_systemtime")] diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index 652dd98683..c22f218976 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -3,7 +3,6 @@ use anyhow::Context; use chrono::{DateTime, Utc}; use consumption_metrics::EventType; use futures::stream::StreamExt; -use serde_with::serde_as; use std::{sync::Arc, time::SystemTime}; use utils::{ id::{TenantId, TimelineId}, @@ -42,13 +41,10 @@ pub(super) enum Name { /// /// This is a denormalization done at the MetricsKey const methods; these should not be constructed /// elsewhere. -#[serde_with::serde_as] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub(crate) struct MetricsKey { - #[serde_as(as = "serde_with::DisplayFromStr")] pub(super) tenant_id: TenantId, - #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] pub(super) timeline_id: Option, diff --git a/pageserver/src/consumption_metrics/upload.rs b/pageserver/src/consumption_metrics/upload.rs index d69d43a2a8..322ed95cc8 100644 --- a/pageserver/src/consumption_metrics/upload.rs +++ b/pageserver/src/consumption_metrics/upload.rs @@ -1,5 +1,4 @@ use consumption_metrics::{Event, EventChunk, IdempotencyKey, CHUNK_SIZE}; -use serde_with::serde_as; use tokio_util::sync::CancellationToken; use tracing::Instrument; @@ -7,12 +6,9 @@ use super::{metrics::Name, Cache, MetricsKey, RawMetric}; use utils::id::{TenantId, TimelineId}; /// How the metrics from pageserver are identified. -#[serde_with::serde_as] #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq)] struct Ids { - #[serde_as(as = "serde_with::DisplayFromStr")] pub(super) tenant_id: TenantId, - #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] pub(super) timeline_id: Option, } diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 7b2db929fa..b6f889c682 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -18,7 +18,6 @@ use hex::FromHex; use remote_storage::{GenericRemoteStorage, RemotePath}; use serde::Deserialize; use serde::Serialize; -use serde_with::serde_as; use thiserror::Error; use tokio; use tokio_util::sync::CancellationToken; @@ -215,7 +214,6 @@ where /// during recovery as startup. const TEMP_SUFFIX: &str = "tmp"; -#[serde_as] #[derive(Debug, Serialize, Deserialize)] struct DeletionList { /// Serialization version, for future use @@ -244,7 +242,6 @@ struct DeletionList { validated: bool, } -#[serde_as] #[derive(Debug, Serialize, Deserialize)] struct DeletionHeader { /// Serialization version, for future use diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 2c46d733d6..a6fb26b298 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -17,7 +17,6 @@ use pageserver_api::models::{ TenantLoadRequest, TenantLocationConfigRequest, }; use remote_storage::GenericRemoteStorage; -use serde_with::{serde_as, DisplayFromStr}; use tenant_size_model::{SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; use tracing::*; @@ -499,10 +498,8 @@ async fn get_lsn_by_timestamp_handler( let result = timeline.find_lsn_for_timestamp(timestamp_pg, &ctx).await?; if version.unwrap_or(0) > 1 { - #[serde_as] #[derive(serde::Serialize)] struct Result { - #[serde_as(as = "DisplayFromStr")] lsn: Lsn, kind: &'static str, } @@ -811,10 +808,8 @@ async fn tenant_size_handler( } /// The type resides in the pageserver not to expose `ModelInputs`. - #[serde_with::serde_as] #[derive(serde::Serialize)] struct TenantHistorySize { - #[serde_as(as = "serde_with::DisplayFromStr")] id: TenantId, /// Size is a mixture of WAL and logical size, so the unit is bytes. /// diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 75ffe09696..38fd426746 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -406,4 +406,123 @@ mod tests { METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION ); } + + #[test] + fn test_metadata_bincode_serde() { + let original_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + // Any version will do here, so use the default + crate::DEFAULT_PG_VERSION, + ); + let metadata_bytes = original_metadata + .to_bytes() + .expect("Cannot create bytes array from metadata"); + + let metadata_bincode_be_bytes = original_metadata + .ser() + .expect("Cannot serialize the metadata"); + + // 8 bytes for the length of the vector + assert_eq!(metadata_bincode_be_bytes.len(), 8 + metadata_bytes.len()); + + let expected_bincode_bytes = { + let mut temp = vec![]; + let len_bytes = metadata_bytes.len().to_be_bytes(); + temp.extend_from_slice(&len_bytes); + temp.extend_from_slice(&metadata_bytes); + temp + }; + assert_eq!(metadata_bincode_be_bytes, expected_bincode_bytes); + + let deserialized_metadata = TimelineMetadata::des(&metadata_bincode_be_bytes).unwrap(); + // Deserialized metadata has the metadata header, which is different from the serialized one. + // Reference: TimelineMetaData::to_bytes() + let expected_metadata = { + let mut temp_metadata = original_metadata; + let body_bytes = temp_metadata + .body + .ser() + .expect("Cannot serialize the metadata body"); + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_FORMAT_VERSION, + checksum: crc32c::crc32c(&body_bytes), + }; + temp_metadata.hdr = hdr; + temp_metadata + }; + assert_eq!(deserialized_metadata, expected_metadata); + } + + #[test] + fn test_metadata_bincode_serde_ensure_roundtrip() { + let original_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + // Any version will do here, so use the default + crate::DEFAULT_PG_VERSION, + ); + let expected_bytes = vec![ + /* bincode length encoding bytes */ + 0, 0, 0, 0, 0, 0, 2, 0, // 8 bytes for the length of the serialized vector + /* TimelineMetadataHeader */ + 4, 37, 101, 34, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2) + /* TimelineMetadataBodyV2 */ + 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes) + 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes) + 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119, + 136, // ancestor_timeline (17 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes) + 0, 0, 0, 15, // pg_version (4 bytes) + /* padding bytes */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + ]; + let metadata_ser_bytes = original_metadata.ser().unwrap(); + assert_eq!(metadata_ser_bytes, expected_bytes); + + let expected_metadata = { + let mut temp_metadata = original_metadata; + let body_bytes = temp_metadata + .body + .ser() + .expect("Cannot serialize the metadata body"); + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_FORMAT_VERSION, + checksum: crc32c::crc32c(&body_bytes), + }; + temp_metadata.hdr = hdr; + temp_metadata + }; + let des_metadata = TimelineMetadata::des(&metadata_ser_bytes).unwrap(); + assert_eq!(des_metadata, expected_metadata); + } } diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index fdab74a8be..fa0679c7a2 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::bin_ser::SerializeError; use crate::tenant::metadata::TimelineMetadata; @@ -58,7 +57,6 @@ impl LayerFileMetadata { /// /// This type needs to be backwards and forwards compatible. When changing the fields, /// remember to add a test case for the changed version. -#[serde_as] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct IndexPart { /// Debugging aid describing the version of this type. @@ -78,7 +76,6 @@ pub struct IndexPart { // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata. // It's duplicated for convenience when reading the serialized structure, but is // private because internally we would read from metadata instead. - #[serde_as(as = "DisplayFromStr")] disk_consistent_lsn: Lsn, #[serde(rename = "metadata_bytes")] diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index e737d3f59c..5e8ee2b99e 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -29,7 +29,6 @@ use tenant_size_model::{Segment, StorageModel}; /// needs. We will convert this into a StorageModel when it's time to perform /// the calculation. /// -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct ModelInputs { pub segments: Vec, @@ -37,11 +36,9 @@ pub struct ModelInputs { } /// A [`Segment`], with some extra information for display purposes -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct SegmentMeta { pub segment: Segment, - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, pub kind: LsnKind, } @@ -77,32 +74,22 @@ pub enum LsnKind { /// Collect all relevant LSNs to the inputs. These will only be helpful in the serialized form as /// part of [`ModelInputs`] from the HTTP api, explaining the inputs. -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct TimelineInputs { - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, - #[serde_as(as = "Option")] pub ancestor_id: Option, - #[serde_as(as = "serde_with::DisplayFromStr")] ancestor_lsn: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] last_record: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] latest_gc_cutoff: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] horizon_cutoff: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] pitr_cutoff: Lsn, /// Cutoff point based on GC settings - #[serde_as(as = "serde_with::DisplayFromStr")] next_gc_cutoff: Lsn, /// Cutoff point calculated from the user-supplied 'max_retention_period' - #[serde_as(as = "Option")] retention_param_cutoff: Option, } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 4608683c4a..baa97b6cf9 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -2936,13 +2936,10 @@ struct CompactLevel0Phase1StatsBuilder { new_deltas_size: Option, } -#[serde_as] #[derive(serde::Serialize)] struct CompactLevel0Phase1Stats { version: u64, - #[serde_as(as = "serde_with::DisplayFromStr")] tenant_id: TenantId, - #[serde_as(as = "serde_with::DisplayFromStr")] timeline_id: TimelineId, read_lock_acquisition_micros: RecordedDuration, read_lock_held_spawn_blocking_startup_micros: RecordedDuration, diff --git a/s3_scrubber/src/cloud_admin_api.rs b/s3_scrubber/src/cloud_admin_api.rs index 1b28dc4dff..151421c84f 100644 --- a/s3_scrubber/src/cloud_admin_api.rs +++ b/s3_scrubber/src/cloud_admin_api.rs @@ -5,6 +5,7 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use hex::FromHex; +use pageserver::tenant::Tenant; use reqwest::{header, Client, StatusCode, Url}; use serde::Deserialize; use tokio::sync::Semaphore; @@ -118,13 +119,18 @@ fn from_nullable_id<'de, D>(deserializer: D) -> Result where D: serde::de::Deserializer<'de>, { - let id_str = String::deserialize(deserializer)?; - if id_str.is_empty() { - // This is a bogus value, but for the purposes of the scrubber all that - // matters is that it doesn't collide with any real IDs. - Ok(TenantId::from([0u8; 16])) + if deserializer.is_human_readable() { + let id_str = String::deserialize(deserializer)?; + if id_str.is_empty() { + // This is a bogus value, but for the purposes of the scrubber all that + // matters is that it doesn't collide with any real IDs. + Ok(TenantId::from([0u8; 16])) + } else { + TenantId::from_hex(&id_str).map_err(|e| serde::de::Error::custom(format!("{e}"))) + } } else { - TenantId::from_hex(&id_str).map_err(|e| serde::de::Error::custom(format!("{e}"))) + let id_arr = <[u8; 16]>::deserialize(deserializer)?; + Ok(TenantId::from(id_arr)) } } @@ -153,7 +159,6 @@ pub struct ProjectData { pub maintenance_set: Option, } -#[serde_with::serde_as] #[derive(Debug, serde::Deserialize)] pub struct BranchData { pub id: BranchId, @@ -161,12 +166,10 @@ pub struct BranchData { pub updated_at: DateTime, pub name: String, pub project_id: ProjectId, - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, #[serde(default)] pub parent_id: Option, #[serde(default)] - #[serde_as(as = "Option")] pub parent_lsn: Option, pub default: bool, pub deleted: bool, diff --git a/safekeeper/src/control_file_upgrade.rs b/safekeeper/src/control_file_upgrade.rs index a7f17df797..a0be2b2054 100644 --- a/safekeeper/src/control_file_upgrade.rs +++ b/safekeeper/src/control_file_upgrade.rs @@ -13,7 +13,7 @@ use utils::{ }; /// Persistent consensus state of the acceptor. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct AcceptorStateV1 { /// acceptor's last term it voted for (advanced in 1 phase) term: Term, @@ -21,7 +21,7 @@ struct AcceptorStateV1 { epoch: Term, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct SafeKeeperStateV1 { /// persistent acceptor state acceptor_state: AcceptorStateV1, @@ -50,7 +50,7 @@ pub struct ServerInfoV2 { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV2 { /// persistent acceptor state pub acceptor_state: AcceptorState, @@ -81,7 +81,7 @@ pub struct ServerInfoV3 { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV3 { /// persistent acceptor state pub acceptor_state: AcceptorState, @@ -101,7 +101,7 @@ pub struct SafeKeeperStateV3 { pub wal_start_lsn: Lsn, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV4 { #[serde(with = "hex")] pub tenant_id: TenantId, @@ -264,3 +264,245 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result } bail!("unsupported safekeeper control file version {}", version) } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use utils::{id::NodeId, Hex}; + + use crate::safekeeper::PersistedPeerInfo; + + use super::*; + + #[test] + fn roundtrip_v1() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV1 { + acceptor_state: AcceptorStateV1 { + term: 42, + epoch: 43, + }, + server: ServerInfoV2 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + #[rustfmt::skip] + let expected = [ + // term + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // epoch + 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // pg_version + 0x0e, 0x00, 0x00, 0x00, + // system_id + 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, + // tenant_id + 0xcf, 0x04, 0x80, 0x92, 0x97, 0x07, 0xee, 0x75, 0x37, 0x23, 0x37, 0xef, 0xaa, 0x5e, 0xcf, 0x96, + // timeline_id + 0x11, 0x2d, 0xed, 0x66, 0x42, 0x2a, 0xa5, 0xe9, 0x53, 0xe5, 0x44, 0x0f, 0xa5, 0x42, 0x7a, 0xc4, + // wal_seg_size + 0x78, 0x56, 0x34, 0x12, + // proposer_uuid + 0xc4, 0x7a, 0x42, 0xa5, 0x0f, 0x44, 0xe5, 0x53, 0xe9, 0xa5, 0x2a, 0x42, 0x66, 0xed, 0x2d, 0x11, + // commit_lsn + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + // truncate_lsn + 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, 0x00, 0x00, + // wal_start_lsn + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV1::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v2() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV2 { + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfoV2 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, + 0x34, 0x12, 0xcf, 0x04, 0x80, 0x92, 0x97, 0x07, 0xee, 0x75, 0x37, 0x23, 0x37, 0xef, + 0xaa, 0x5e, 0xcf, 0x96, 0x11, 0x2d, 0xed, 0x66, 0x42, 0x2a, 0xa5, 0xe9, 0x53, 0xe5, + 0x44, 0x0f, 0xa5, 0x42, 0x7a, 0xc4, 0x78, 0x56, 0x34, 0x12, 0xc4, 0x7a, 0x42, 0xa5, + 0x0f, 0x44, 0xe5, 0x53, 0xe9, 0xa5, 0x2a, 0x42, 0x66, 0xed, 0x2d, 0x11, 0x78, 0x02, + 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV2::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v3() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV3 { + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfoV3 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x66, 0x30, 0x34, + 0x38, 0x30, 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, + 0x32, 0x33, 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, + 0x36, 0x36, 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, + 0x34, 0x34, 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x34, 0x37, 0x61, + 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, + 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV3::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v4() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV4 { + tenant_id, + timeline_id, + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfo { + pg_version: 14, + system_id: 0x1234567887654321, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + peers: PersistedPeers(vec![( + NodeId(1), + PersistedPeerInfo { + backup_lsn: Lsn(1234567000), + term: 42, + flush_lsn: Lsn(1234567800 - 8), + commit_lsn: Lsn(1234567600), + }, + )]), + commit_lsn: Lsn(1234567800), + s3_wal_lsn: Lsn(1234567300), + peer_horizon_lsn: Lsn(9999999), + remote_consistent_lsn: Lsn(1234560000), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x66, 0x30, 0x34, 0x38, 0x30, + 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, 0x32, 0x33, + 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, 0x20, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, 0x36, 0x36, + 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, 0x34, 0x34, + 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, 0x2a, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x34, 0x37, 0x61, + 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, + 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x84, 0x00, 0x96, 0x49, 0x00, 0x00, + 0x00, 0x00, 0x7f, 0x96, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x95, 0x49, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0xff, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, + 0x00, 0x00, 0xb0, 0x01, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV4::des(&ser).unwrap(); + + assert_eq!(state, deser); + } +} diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index 8cbc0aa47f..daf9255ecb 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -14,7 +14,6 @@ use postgres_ffi::XLogSegNo; use serde::Deserialize; use serde::Serialize; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::NodeId; use utils::id::TenantTimelineId; use utils::id::{TenantId, TimelineId}; @@ -136,12 +135,9 @@ pub struct Config { pub wal_backup_enabled: bool, } -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct Timeline { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub control_file: Option, pub memory: Option, diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 474d636441..06b903719e 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -4,7 +4,6 @@ use once_cell::sync::Lazy; use postgres_ffi::WAL_SEGMENT_SIZE; use safekeeper_api::models::SkTimelineInfo; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::collections::{HashMap, HashSet}; use std::fmt; use std::str::FromStr; @@ -67,11 +66,9 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { /// Same as TermLsn, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. -#[serde_as] #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct TermSwitchApiEntry { pub term: Term, - #[serde_as(as = "DisplayFromStr")] pub lsn: Lsn, } @@ -93,28 +90,18 @@ pub struct AcceptorStateStatus { } /// Info about timeline on safekeeper ready for reporting. -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct TimelineStatus { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub acceptor_state: AcceptorStateStatus, pub pg_info: ServerInfo, - #[serde_as(as = "DisplayFromStr")] pub flush_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub timeline_start_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub local_start_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub backup_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub peer_horizon_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, pub peers: Vec, pub walsenders: Vec, diff --git a/safekeeper/src/json_ctrl.rs b/safekeeper/src/json_ctrl.rs index 2ad2ca7706..303bdd67fe 100644 --- a/safekeeper/src/json_ctrl.rs +++ b/safekeeper/src/json_ctrl.rs @@ -44,8 +44,11 @@ pub struct AppendLogicalMessage { // fields from AppendRequestHeader pub term: Term, + #[serde(with = "utils::lsn::serde_as_u64")] pub epoch_start_lsn: Lsn, + #[serde(with = "utils::lsn::serde_as_u64")] pub begin_lsn: Lsn, + #[serde(with = "utils::lsn::serde_as_u64")] pub truncate_lsn: Lsn, pub pg_version: u32, } diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index e2f1b9fcff..ad3a18a536 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -6,8 +6,6 @@ use tokio::io::AsyncWriteExt; use tracing::info; use utils::id::{TenantId, TenantTimelineId, TimelineId}; -use serde_with::{serde_as, DisplayFromStr}; - use crate::{ control_file, debug_dump, http::routes::TimelineStatus, @@ -16,12 +14,9 @@ use crate::{ }; /// Info about timeline on safekeeper ready for reporting. -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct Request { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub http_hosts: Vec, } diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 1437bdb50e..47a624281d 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -52,7 +52,7 @@ impl From<(Term, Lsn)> for TermLsn { } } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq)] pub struct TermHistory(pub Vec); impl TermHistory { @@ -178,7 +178,7 @@ impl fmt::Debug for TermHistory { pub type PgUuid = [u8; 16]; /// Persistent consensus state of the acceptor. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct AcceptorState { /// acceptor's last term it voted for (advanced in 1 phase) pub term: Term, @@ -209,16 +209,16 @@ pub struct ServerInfo { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct PersistedPeerInfo { /// LSN up to which safekeeper offloaded WAL to s3. - backup_lsn: Lsn, + pub backup_lsn: Lsn, /// Term of the last entry. - term: Term, + pub term: Term, /// LSN of the last record. - flush_lsn: Lsn, + pub flush_lsn: Lsn, /// Up to which LSN safekeeper regards its WAL as committed. - commit_lsn: Lsn, + pub commit_lsn: Lsn, } impl PersistedPeerInfo { @@ -232,12 +232,12 @@ impl PersistedPeerInfo { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct PersistedPeers(pub Vec<(NodeId, PersistedPeerInfo)>); /// Persistent information stored on safekeeper node /// On disk data is prefixed by magic and format version and followed by checksum. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperState { #[serde(with = "hex")] pub tenant_id: TenantId, @@ -1096,7 +1096,7 @@ mod tests { use super::*; use crate::wal_storage::Storage; - use std::{ops::Deref, time::Instant}; + use std::{ops::Deref, str::FromStr, time::Instant}; // fake storage for tests struct InMemoryState { @@ -1314,4 +1314,98 @@ mod tests { }) ); } + + #[test] + fn test_sk_state_bincode_serde_roundtrip() { + use utils::Hex; + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperState { + tenant_id, + timeline_id, + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfo { + pg_version: 14, + system_id: 0x1234567887654321, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + timeline_start_lsn: Lsn(0x12345600), + local_start_lsn: Lsn(0x12), + commit_lsn: Lsn(1234567800), + backup_lsn: Lsn(1234567300), + peer_horizon_lsn: Lsn(9999999), + remote_consistent_lsn: Lsn(1234560000), + peers: PersistedPeers(vec![( + NodeId(1), + PersistedPeerInfo { + backup_lsn: Lsn(1234567000), + term: 42, + flush_lsn: Lsn(1234567800 - 8), + commit_lsn: Lsn(1234567600), + }, + )]), + }; + + let ser = state.ser().unwrap(); + + #[rustfmt::skip] + let expected = [ + // tenant_id as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x66, 0x30, 0x34, 0x38, 0x30, 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, 0x32, 0x33, 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, + // timeline_id as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, 0x36, 0x36, 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, 0x34, 0x34, 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, + // term + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // length prefix + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // unsure why this order is swapped + 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // pg_version + 0x0e, 0x00, 0x00, 0x00, + // systemid + 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, + // wal_seg_size + 0x78, 0x56, 0x34, 0x12, + // pguuid as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x34, 0x37, 0x61, 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + + // timeline_start_lsn + 0x00, 0x56, 0x34, 0x12, 0x00, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x00, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0x96, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe4, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + // length prefix for persistentpeers + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // nodeid + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // backuplsn + 0x58, 0xff, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0xb0, 0x01, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperState::des(&ser).unwrap(); + + assert_eq!(deser, state); + } } diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index cd765dcbce..44f14f8c7e 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -16,7 +16,6 @@ use postgres_ffi::get_current_timestamp; use postgres_ffi::{TimestampTz, MAX_SEND_SIZE}; use pq_proto::{BeMessage, WalSndKeepAlive, XLogDataBody}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use tokio::io::{AsyncRead, AsyncWrite}; use utils::id::TenantTimelineId; use utils::lsn::AtomicLsn; @@ -313,10 +312,8 @@ impl WalSendersShared { } // Serialized is used only for pretty printing in json. -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WalSenderState { - #[serde_as(as = "DisplayFromStr")] ttid: TenantTimelineId, addr: SocketAddr, conn_id: ConnectionId, diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 96d844fa7a..2ba871207e 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -5,10 +5,8 @@ use anyhow::{anyhow, bail, Result}; use camino::Utf8PathBuf; use postgres_ffi::XLogSegNo; use serde::{Deserialize, Serialize}; -use serde_with::serde_as; use tokio::fs; -use serde_with::DisplayFromStr; use std::cmp::max; use std::sync::Arc; use std::time::Duration; @@ -42,7 +40,6 @@ use crate::SafeKeeperConf; use crate::{debug_dump, wal_storage}; /// Things safekeeper should know about timeline state on peers. -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerInfo { pub sk_id: NodeId, @@ -50,13 +47,10 @@ pub struct PeerInfo { /// Term of the last entry. pub last_log_term: Term, /// LSN of the last record. - #[serde_as(as = "DisplayFromStr")] pub flush_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, /// Since which LSN safekeeper has WAL. TODO: remove this once we fill new /// sk since backup_lsn. - #[serde_as(as = "DisplayFromStr")] pub local_start_lsn: Lsn, /// When info was received. Serde annotations are not very useful but make /// the code compile -- we don't rely on this field externally. From 6defa2b5d551dbc4c45e47a28c38e66d9ff33cf5 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 6 Nov 2023 12:39:20 +0000 Subject: [PATCH 24/29] pageserver: add `Gate` as a partner to CancellationToken for safe shutdown of `Tenant` & `Timeline` (#5711) ## Problem When shutting down a Tenant, it isn't just important to cause any background tasks to stop. It's also important to wait until they have stopped before declaring shutdown complete, in cases where we may re-use the tenant's local storage for something else, such as running in secondary mode, or creating a new tenant with the same ID. ## Summary of changes A `Gate` class is added, inspired by [seastar::gate](https://docs.seastar.io/master/classseastar_1_1gate.html). For types that have an important lifetime that corresponds to some physical resource, use of a Gate as well as a CancellationToken provides a robust pattern for async requests & shutdown: - Requests must always acquire the gate as long as they are using the object - Shutdown must set the cancellation token, and then `close()` the gate to wait for requests in progress before returning. This is not for memory safety: it's for expressing the difference between "Arc exists", and "This tenant's files on disk are eligible to be read/written". - Both Tenant and Timeline get a Gate & CancellationToken. - The Timeline gate is held during eviction of layers, and during page_service requests. - Existing cancellation support in page_service is refined to use the timeline-scope cancellation token instead of a process-scope cancellation token. This replaces the use of `task_mgr::associate_with`: tasks no longer change their tenant/timelineidentity after being spawned. The Tenant's Gate is not yet used, but will be important for Tenant-scoped operations in secondary mode, where we must ensure that our secondary-mode downloads for a tenant are gated wrt the activity of an attached Tenant. This is part of a broader move away from using the global-state driven `task_mgr` shutdown tokens: - less global state where we rely on implicit knowledge of what task a given function is running in, and more explicit references to the cancellation token that a particular function/type will respect, making shutdown easier to reason about. - eventually avoid the big global TASKS mutex. --------- Co-authored-by: Joonas Koivunen --- libs/postgres_backend/src/lib.rs | 17 +- libs/utils/src/sync.rs | 2 + libs/utils/src/sync/gate.rs | 151 ++++++++++++++++++ pageserver/src/disk_usage_eviction_task.rs | 7 +- pageserver/src/http/routes.rs | 3 + pageserver/src/lib.rs | 17 +- pageserver/src/page_service.rs | 69 ++++---- pageserver/src/pgdatadir_mapping.rs | 18 ++- pageserver/src/task_mgr.rs | 47 ++---- pageserver/src/tenant.rs | 30 ++++ pageserver/src/tenant/size.rs | 10 +- pageserver/src/tenant/timeline.rs | 76 ++++++--- pageserver/src/tenant/timeline/delete.rs | 11 ++ .../src/tenant/timeline/eviction_task.rs | 5 +- .../walreceiver/connection_manager.rs | 11 +- ...test_pageserver_restarts_under_workload.py | 4 + 16 files changed, 358 insertions(+), 120 deletions(-) create mode 100644 libs/utils/src/sync/gate.rs diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index ad3af4e794..455fe7a481 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -728,12 +728,17 @@ impl PostgresBackend { trace!("got query {query_string:?}"); if let Err(e) = handler.process_query(self, query_string).await { - log_query_error(query_string, &e); - let short_error = short_error(&e); - self.write_message_noflush(&BeMessage::ErrorResponse( - &short_error, - Some(e.pg_error_code()), - ))?; + match e { + QueryError::Shutdown => return Ok(ProcessMsgResult::Break), + e => { + log_query_error(query_string, &e); + let short_error = short_error(&e); + self.write_message_noflush(&BeMessage::ErrorResponse( + &short_error, + Some(e.pg_error_code()), + ))?; + } + } } self.write_message_noflush(&BeMessage::ReadyForQuery)?; } diff --git a/libs/utils/src/sync.rs b/libs/utils/src/sync.rs index 125eeca129..2ee8f35449 100644 --- a/libs/utils/src/sync.rs +++ b/libs/utils/src/sync.rs @@ -1 +1,3 @@ pub mod heavier_once_cell; + +pub mod gate; diff --git a/libs/utils/src/sync/gate.rs b/libs/utils/src/sync/gate.rs new file mode 100644 index 0000000000..1391d238e6 --- /dev/null +++ b/libs/utils/src/sync/gate.rs @@ -0,0 +1,151 @@ +use std::{sync::Arc, time::Duration}; + +/// Gates are a concurrency helper, primarily used for implementing safe shutdown. +/// +/// Users of a resource call `enter()` to acquire a GateGuard, and the owner of +/// the resource calls `close()` when they want to ensure that all holders of guards +/// have released them, and that no future guards will be issued. +pub struct Gate { + /// Each caller of enter() takes one unit from the semaphore. In close(), we + /// take all the units to ensure all GateGuards are destroyed. + sem: Arc, + + /// For observability only: a name that will be used to log warnings if a particular + /// gate is holding up shutdown + name: String, +} + +/// RAII guard for a [`Gate`]: as long as this exists, calls to [`Gate::close`] will +/// not complete. +#[derive(Debug)] +pub struct GateGuard(tokio::sync::OwnedSemaphorePermit); + +/// Observability helper: every `warn_period`, emit a log warning that we're still waiting on this gate +async fn warn_if_stuck( + fut: Fut, + name: &str, + warn_period: std::time::Duration, +) -> ::Output { + let started = std::time::Instant::now(); + + let mut fut = std::pin::pin!(fut); + + loop { + match tokio::time::timeout(warn_period, &mut fut).await { + Ok(ret) => return ret, + Err(_) => { + tracing::warn!( + gate = name, + elapsed_ms = started.elapsed().as_millis(), + "still waiting, taking longer than expected..." + ); + } + } + } +} + +#[derive(Debug)] +pub enum GateError { + GateClosed, +} + +impl Gate { + const MAX_UNITS: u32 = u32::MAX; + + pub fn new(name: String) -> Self { + Self { + sem: Arc::new(tokio::sync::Semaphore::new(Self::MAX_UNITS as usize)), + name, + } + } + + /// Acquire a guard that will prevent close() calls from completing. If close() + /// was already called, this will return an error which should be interpreted + /// as "shutting down". + /// + /// This function would typically be used from e.g. request handlers. While holding + /// the guard returned from this function, it is important to respect a CancellationToken + /// to avoid blocking close() indefinitely: typically types that contain a Gate will + /// also contain a CancellationToken. + pub fn enter(&self) -> Result { + self.sem + .clone() + .try_acquire_owned() + .map(GateGuard) + .map_err(|_| GateError::GateClosed) + } + + /// Types with a shutdown() method and a gate should call this method at the + /// end of shutdown, to ensure that all GateGuard holders are done. + /// + /// This will wait for all guards to be destroyed. For this to complete promptly, it is + /// important that the holders of such guards are respecting a CancellationToken which has + /// been cancelled before entering this function. + pub async fn close(&self) { + warn_if_stuck(self.do_close(), &self.name, Duration::from_millis(1000)).await + } + + async fn do_close(&self) { + tracing::debug!(gate = self.name, "Closing Gate..."); + match self.sem.acquire_many(Self::MAX_UNITS).await { + Ok(_units) => { + // While holding all units, close the semaphore. All subsequent calls to enter() will fail. + self.sem.close(); + } + Err(_) => { + // Semaphore closed: we are the only function that can do this, so it indicates a double-call. + // This is legal. Timeline::shutdown for example is not protected from being called more than + // once. + tracing::debug!(gate = self.name, "Double close") + } + } + tracing::debug!(gate = self.name, "Closed Gate.") + } +} + +#[cfg(test)] +mod tests { + use futures::FutureExt; + + use super::*; + + #[tokio::test] + async fn test_idle_gate() { + // Having taken no gates, we should not be blocked in close + let gate = Gate::new("test".to_string()); + gate.close().await; + + // If a guard is dropped before entering, close should not be blocked + let gate = Gate::new("test".to_string()); + let guard = gate.enter().unwrap(); + drop(guard); + gate.close().await; + + // Entering a closed guard fails + gate.enter().expect_err("enter should fail after close"); + } + + #[tokio::test] + async fn test_busy_gate() { + let gate = Gate::new("test".to_string()); + + let guard = gate.enter().unwrap(); + + let mut close_fut = std::pin::pin!(gate.close()); + + // Close should be blocked + assert!(close_fut.as_mut().now_or_never().is_none()); + + // Attempting to enter() should fail, even though close isn't done yet. + gate.enter() + .expect_err("enter should fail after entering close"); + + drop(guard); + + // Guard is gone, close should finish + assert!(close_fut.as_mut().now_or_never().is_some()); + + // Attempting to enter() is still forbidden + gate.enter().expect_err("enter should fail finishing close"); + } +} diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index 413c941bc4..bc4bf51862 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -403,7 +403,7 @@ pub async fn disk_usage_eviction_task_iteration_impl( return (evicted_bytes, evictions_failed); }; - let results = timeline.evict_layers(&batch, &cancel).await; + let results = timeline.evict_layers(&batch).await; match results { Ok(results) => { @@ -554,6 +554,11 @@ async fn collect_eviction_candidates( } }; + if tenant.cancel.is_cancelled() { + info!(%tenant_id, "Skipping tenant for eviction, it is shutting down"); + continue; + } + // collect layers from all timelines in this tenant // // If one of the timelines becomes `!is_active()` during the iteration, diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index a6fb26b298..db728f243e 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -396,6 +396,9 @@ async fn timeline_create_handler( Err(e @ tenant::CreateTimelineError::AncestorNotActive) => { json_response(StatusCode::SERVICE_UNAVAILABLE, HttpErrorBody::from_msg(e.to_string())) } + Err(tenant::CreateTimelineError::ShuttingDown) => { + json_response(StatusCode::SERVICE_UNAVAILABLE,HttpErrorBody::from_msg("tenant shutting down".to_string())) + } Err(tenant::CreateTimelineError::Other(err)) => Err(ApiError::InternalServerError(err)), } } diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index e71130e8af..04c3ae9746 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -61,14 +61,6 @@ pub async fn shutdown_pageserver(deletion_queue: Option, exit_cod ) .await; - // Shut down any page service tasks. - timed( - task_mgr::shutdown_tasks(Some(TaskKind::PageRequestHandler), None, None), - "shutdown PageRequestHandlers", - Duration::from_secs(1), - ) - .await; - // Shut down all the tenants. This flushes everything to disk and kills // the checkpoint and GC tasks. timed( @@ -78,6 +70,15 @@ pub async fn shutdown_pageserver(deletion_queue: Option, exit_cod ) .await; + // Shut down any page service tasks: any in-progress work for particular timelines or tenants + // should already have been canclled via mgr::shutdown_all_tenants + timed( + task_mgr::shutdown_tasks(Some(TaskKind::PageRequestHandler), None, None), + "shutdown PageRequestHandlers", + Duration::from_secs(1), + ) + .await; + // Best effort to persist any outstanding deletions, to avoid leaking objects if let Some(mut deletion_queue) = deletion_queue { deletion_queue.shutdown(Duration::from_secs(5)).await; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 536334d051..334aee3dd1 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -223,13 +223,7 @@ async fn page_service_conn_main( // and create a child per-query context when it invokes process_query. // But it's in a shared crate, so, we store connection_ctx inside PageServerHandler // and create the per-query context in process_query ourselves. - let mut conn_handler = PageServerHandler::new( - conf, - broker_client, - auth, - connection_ctx, - task_mgr::shutdown_token(), - ); + let mut conn_handler = PageServerHandler::new(conf, broker_client, auth, connection_ctx); let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; match pgbackend @@ -263,10 +257,6 @@ struct PageServerHandler { /// For each query received over the connection, /// `process_query` creates a child context from this one. connection_ctx: RequestContext, - - /// A token that should fire when the tenant transitions from - /// attached state, or when the pageserver is shutting down. - cancel: CancellationToken, } impl PageServerHandler { @@ -275,7 +265,6 @@ impl PageServerHandler { broker_client: storage_broker::BrokerClientChannel, auth: Option>, connection_ctx: RequestContext, - cancel: CancellationToken, ) -> Self { PageServerHandler { _conf: conf, @@ -283,7 +272,6 @@ impl PageServerHandler { auth, claims: None, connection_ctx, - cancel, } } @@ -291,7 +279,11 @@ impl PageServerHandler { /// this rather than naked flush() in order to shut down promptly. Without this, we would /// block shutdown of a tenant if a postgres client was failing to consume bytes we send /// in the flush. - async fn flush_cancellable(&self, pgb: &mut PostgresBackend) -> Result<(), QueryError> + async fn flush_cancellable( + &self, + pgb: &mut PostgresBackend, + cancel: &CancellationToken, + ) -> Result<(), QueryError> where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { @@ -299,7 +291,7 @@ impl PageServerHandler { flush_r = pgb.flush() => { Ok(flush_r?) }, - _ = self.cancel.cancelled() => { + _ = cancel.cancelled() => { Err(QueryError::Shutdown) } ) @@ -308,6 +300,7 @@ impl PageServerHandler { fn copyin_stream<'a, IO>( &'a self, pgb: &'a mut PostgresBackend, + cancel: &'a CancellationToken, ) -> impl Stream> + 'a where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, @@ -317,7 +310,7 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = self.cancel.cancelled() => { + _ = cancel.cancelled() => { // We were requested to shut down. let msg = "pageserver is shutting down"; let _ = pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, None)); @@ -357,7 +350,7 @@ impl PageServerHandler { let query_error = QueryError::Disconnected(ConnectionError::Io(io::Error::new(io::ErrorKind::ConnectionReset, msg))); // error can't happen here, ErrorResponse serialization should be always ok pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, Some(query_error.pg_error_code()))).map_err(|e| e.into_io_error())?; - self.flush_cancellable(pgb).await.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + self.flush_cancellable(pgb, cancel).await.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?; } Err(QueryError::Disconnected(ConnectionError::Io(io_error))) => { @@ -384,10 +377,6 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - // NOTE: pagerequests handler exits when connection is closed, - // so there is no need to reset the association - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); - // Make request tracer if needed let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; let mut tracer = if tenant.get_trace_read_requests() { @@ -405,9 +394,14 @@ impl PageServerHandler { .get_timeline(timeline_id, true) .map_err(|e| anyhow::anyhow!(e))?; + // Avoid starting new requests if the timeline has already started shutting down, + // and block timeline shutdown until this request is complete, or drops out due + // to cancellation. + let _timeline_guard = timeline.gate.enter().map_err(|_| QueryError::Shutdown)?; + // switch client to COPYBOTH pgb.write_message_noflush(&BeMessage::CopyBothResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; let metrics = metrics::SmgrQueryTimePerTimeline::new(&tenant_id, &timeline_id); @@ -415,7 +409,7 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = self.cancel.cancelled() => { + _ = timeline.cancel.cancelled() => { // We were requested to shut down. info!("shutdown request received in page handler"); return Err(QueryError::Shutdown) @@ -490,9 +484,20 @@ impl PageServerHandler { } }; + if let Err(e) = &response { + if timeline.cancel.is_cancelled() { + // If we fail to fulfil a request during shutdown, which may be _because_ of + // shutdown, then do not send the error to the client. Instead just drop the + // connection. + span.in_scope(|| info!("dropped response during shutdown: {e:#}")); + return Err(QueryError::Shutdown); + } + } + let response = response.unwrap_or_else(|e| { // print the all details to the log with {:#}, but for the client the - // error message is enough + // error message is enough. Do not log if shutting down, as the anyhow::Error + // here includes cancellation which is not an error. span.in_scope(|| error!("error reading relation or page version: {:#}", e)); PagestreamBeMessage::Error(PagestreamErrorResponse { message: e.to_string(), @@ -500,7 +505,7 @@ impl PageServerHandler { }); pgb.write_message_noflush(&BeMessage::CopyData(&response.serialize()))?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; } Ok(()) } @@ -522,7 +527,6 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); // Create empty timeline info!("creating new timeline"); let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; @@ -543,9 +547,9 @@ impl PageServerHandler { // Import basebackup provided via CopyData info!("importing basebackup"); pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &tenant.cancel).await?; - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb))); + let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &tenant.cancel))); timeline .import_basebackup_from_tar( &mut copyin_reader, @@ -582,7 +586,6 @@ impl PageServerHandler { IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { debug_assert_current_span_has_tenant_and_timeline_id(); - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; let last_record_lsn = timeline.get_last_record_lsn(); @@ -598,8 +601,8 @@ impl PageServerHandler { // Import wal provided via CopyData info!("importing wal"); pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb).await?; - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb))); + self.flush_cancellable(pgb, &timeline.cancel).await?; + let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &timeline.cancel))); import_wal_from_tar(&timeline, &mut copyin_reader, start_lsn, end_lsn, &ctx).await?; info!("wal import complete"); @@ -807,7 +810,7 @@ impl PageServerHandler { // switch client to COPYOUT pgb.write_message_noflush(&BeMessage::CopyOutResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; // Send a tarball of the latest layer on the timeline. Compress if not // fullbackup. TODO Compress in that case too (tests need to be updated) @@ -859,7 +862,7 @@ impl PageServerHandler { } pgb.write_message_noflush(&BeMessage::CopyDone)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; let basebackup_after = started .elapsed() diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index daadf6abd4..88974588d4 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -44,6 +44,17 @@ pub enum CalculateLogicalSizeError { Other(#[from] anyhow::Error), } +impl From for CalculateLogicalSizeError { + fn from(pre: PageReconstructError) -> Self { + match pre { + PageReconstructError::AncestorStopping(_) | PageReconstructError::Cancelled => { + Self::Cancelled + } + _ => Self::Other(pre.into()), + } + } +} + #[derive(Debug, thiserror::Error)] pub enum RelationError { #[error("Relation Already Exists")] @@ -573,7 +584,7 @@ impl Timeline { crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id(); // Fetch list of database dirs and iterate them - let buf = self.get(DBDIR_KEY, lsn, ctx).await.context("read dbdir")?; + let buf = self.get(DBDIR_KEY, lsn, ctx).await?; let dbdir = DbDirectory::des(&buf).context("deserialize db directory")?; let mut total_size: u64 = 0; @@ -587,10 +598,7 @@ impl Timeline { return Err(CalculateLogicalSizeError::Cancelled); } let relsize_key = rel_size_to_key(rel); - let mut buf = self - .get(relsize_key, lsn, ctx) - .await - .with_context(|| format!("read relation size of {rel:?}"))?; + let mut buf = self.get(relsize_key, lsn, ctx).await?; let relsize = buf.get_u32_le(); total_size += relsize as u64; diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 017322ffb2..4270b6edb0 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -299,10 +299,6 @@ pub enum TaskKind { #[derive(Default)] struct MutableTaskState { - /// Tenant and timeline that this task is associated with. - tenant_id: Option, - timeline_id: Option, - /// Handle for waiting for the task to exit. It can be None, if the /// the task has already exited. join_handle: Option>, @@ -319,6 +315,11 @@ struct PageServerTask { // To request task shutdown, just cancel this token. cancel: CancellationToken, + /// Tasks may optionally be launched for a particular tenant/timeline, enabling + /// later cancelling tasks for that tenant/timeline in [`shutdown_tasks`] + tenant_id: Option, + timeline_id: Option, + mutable: Mutex, } @@ -344,11 +345,9 @@ where kind, name: name.to_string(), cancel: cancel.clone(), - mutable: Mutex::new(MutableTaskState { - tenant_id, - timeline_id, - join_handle: None, - }), + tenant_id, + timeline_id, + mutable: Mutex::new(MutableTaskState { join_handle: None }), }); TASKS.lock().unwrap().insert(task_id, Arc::clone(&task)); @@ -418,8 +417,6 @@ async fn task_finish( let mut shutdown_process = false; { - let task_mut = task.mutable.lock().unwrap(); - match result { Ok(Ok(())) => { debug!("Task '{}' exited normally", task_name); @@ -428,13 +425,13 @@ async fn task_finish( if shutdown_process_on_error { error!( "Shutting down: task '{}' tenant_id: {:?}, timeline_id: {:?} exited with error: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); shutdown_process = true; } else { error!( "Task '{}' tenant_id: {:?}, timeline_id: {:?} exited with error: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); } } @@ -442,13 +439,13 @@ async fn task_finish( if shutdown_process_on_error { error!( "Shutting down: task '{}' tenant_id: {:?}, timeline_id: {:?} panicked: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); shutdown_process = true; } else { error!( "Task '{}' tenant_id: {:?}, timeline_id: {:?} panicked: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); } } @@ -460,17 +457,6 @@ async fn task_finish( } } -// expected to be called from the task of the given id. -pub fn associate_with(tenant_id: Option, timeline_id: Option) { - CURRENT_TASK.with(|ct| { - let mut task_mut = ct.mutable.lock().unwrap(); - task_mut.tenant_id = tenant_id; - task_mut.timeline_id = timeline_id; - }); -} - -/// Is there a task running that matches the criteria - /// Signal and wait for tasks to shut down. /// /// @@ -493,17 +479,16 @@ pub async fn shutdown_tasks( { let tasks = TASKS.lock().unwrap(); for task in tasks.values() { - let task_mut = task.mutable.lock().unwrap(); if (kind.is_none() || Some(task.kind) == kind) - && (tenant_id.is_none() || task_mut.tenant_id == tenant_id) - && (timeline_id.is_none() || task_mut.timeline_id == timeline_id) + && (tenant_id.is_none() || task.tenant_id == tenant_id) + && (timeline_id.is_none() || task.timeline_id == timeline_id) { task.cancel.cancel(); victim_tasks.push(( Arc::clone(task), task.kind, - task_mut.tenant_id, - task_mut.timeline_id, + task.tenant_id, + task.timeline_id, )); } } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 3a426ac87b..55815a4956 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -26,6 +26,7 @@ use tracing::*; use utils::completion; use utils::crashsafe::path_with_suffix_extension; use utils::fs_ext; +use utils::sync::gate::Gate; use std::cmp::min; use std::collections::hash_map::Entry; @@ -252,6 +253,14 @@ pub struct Tenant { eviction_task_tenant_state: tokio::sync::Mutex, pub(crate) delete_progress: Arc>, + + // Cancellation token fires when we have entered shutdown(). This is a parent of + // Timelines' cancellation token. + pub(crate) cancel: CancellationToken, + + // Users of the Tenant such as the page service must take this Gate to avoid + // trying to use a Tenant which is shutting down. + pub(crate) gate: Gate, } pub(crate) enum WalRedoManager { @@ -395,6 +404,8 @@ pub enum CreateTimelineError { AncestorLsn(anyhow::Error), #[error("ancestor timeline is not active")] AncestorNotActive, + #[error("tenant shutting down")] + ShuttingDown, #[error(transparent)] Other(#[from] anyhow::Error), } @@ -1524,6 +1535,11 @@ impl Tenant { ))); } + let _gate = self + .gate + .enter() + .map_err(|_| CreateTimelineError::ShuttingDown)?; + if let Ok(existing) = self.get_timeline(new_timeline_id, false) { debug!("timeline {new_timeline_id} already exists"); @@ -1808,6 +1824,7 @@ impl Tenant { freeze_and_flush: bool, ) -> Result<(), completion::Barrier> { span::debug_assert_current_span_has_tenant_id(); + // Set tenant (and its timlines) to Stoppping state. // // Since we can only transition into Stopping state after activation is complete, @@ -1846,6 +1863,7 @@ impl Tenant { js.spawn(async move { timeline.shutdown(freeze_and_flush).instrument(span).await }); }) }; + tracing::info!("Waiting for timelines..."); while let Some(res) = js.join_next().await { match res { Ok(()) => {} @@ -1855,12 +1873,21 @@ impl Tenant { } } + // We cancel the Tenant's cancellation token _after_ the timelines have all shut down. This permits + // them to continue to do work during their shutdown methods, e.g. flushing data. + tracing::debug!("Cancelling CancellationToken"); + self.cancel.cancel(); + // shutdown all tenant and timeline tasks: gc, compaction, page service // No new tasks will be started for this tenant because it's in `Stopping` state. // // this will additionally shutdown and await all timeline tasks. + tracing::debug!("Waiting for tasks..."); task_mgr::shutdown_tasks(None, Some(self.tenant_id), None).await; + // Wait for any in-flight operations to complete + self.gate.close().await; + Ok(()) } @@ -2267,6 +2294,7 @@ impl Tenant { initial_logical_size_can_start.cloned(), initial_logical_size_attempt.cloned().flatten(), state, + self.cancel.child_token(), ); Ok(timeline) @@ -2356,6 +2384,8 @@ impl Tenant { cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)), eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()), delete_progress: Arc::new(tokio::sync::Mutex::new(DeleteTenantFlow::default())), + cancel: CancellationToken::default(), + gate: Gate::new(format!("Tenant<{tenant_id}>")), } } diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index 5e8ee2b99e..e4df94b8e9 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -406,10 +406,12 @@ async fn fill_logical_sizes( have_any_error = true; } Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Err(error)))) => { - warn!( - timeline_id=%timeline.timeline_id, - "failed to calculate logical size at {lsn}: {error:#}" - ); + if !matches!(error, CalculateLogicalSizeError::Cancelled) { + warn!( + timeline_id=%timeline.timeline_id, + "failed to calculate logical size at {lsn}: {error:#}" + ); + } have_any_error = true; } Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Ok(size)))) => { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index baa97b6cf9..36629e0655 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -23,7 +23,7 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::id::TenantTimelineId; +use utils::{id::TenantTimelineId, sync::gate::Gate}; use std::cmp::{max, min, Ordering}; use std::collections::{BinaryHeap, HashMap, HashSet}; @@ -310,6 +310,13 @@ pub struct Timeline { /// Load or creation time information about the disk_consistent_lsn and when the loading /// happened. Used for consumption metrics. pub(crate) loaded_at: (Lsn, SystemTime), + + /// Gate to prevent shutdown completing while I/O is still happening to this timeline's data + pub(crate) gate: Gate, + + /// Cancellation token scoped to this timeline: anything doing long-running work relating + /// to the timeline should drop out when this token fires. + pub(crate) cancel: CancellationToken, } pub struct WalReceiverInfo { @@ -786,7 +793,11 @@ impl Timeline { // as an empty timeline. Also in unit tests, when we use the timeline // as a simple key-value store, ignoring the datadir layout. Log the // error but continue. - error!("could not compact, repartitioning keyspace failed: {err:?}"); + // + // Suppress error when it's due to cancellation + if !self.cancel.is_cancelled() { + error!("could not compact, repartitioning keyspace failed: {err:?}"); + } } }; @@ -884,7 +895,12 @@ impl Timeline { pub async fn shutdown(self: &Arc, freeze_and_flush: bool) { debug_assert_current_span_has_tenant_and_timeline_id(); + // Signal any subscribers to our cancellation token to drop out + tracing::debug!("Cancelling CancellationToken"); + self.cancel.cancel(); + // prevent writes to the InMemoryLayer + tracing::debug!("Waiting for WalReceiverManager..."); task_mgr::shutdown_tasks( Some(TaskKind::WalReceiverManager), Some(self.tenant_id), @@ -920,6 +936,16 @@ impl Timeline { warn!("failed to await for frozen and flushed uploads: {e:#}"); } } + + // Page request handlers might be waiting for LSN to advance: they do not respect Timeline::cancel + // while doing so. + self.last_record_lsn.shutdown(); + + tracing::debug!("Waiting for tasks..."); + task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(self.timeline_id)).await; + + // Finally wait until any gate-holders are complete + self.gate.close().await; } pub fn set_state(&self, new_state: TimelineState) { @@ -1048,6 +1074,11 @@ impl Timeline { /// Like [`evict_layer_batch`](Self::evict_layer_batch), but for just one layer. /// Additional case `Ok(None)` covers the case where the layer could not be found by its `layer_file_name`. pub async fn evict_layer(&self, layer_file_name: &str) -> anyhow::Result> { + let _gate = self + .gate + .enter() + .map_err(|_| anyhow::anyhow!("Shutting down"))?; + let Some(local_layer) = self.find_layer(layer_file_name).await else { return Ok(None); }; @@ -1063,9 +1094,8 @@ impl Timeline { .as_ref() .ok_or_else(|| anyhow::anyhow!("remote storage not configured; cannot evict"))?; - let cancel = CancellationToken::new(); let results = self - .evict_layer_batch(remote_client, &[local_layer], &cancel) + .evict_layer_batch(remote_client, &[local_layer]) .await?; assert_eq!(results.len(), 1); let result: Option> = results.into_iter().next().unwrap(); @@ -1080,15 +1110,18 @@ impl Timeline { pub(crate) async fn evict_layers( &self, layers_to_evict: &[Layer], - cancel: &CancellationToken, ) -> anyhow::Result>>> { + let _gate = self + .gate + .enter() + .map_err(|_| anyhow::anyhow!("Shutting down"))?; + let remote_client = self .remote_client .as_ref() .context("timeline must have RemoteTimelineClient")?; - self.evict_layer_batch(remote_client, layers_to_evict, cancel) - .await + self.evict_layer_batch(remote_client, layers_to_evict).await } /// Evict multiple layers at once, continuing through errors. @@ -1109,7 +1142,6 @@ impl Timeline { &self, remote_client: &Arc, layers_to_evict: &[Layer], - cancel: &CancellationToken, ) -> anyhow::Result>>> { // ensure that the layers have finished uploading // (don't hold the layer_removal_cs while we do it, we're not removing anything yet) @@ -1157,7 +1189,7 @@ impl Timeline { }; tokio::select! { - _ = cancel.cancelled() => {}, + _ = self.cancel.cancelled() => {}, _ = join => {} } @@ -1267,6 +1299,7 @@ impl Timeline { initial_logical_size_can_start: Option, initial_logical_size_attempt: Option, state: TimelineState, + cancel: CancellationToken, ) -> Arc { let disk_consistent_lsn = metadata.disk_consistent_lsn(); let (state, _) = watch::channel(state); @@ -1367,6 +1400,8 @@ impl Timeline { initial_logical_size_can_start, initial_logical_size_attempt: Mutex::new(initial_logical_size_attempt), + cancel, + gate: Gate::new(format!("Timeline<{tenant_id}/{timeline_id}>")), }; result.repartition_threshold = result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; @@ -2030,6 +2065,10 @@ impl Timeline { let mut cont_lsn = Lsn(request_lsn.0 + 1); 'outer: loop { + if self.cancel.is_cancelled() { + return Err(PageReconstructError::Cancelled); + } + // The function should have updated 'state' //info!("CALLED for {} at {}: {:?} with {} records, cached {}", key, cont_lsn, result, reconstruct_state.records.len(), cached_lsn); match result { @@ -4366,25 +4405,10 @@ mod tests { .expect("should had been resident") .drop_eviction_guard(); - let cancel = tokio_util::sync::CancellationToken::new(); let batch = [layer]; - let first = { - let cancel = cancel.child_token(); - async { - let cancel = cancel; - timeline - .evict_layer_batch(&rc, &batch, &cancel) - .await - .unwrap() - } - }; - let second = async { - timeline - .evict_layer_batch(&rc, &batch, &cancel) - .await - .unwrap() - }; + let first = async { timeline.evict_layer_batch(&rc, &batch).await.unwrap() }; + let second = async { timeline.evict_layer_batch(&rc, &batch).await.unwrap() }; let (first, second) = tokio::join!(first, second); diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 6d30664515..56a99a25cf 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -17,6 +17,7 @@ use crate::{ deletion_queue::DeletionQueueClient, task_mgr::{self, TaskKind}, tenant::{ + debug_assert_current_span_has_tenant_and_timeline_id, metadata::TimelineMetadata, remote_timeline_client::{ self, PersistIndexPartWithDeletedFlagError, RemoteTimelineClient, @@ -30,6 +31,11 @@ use super::{Timeline, TimelineResources}; /// Now that the Timeline is in Stopping state, request all the related tasks to shut down. async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { + debug_assert_current_span_has_tenant_and_timeline_id(); + // Notify any timeline work to drop out of loops/requests + tracing::debug!("Cancelling CancellationToken"); + timeline.cancel.cancel(); + // Stop the walreceiver first. debug!("waiting for wal receiver to shutdown"); let maybe_started_walreceiver = { timeline.walreceiver.lock().unwrap().take() }; @@ -74,6 +80,11 @@ async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { "failpoint: timeline-delete-before-index-deleted-at" ))? }); + + tracing::debug!("Waiting for gate..."); + timeline.gate.close().await; + tracing::debug!("Shutdown complete"); + Ok(()) } diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index dc5c71bbe1..52c53a5c3b 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -277,10 +277,7 @@ impl Timeline { Some(c) => c, }; - let results = match self - .evict_layer_batch(remote_client, &candidates, cancel) - .await - { + let results = match self.evict_layer_batch(remote_client, &candidates).await { Err(pre_err) => { stats.errors += candidates.len(); error!("could not do any evictions: {pre_err:#}"); diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index e28c4a5f69..3077712445 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -426,7 +426,7 @@ impl ConnectionManagerState { timeline, new_sk.wal_source_connconf, events_sender, - cancellation, + cancellation.clone(), connect_timeout, ctx, node_id, @@ -447,7 +447,14 @@ impl ConnectionManagerState { } WalReceiverError::Other(e) => { // give out an error to have task_mgr give it a really verbose logging - Err(e).context("walreceiver connection handling failure") + if cancellation.is_cancelled() { + // Ideally we would learn about this via some path other than Other, but + // that requires refactoring all the intermediate layers of ingest code + // that only emit anyhow::Error + Ok(()) + } else { + Err(e).context("walreceiver connection handling failure") + } } } } diff --git a/test_runner/regress/test_pageserver_restarts_under_workload.py b/test_runner/regress/test_pageserver_restarts_under_workload.py index 65569f3bac..d07b8dbe6b 100644 --- a/test_runner/regress/test_pageserver_restarts_under_workload.py +++ b/test_runner/regress/test_pageserver_restarts_under_workload.py @@ -17,6 +17,10 @@ def test_pageserver_restarts_under_worload(neon_simple_env: NeonEnv, pg_bin: PgB n_restarts = 10 scale = 10 + # Pageserver currently logs requests on non-active tenants at error level + # https://github.com/neondatabase/neon/issues/5784 + env.pageserver.allowed_errors.append(".* will not become active. Current state: Stopping.*") + def run_pgbench(connstr: str): log.info(f"Start a pgbench workload on pg {connstr}") pg_bin.run_capture(["pgbench", "-i", f"-s{scale}", connstr]) From dc725672882e6c0ae2e5afbc87ac00829a4d8ffe Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Mon, 6 Nov 2023 08:58:20 -0500 Subject: [PATCH 25/29] Layer flush minor speedup (#5765) Convert keys to `i128` before sorting --- pageserver/src/tenant.rs | 14 +- pageserver/src/tenant/disk_btree_test_data.rs | 4000 ++++++++--------- .../tenant/storage_layer/inmemory_layer.rs | 11 +- 3 files changed, 2015 insertions(+), 2010 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 55815a4956..ad538a82fd 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3722,7 +3722,7 @@ mod tests { use tokio_util::sync::CancellationToken; static TEST_KEY: Lazy = - Lazy::new(|| Key::from_slice(&hex!("112222222233333333444444445500000001"))); + Lazy::new(|| Key::from_slice(&hex!("010000000033333333444444445500000001"))); #[tokio::test] async fn test_basic() -> anyhow::Result<()> { @@ -3818,9 +3818,9 @@ mod tests { let writer = tline.writer().await; #[allow(non_snake_case)] - let TEST_KEY_A: Key = Key::from_hex("112222222233333333444444445500000001").unwrap(); + let TEST_KEY_A: Key = Key::from_hex("110000000033333333444444445500000001").unwrap(); #[allow(non_snake_case)] - let TEST_KEY_B: Key = Key::from_hex("112222222233333333444444445500000002").unwrap(); + let TEST_KEY_B: Key = Key::from_hex("110000000033333333444444445500000002").unwrap(); // Insert a value on the timeline writer @@ -4404,7 +4404,7 @@ mod tests { let mut keyspace = KeySpaceAccum::new(); - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut blknum = 0; for _ in 0..50 { for _ in 0..10000 { @@ -4450,7 +4450,7 @@ mod tests { const NUM_KEYS: usize = 1000; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut keyspace = KeySpaceAccum::new(); @@ -4531,7 +4531,7 @@ mod tests { const NUM_KEYS: usize = 1000; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut keyspace = KeySpaceAccum::new(); @@ -4622,7 +4622,7 @@ mod tests { const NUM_KEYS: usize = 100; const NUM_TLINES: usize = 50; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); // Track page mutation lsns across different timelines. let mut updated = [[Lsn(0); NUM_KEYS]; NUM_TLINES]; diff --git a/pageserver/src/tenant/disk_btree_test_data.rs b/pageserver/src/tenant/disk_btree_test_data.rs index 9462573f03..4f835f55af 100644 --- a/pageserver/src/tenant/disk_btree_test_data.rs +++ b/pageserver/src/tenant/disk_btree_test_data.rs @@ -10,2004 +10,2004 @@ use hex_literal::hex; /// by a delta layer, for evaluating how well the prefix compression works. #[rustfmt::skip] pub static TEST_DATA: [([u8; 26], u64); 2000] = [ - (hex!("0122222222333333334444444455000000000000000000000010"), 0x004001), - (hex!("0122222222333333334444444455000000000000000000007cb0"), 0x0040a1), - (hex!("0122222222333333334444444455000000010000000000000020"), 0x004141), - (hex!("0122222222333333334444444455000000020000000000000030"), 0x0041e1), - (hex!("01222222223333333344444444550000000200000000000051a0"), 0x004281), - (hex!("0122222222333333334444444455000000030000000000000040"), 0x004321), - (hex!("0122222222333333334444444455000000030000000000006cf0"), 0x0043c1), - (hex!("0122222222333333334444444455000000030000000000007140"), 0x004461), - (hex!("0122222222333333334444444455000000040000000000000050"), 0x004501), - (hex!("01222222223333333344444444550000000400000000000047f0"), 0x0045a1), - (hex!("01222222223333333344444444550000000400000000000072b0"), 0x004641), - (hex!("0122222222333333334444444455000000050000000000000060"), 0x0046e1), - (hex!("0122222222333333334444444455000000050000000000005550"), 0x004781), - (hex!("0122222222333333334444444455000000060000000000000070"), 0x004821), - (hex!("01222222223333333344444444550000000600000000000044a0"), 0x0048c1), - (hex!("0122222222333333334444444455000000060000000000006870"), 0x004961), - (hex!("0122222222333333334444444455000000070000000000000080"), 0x004a01), - (hex!("0122222222333333334444444455000000080000000000000090"), 0x004aa1), - (hex!("0122222222333333334444444455000000080000000000004150"), 0x004b41), - (hex!("01222222223333333344444444550000000900000000000000a0"), 0x004be1), - (hex!("01222222223333333344444444550000000a00000000000000b0"), 0x004c81), - (hex!("01222222223333333344444444550000000a0000000000006680"), 0x004d21), - (hex!("01222222223333333344444444550000000b00000000000000c0"), 0x004dc1), - (hex!("01222222223333333344444444550000000b0000000000006230"), 0x004e61), - (hex!("01222222223333333344444444550000000c00000000000000d0"), 0x004f01), - (hex!("01222222223333333344444444550000000d00000000000000e0"), 0x004fa1), - (hex!("01222222223333333344444444550000000e00000000000000f0"), 0x005041), - (hex!("01222222223333333344444444550000000e0000000000006000"), 0x0050e1), - (hex!("01222222223333333344444444550000000f0000000000000100"), 0x005181), - (hex!("01222222223333333344444444550000000f00000000000053c0"), 0x005221), - (hex!("01222222223333333344444444550000000f0000000000006580"), 0x0052c1), - (hex!("0122222222333333334444444455000000100000000000000110"), 0x005361), - (hex!("01222222223333333344444444550000001000000000000046c0"), 0x005401), - (hex!("0122222222333333334444444455000000100000000000004e40"), 0x0054a1), - (hex!("0122222222333333334444444455000000110000000000000120"), 0x005541), - (hex!("0122222222333333334444444455000000120000000000000130"), 0x0055e1), - (hex!("01222222223333333344444444550000001200000000000066d0"), 0x005681), - (hex!("0122222222333333334444444455000000130000000000000140"), 0x005721), - (hex!("0122222222333333334444444455000000130000000000007710"), 0x0057c1), - (hex!("0122222222333333334444444455000000140000000000000150"), 0x005861), - (hex!("0122222222333333334444444455000000140000000000006c40"), 0x005901), - (hex!("0122222222333333334444444455000000150000000000000160"), 0x0059a1), - (hex!("0122222222333333334444444455000000150000000000005990"), 0x005a41), - (hex!("0122222222333333334444444455000000160000000000000170"), 0x005ae1), - (hex!("0122222222333333334444444455000000160000000000005530"), 0x005b81), - (hex!("0122222222333333334444444455000000170000000000000180"), 0x005c21), - (hex!("0122222222333333334444444455000000170000000000004290"), 0x005cc1), - (hex!("0122222222333333334444444455000000180000000000000190"), 0x005d61), - (hex!("01222222223333333344444444550000001800000000000051c0"), 0x005e01), - (hex!("01222222223333333344444444550000001900000000000001a0"), 0x005ea1), - (hex!("0122222222333333334444444455000000190000000000005420"), 0x005f41), - (hex!("0122222222333333334444444455000000190000000000005770"), 0x005fe1), - (hex!("01222222223333333344444444550000001900000000000079d0"), 0x006081), - (hex!("01222222223333333344444444550000001a00000000000001b0"), 0x006121), - (hex!("01222222223333333344444444550000001a0000000000006f70"), 0x0061c1), - (hex!("01222222223333333344444444550000001a0000000000007150"), 0x006261), - (hex!("01222222223333333344444444550000001b00000000000001c0"), 0x006301), - (hex!("01222222223333333344444444550000001b0000000000005070"), 0x0063a1), - (hex!("01222222223333333344444444550000001c00000000000001d0"), 0x006441), - (hex!("01222222223333333344444444550000001d00000000000001e0"), 0x0064e1), - (hex!("01222222223333333344444444550000001e00000000000001f0"), 0x006581), - (hex!("01222222223333333344444444550000001e0000000000005650"), 0x006621), - (hex!("01222222223333333344444444550000001f0000000000000200"), 0x0066c1), - (hex!("01222222223333333344444444550000001f0000000000006ca0"), 0x006761), - (hex!("0122222222333333334444444455000000200000000000000210"), 0x006801), - (hex!("0122222222333333334444444455000000200000000000005fc0"), 0x0068a1), - (hex!("0122222222333333334444444455000000210000000000000220"), 0x006941), - (hex!("0122222222333333334444444455000000210000000000006430"), 0x0069e1), - (hex!("0122222222333333334444444455000000220000000000000230"), 0x006a81), - (hex!("01222222223333333344444444550000002200000000000040e0"), 0x006b21), - (hex!("0122222222333333334444444455000000230000000000000240"), 0x006bc1), - (hex!("01222222223333333344444444550000002300000000000042d0"), 0x006c61), - (hex!("0122222222333333334444444455000000240000000000000250"), 0x006d01), - (hex!("0122222222333333334444444455000000250000000000000260"), 0x006da1), - (hex!("01222222223333333344444444550000002500000000000058c0"), 0x006e41), - (hex!("0122222222333333334444444455000000260000000000000270"), 0x006ee1), - (hex!("0122222222333333334444444455000000260000000000004020"), 0x006f81), - (hex!("0122222222333333334444444455000000270000000000000280"), 0x007021), - (hex!("0122222222333333334444444455000000280000000000000290"), 0x0070c1), - (hex!("0122222222333333334444444455000000280000000000007c00"), 0x007161), - (hex!("01222222223333333344444444550000002900000000000002a0"), 0x007201), - (hex!("01222222223333333344444444550000002a00000000000002b0"), 0x0072a1), - (hex!("01222222223333333344444444550000002b00000000000002c0"), 0x007341), - (hex!("01222222223333333344444444550000002c00000000000002d0"), 0x0073e1), - (hex!("01222222223333333344444444550000002c00000000000041b0"), 0x007481), - (hex!("01222222223333333344444444550000002c0000000000004c30"), 0x007521), - (hex!("01222222223333333344444444550000002d00000000000002e0"), 0x0075c1), - (hex!("01222222223333333344444444550000002d0000000000005e40"), 0x007661), - (hex!("01222222223333333344444444550000002d0000000000006990"), 0x007701), - (hex!("01222222223333333344444444550000002e00000000000002f0"), 0x0077a1), - (hex!("01222222223333333344444444550000002f0000000000000300"), 0x007841), - (hex!("01222222223333333344444444550000002f0000000000004a70"), 0x0078e1), - (hex!("01222222223333333344444444550000002f0000000000006b40"), 0x007981), - (hex!("0122222222333333334444444455000000300000000000000310"), 0x007a21), - (hex!("0122222222333333334444444455000000310000000000000320"), 0x007ac1), - (hex!("0122222222333333334444444455000000320000000000000330"), 0x007b61), - (hex!("01222222223333333344444444550000003200000000000041a0"), 0x007c01), - (hex!("0122222222333333334444444455000000320000000000007340"), 0x007ca1), - (hex!("0122222222333333334444444455000000320000000000007730"), 0x007d41), - (hex!("0122222222333333334444444455000000330000000000000340"), 0x007de1), - (hex!("01222222223333333344444444550000003300000000000055a0"), 0x007e81), - (hex!("0122222222333333334444444455000000340000000000000350"), 0x007f21), - (hex!("0122222222333333334444444455000000350000000000000360"), 0x007fc1), - (hex!("01222222223333333344444444550000003500000000000077a0"), 0x008061), - (hex!("0122222222333333334444444455000000360000000000000370"), 0x008101), - (hex!("0122222222333333334444444455000000370000000000000380"), 0x0081a1), - (hex!("0122222222333333334444444455000000380000000000000390"), 0x008241), - (hex!("01222222223333333344444444550000003900000000000003a0"), 0x0082e1), - (hex!("01222222223333333344444444550000003a00000000000003b0"), 0x008381), - (hex!("01222222223333333344444444550000003a00000000000071c0"), 0x008421), - (hex!("01222222223333333344444444550000003b00000000000003c0"), 0x0084c1), - (hex!("01222222223333333344444444550000003c00000000000003d0"), 0x008561), - (hex!("01222222223333333344444444550000003d00000000000003e0"), 0x008601), - (hex!("01222222223333333344444444550000003e00000000000003f0"), 0x0086a1), - (hex!("01222222223333333344444444550000003e00000000000062e0"), 0x008741), - (hex!("01222222223333333344444444550000003f0000000000000400"), 0x0087e1), - (hex!("0122222222333333334444444455000000400000000000000410"), 0x008881), - (hex!("0122222222333333334444444455000000400000000000004460"), 0x008921), - (hex!("0122222222333333334444444455000000400000000000005b90"), 0x0089c1), - (hex!("01222222223333333344444444550000004000000000000079b0"), 0x008a61), - (hex!("0122222222333333334444444455000000410000000000000420"), 0x008b01), - (hex!("0122222222333333334444444455000000420000000000000430"), 0x008ba1), - (hex!("0122222222333333334444444455000000420000000000005640"), 0x008c41), - (hex!("0122222222333333334444444455000000430000000000000440"), 0x008ce1), - (hex!("01222222223333333344444444550000004300000000000072a0"), 0x008d81), - (hex!("0122222222333333334444444455000000440000000000000450"), 0x008e21), - (hex!("0122222222333333334444444455000000450000000000000460"), 0x008ec1), - (hex!("0122222222333333334444444455000000450000000000005750"), 0x008f61), - (hex!("01222222223333333344444444550000004500000000000077b0"), 0x009001), - (hex!("0122222222333333334444444455000000460000000000000470"), 0x0090a1), - (hex!("0122222222333333334444444455000000470000000000000480"), 0x009141), - (hex!("0122222222333333334444444455000000480000000000000490"), 0x0091e1), - (hex!("01222222223333333344444444550000004800000000000069e0"), 0x009281), - (hex!("01222222223333333344444444550000004900000000000004a0"), 0x009321), - (hex!("0122222222333333334444444455000000490000000000007370"), 0x0093c1), - (hex!("01222222223333333344444444550000004a00000000000004b0"), 0x009461), - (hex!("01222222223333333344444444550000004a0000000000005cb0"), 0x009501), - (hex!("01222222223333333344444444550000004b00000000000004c0"), 0x0095a1), - (hex!("01222222223333333344444444550000004c00000000000004d0"), 0x009641), - (hex!("01222222223333333344444444550000004c0000000000004880"), 0x0096e1), - (hex!("01222222223333333344444444550000004c0000000000007a40"), 0x009781), - (hex!("01222222223333333344444444550000004d00000000000004e0"), 0x009821), - (hex!("01222222223333333344444444550000004d0000000000006390"), 0x0098c1), - (hex!("01222222223333333344444444550000004e00000000000004f0"), 0x009961), - (hex!("01222222223333333344444444550000004e0000000000004db0"), 0x009a01), - (hex!("01222222223333333344444444550000004f0000000000000500"), 0x009aa1), - (hex!("0122222222333333334444444455000000500000000000000510"), 0x009b41), - (hex!("0122222222333333334444444455000000510000000000000520"), 0x009be1), - (hex!("01222222223333333344444444550000005100000000000069c0"), 0x009c81), - (hex!("0122222222333333334444444455000000520000000000000530"), 0x009d21), - (hex!("0122222222333333334444444455000000520000000000006e60"), 0x009dc1), - (hex!("01222222223333333344444444550000005200000000000070c0"), 0x009e61), - (hex!("0122222222333333334444444455000000530000000000000540"), 0x009f01), - (hex!("0122222222333333334444444455000000530000000000005840"), 0x009fa1), - (hex!("0122222222333333334444444455000000540000000000000550"), 0x00a041), - (hex!("01222222223333333344444444550000005400000000000043e0"), 0x00a0e1), - (hex!("01222222223333333344444444550000005400000000000074e0"), 0x00a181), - (hex!("0122222222333333334444444455000000550000000000000560"), 0x00a221), - (hex!("0122222222333333334444444455000000550000000000003ee0"), 0x00a2c1), - (hex!("0122222222333333334444444455000000560000000000000570"), 0x00a361), - (hex!("0122222222333333334444444455000000570000000000000580"), 0x00a401), - (hex!("0122222222333333334444444455000000570000000000007030"), 0x00a4a1), - (hex!("0122222222333333334444444455000000580000000000000590"), 0x00a541), - (hex!("0122222222333333334444444455000000580000000000005340"), 0x00a5e1), - (hex!("01222222223333333344444444550000005800000000000059f0"), 0x00a681), - (hex!("0122222222333333334444444455000000580000000000006930"), 0x00a721), - (hex!("01222222223333333344444444550000005900000000000005a0"), 0x00a7c1), - (hex!("0122222222333333334444444455000000590000000000003f90"), 0x00a861), - (hex!("01222222223333333344444444550000005a00000000000005b0"), 0x00a901), - (hex!("01222222223333333344444444550000005b00000000000005c0"), 0x00a9a1), - (hex!("01222222223333333344444444550000005b00000000000062c0"), 0x00aa41), - (hex!("01222222223333333344444444550000005c00000000000005d0"), 0x00aae1), - (hex!("01222222223333333344444444550000005c0000000000005a70"), 0x00ab81), - (hex!("01222222223333333344444444550000005c0000000000005dd0"), 0x00ac21), - (hex!("01222222223333333344444444550000005d00000000000005e0"), 0x00acc1), - (hex!("01222222223333333344444444550000005d0000000000005730"), 0x00ad61), - (hex!("01222222223333333344444444550000005e00000000000005f0"), 0x00ae01), - (hex!("01222222223333333344444444550000005e0000000000004f40"), 0x00aea1), - (hex!("01222222223333333344444444550000005f0000000000000600"), 0x00af41), - (hex!("0122222222333333334444444455000000600000000000000610"), 0x00afe1), - (hex!("0122222222333333334444444455000000600000000000007c40"), 0x00b081), - (hex!("0122222222333333334444444455000000610000000000000620"), 0x00b121), - (hex!("0122222222333333334444444455000000610000000000007860"), 0x00b1c1), - (hex!("0122222222333333334444444455000000620000000000000630"), 0x00b261), - (hex!("0122222222333333334444444455000000620000000000005050"), 0x00b301), - (hex!("0122222222333333334444444455000000630000000000000640"), 0x00b3a1), - (hex!("0122222222333333334444444455000000640000000000000650"), 0x00b441), - (hex!("0122222222333333334444444455000000650000000000000660"), 0x00b4e1), - (hex!("0122222222333333334444444455000000650000000000005330"), 0x00b581), - (hex!("0122222222333333334444444455000000660000000000000670"), 0x00b621), - (hex!("0122222222333333334444444455000000660000000000004e20"), 0x00b6c1), - (hex!("0122222222333333334444444455000000660000000000005ee0"), 0x00b761), - (hex!("0122222222333333334444444455000000660000000000006360"), 0x00b801), - (hex!("0122222222333333334444444455000000670000000000000680"), 0x00b8a1), - (hex!("0122222222333333334444444455000000670000000000004040"), 0x00b941), - (hex!("0122222222333333334444444455000000680000000000000690"), 0x00b9e1), - (hex!("0122222222333333334444444455000000680000000000003f80"), 0x00ba81), - (hex!("01222222223333333344444444550000006800000000000041e0"), 0x00bb21), - (hex!("01222222223333333344444444550000006900000000000006a0"), 0x00bbc1), - (hex!("0122222222333333334444444455000000690000000000006080"), 0x00bc61), - (hex!("01222222223333333344444444550000006a00000000000006b0"), 0x00bd01), - (hex!("01222222223333333344444444550000006a00000000000042f0"), 0x00bda1), - (hex!("01222222223333333344444444550000006b00000000000006c0"), 0x00be41), - (hex!("01222222223333333344444444550000006b00000000000052f0"), 0x00bee1), - (hex!("01222222223333333344444444550000006b0000000000005980"), 0x00bf81), - (hex!("01222222223333333344444444550000006b0000000000006170"), 0x00c021), - (hex!("01222222223333333344444444550000006c00000000000006d0"), 0x00c0c1), - (hex!("01222222223333333344444444550000006d00000000000006e0"), 0x00c161), - (hex!("01222222223333333344444444550000006d0000000000006fb0"), 0x00c201), - (hex!("01222222223333333344444444550000006e00000000000006f0"), 0x00c2a1), - (hex!("01222222223333333344444444550000006e00000000000065b0"), 0x00c341), - (hex!("01222222223333333344444444550000006e0000000000007970"), 0x00c3e1), - (hex!("01222222223333333344444444550000006f0000000000000700"), 0x00c481), - (hex!("01222222223333333344444444550000006f0000000000005900"), 0x00c521), - (hex!("01222222223333333344444444550000006f0000000000006d90"), 0x00c5c1), - (hex!("0122222222333333334444444455000000700000000000000710"), 0x00c661), - (hex!("01222222223333333344444444550000007000000000000045c0"), 0x00c701), - (hex!("0122222222333333334444444455000000700000000000004d40"), 0x00c7a1), - (hex!("0122222222333333334444444455000000710000000000000720"), 0x00c841), - (hex!("0122222222333333334444444455000000710000000000004dc0"), 0x00c8e1), - (hex!("0122222222333333334444444455000000710000000000007550"), 0x00c981), - (hex!("0122222222333333334444444455000000720000000000000730"), 0x00ca21), - (hex!("0122222222333333334444444455000000720000000000003ec0"), 0x00cac1), - (hex!("01222222223333333344444444550000007200000000000045a0"), 0x00cb61), - (hex!("0122222222333333334444444455000000720000000000006770"), 0x00cc01), - (hex!("0122222222333333334444444455000000720000000000006bc0"), 0x00cca1), - (hex!("0122222222333333334444444455000000730000000000000740"), 0x00cd41), - (hex!("0122222222333333334444444455000000730000000000005250"), 0x00cde1), - (hex!("01222222223333333344444444550000007300000000000075f0"), 0x00ce81), - (hex!("0122222222333333334444444455000000740000000000000750"), 0x00cf21), - (hex!("0122222222333333334444444455000000740000000000003ff0"), 0x00cfc1), - (hex!("01222222223333333344444444550000007400000000000079e0"), 0x00d061), - (hex!("0122222222333333334444444455000000750000000000000760"), 0x00d101), - (hex!("0122222222333333334444444455000000750000000000004310"), 0x00d1a1), - (hex!("0122222222333333334444444455000000760000000000000770"), 0x00d241), - (hex!("0122222222333333334444444455000000770000000000000780"), 0x00d2e1), - (hex!("01222222223333333344444444550000007700000000000062f0"), 0x00d381), - (hex!("0122222222333333334444444455000000770000000000006940"), 0x00d421), - (hex!("0122222222333333334444444455000000780000000000000790"), 0x00d4c1), - (hex!("01222222223333333344444444550000007900000000000007a0"), 0x00d561), - (hex!("0122222222333333334444444455000000790000000000007af0"), 0x00d601), - (hex!("01222222223333333344444444550000007a00000000000007b0"), 0x00d6a1), - (hex!("01222222223333333344444444550000007b00000000000007c0"), 0x00d741), - (hex!("01222222223333333344444444550000007b00000000000067e0"), 0x00d7e1), - (hex!("01222222223333333344444444550000007b0000000000007890"), 0x00d881), - (hex!("01222222223333333344444444550000007c00000000000007d0"), 0x00d921), - (hex!("01222222223333333344444444550000007d00000000000007e0"), 0x00d9c1), - (hex!("01222222223333333344444444550000007e00000000000007f0"), 0x00da61), - (hex!("01222222223333333344444444550000007f0000000000000800"), 0x00db01), - (hex!("01222222223333333344444444550000007f0000000000005be0"), 0x00dba1), - (hex!("0122222222333333334444444455000000800000000000000810"), 0x00dc41), - (hex!("0122222222333333334444444455000000810000000000000820"), 0x00dce1), - (hex!("0122222222333333334444444455000000810000000000007190"), 0x00dd81), - (hex!("0122222222333333334444444455000000820000000000000830"), 0x00de21), - (hex!("0122222222333333334444444455000000820000000000004ab0"), 0x00dec1), - (hex!("0122222222333333334444444455000000830000000000000840"), 0x00df61), - (hex!("0122222222333333334444444455000000830000000000006720"), 0x00e001), - (hex!("0122222222333333334444444455000000840000000000000850"), 0x00e0a1), - (hex!("0122222222333333334444444455000000850000000000000860"), 0x00e141), - (hex!("01222222223333333344444444550000008500000000000054f0"), 0x00e1e1), - (hex!("0122222222333333334444444455000000850000000000007920"), 0x00e281), - (hex!("0122222222333333334444444455000000860000000000000870"), 0x00e321), - (hex!("01222222223333333344444444550000008600000000000060e0"), 0x00e3c1), - (hex!("0122222222333333334444444455000000860000000000006be0"), 0x00e461), - (hex!("0122222222333333334444444455000000870000000000000880"), 0x00e501), - (hex!("0122222222333333334444444455000000870000000000006820"), 0x00e5a1), - (hex!("0122222222333333334444444455000000880000000000000890"), 0x00e641), - (hex!("01222222223333333344444444550000008900000000000008a0"), 0x00e6e1), - (hex!("0122222222333333334444444455000000890000000000007c30"), 0x00e781), - (hex!("01222222223333333344444444550000008a00000000000008b0"), 0x00e821), - (hex!("01222222223333333344444444550000008b00000000000008c0"), 0x00e8c1), - (hex!("01222222223333333344444444550000008b0000000000005910"), 0x00e961), - (hex!("01222222223333333344444444550000008b0000000000006fe0"), 0x00ea01), - (hex!("01222222223333333344444444550000008c00000000000008d0"), 0x00eaa1), - (hex!("01222222223333333344444444550000008c0000000000006800"), 0x00eb41), - (hex!("01222222223333333344444444550000008d00000000000008e0"), 0x00ebe1), - (hex!("01222222223333333344444444550000008d0000000000005810"), 0x00ec81), - (hex!("01222222223333333344444444550000008d0000000000007c90"), 0x00ed21), - (hex!("01222222223333333344444444550000008e00000000000008f0"), 0x00edc1), - (hex!("01222222223333333344444444550000008e00000000000058f0"), 0x00ee61), - (hex!("01222222223333333344444444550000008f0000000000000900"), 0x00ef01), - (hex!("01222222223333333344444444550000008f0000000000005a30"), 0x00efa1), - (hex!("0122222222333333334444444455000000900000000000000910"), 0x00f041), - (hex!("0122222222333333334444444455000000900000000000006130"), 0x00f0e1), - (hex!("0122222222333333334444444455000000900000000000006550"), 0x00f181), - (hex!("0122222222333333334444444455000000910000000000000920"), 0x00f221), - (hex!("01222222223333333344444444550000009100000000000079f0"), 0x00f2c1), - (hex!("0122222222333333334444444455000000920000000000000930"), 0x00f361), - (hex!("0122222222333333334444444455000000920000000000005620"), 0x00f401), - (hex!("0122222222333333334444444455000000920000000000005e90"), 0x00f4a1), - (hex!("01222222223333333344444444550000009200000000000063d0"), 0x00f541), - (hex!("01222222223333333344444444550000009200000000000076c0"), 0x00f5e1), - (hex!("0122222222333333334444444455000000930000000000000940"), 0x00f681), - (hex!("01222222223333333344444444550000009300000000000044e0"), 0x00f721), - (hex!("0122222222333333334444444455000000940000000000000950"), 0x00f7c1), - (hex!("0122222222333333334444444455000000940000000000007a30"), 0x00f861), - (hex!("0122222222333333334444444455000000950000000000000960"), 0x00f901), - (hex!("0122222222333333334444444455000000950000000000007a70"), 0x00f9a1), - (hex!("0122222222333333334444444455000000960000000000000970"), 0x00fa41), - (hex!("0122222222333333334444444455000000970000000000000980"), 0x00fae1), - (hex!("0122222222333333334444444455000000970000000000007330"), 0x00fb81), - (hex!("0122222222333333334444444455000000980000000000000990"), 0x00fc21), - (hex!("0122222222333333334444444455000000980000000000005af0"), 0x00fcc1), - (hex!("0122222222333333334444444455000000980000000000007ae0"), 0x00fd61), - (hex!("01222222223333333344444444550000009900000000000009a0"), 0x00fe01), - (hex!("0122222222333333334444444455000000990000000000005160"), 0x00fea1), - (hex!("0122222222333333334444444455000000990000000000006850"), 0x00ff41), - (hex!("01222222223333333344444444550000009a00000000000009b0"), 0x00ffe1), - (hex!("01222222223333333344444444550000009b00000000000009c0"), 0x010081), - (hex!("01222222223333333344444444550000009b0000000000005010"), 0x010121), - (hex!("01222222223333333344444444550000009c00000000000009d0"), 0x0101c1), - (hex!("01222222223333333344444444550000009c00000000000042e0"), 0x010261), - (hex!("01222222223333333344444444550000009d00000000000009e0"), 0x010301), - (hex!("01222222223333333344444444550000009d00000000000057f0"), 0x0103a1), - (hex!("01222222223333333344444444550000009e00000000000009f0"), 0x010441), - (hex!("01222222223333333344444444550000009e0000000000004ef0"), 0x0104e1), - (hex!("01222222223333333344444444550000009f0000000000000a00"), 0x010581), - (hex!("01222222223333333344444444550000009f0000000000006110"), 0x010621), - (hex!("0122222222333333334444444455000000a00000000000000a10"), 0x0106c1), - (hex!("0122222222333333334444444455000000a10000000000000a20"), 0x010761), - (hex!("0122222222333333334444444455000000a100000000000040d0"), 0x010801), - (hex!("0122222222333333334444444455000000a10000000000007670"), 0x0108a1), - (hex!("0122222222333333334444444455000000a20000000000000a30"), 0x010941), - (hex!("0122222222333333334444444455000000a200000000000074d0"), 0x0109e1), - (hex!("0122222222333333334444444455000000a30000000000000a40"), 0x010a81), - (hex!("0122222222333333334444444455000000a30000000000004c90"), 0x010b21), - (hex!("0122222222333333334444444455000000a40000000000000a50"), 0x010bc1), - (hex!("0122222222333333334444444455000000a50000000000000a60"), 0x010c61), - (hex!("0122222222333333334444444455000000a60000000000000a70"), 0x010d01), - (hex!("0122222222333333334444444455000000a60000000000006d80"), 0x010da1), - (hex!("0122222222333333334444444455000000a60000000000007830"), 0x010e41), - (hex!("0122222222333333334444444455000000a70000000000000a80"), 0x010ee1), - (hex!("0122222222333333334444444455000000a700000000000064f0"), 0x010f81), - (hex!("0122222222333333334444444455000000a80000000000000a90"), 0x011021), - (hex!("0122222222333333334444444455000000a90000000000000aa0"), 0x0110c1), - (hex!("0122222222333333334444444455000000a90000000000005e30"), 0x011161), - (hex!("0122222222333333334444444455000000aa0000000000000ab0"), 0x011201), - (hex!("0122222222333333334444444455000000ab0000000000000ac0"), 0x0112a1), - (hex!("0122222222333333334444444455000000ac0000000000000ad0"), 0x011341), - (hex!("0122222222333333334444444455000000ac0000000000006d20"), 0x0113e1), - (hex!("0122222222333333334444444455000000ac0000000000007000"), 0x011481), - (hex!("0122222222333333334444444455000000ad0000000000000ae0"), 0x011521), - (hex!("0122222222333333334444444455000000ae0000000000000af0"), 0x0115c1), - (hex!("0122222222333333334444444455000000ae0000000000004a10"), 0x011661), - (hex!("0122222222333333334444444455000000af0000000000000b00"), 0x011701), - (hex!("0122222222333333334444444455000000af0000000000004e10"), 0x0117a1), - (hex!("0122222222333333334444444455000000b00000000000000b10"), 0x011841), - (hex!("0122222222333333334444444455000000b00000000000004280"), 0x0118e1), - (hex!("0122222222333333334444444455000000b000000000000077e0"), 0x011981), - (hex!("0122222222333333334444444455000000b10000000000000b20"), 0x011a21), - (hex!("0122222222333333334444444455000000b20000000000000b30"), 0x011ac1), - (hex!("0122222222333333334444444455000000b30000000000000b40"), 0x011b61), - (hex!("0122222222333333334444444455000000b30000000000004bc0"), 0x011c01), - (hex!("0122222222333333334444444455000000b40000000000000b50"), 0x011ca1), - (hex!("0122222222333333334444444455000000b50000000000000b60"), 0x011d41), - (hex!("0122222222333333334444444455000000b50000000000004fa0"), 0x011de1), - (hex!("0122222222333333334444444455000000b50000000000006a60"), 0x011e81), - (hex!("0122222222333333334444444455000000b60000000000000b70"), 0x011f21), - (hex!("0122222222333333334444444455000000b60000000000005630"), 0x011fc1), - (hex!("0122222222333333334444444455000000b70000000000000b80"), 0x012061), - (hex!("0122222222333333334444444455000000b80000000000000b90"), 0x012101), - (hex!("0122222222333333334444444455000000b80000000000006f80"), 0x0121a1), - (hex!("0122222222333333334444444455000000b90000000000000ba0"), 0x012241), - (hex!("0122222222333333334444444455000000ba0000000000000bb0"), 0x0122e1), - (hex!("0122222222333333334444444455000000bb0000000000000bc0"), 0x012381), - (hex!("0122222222333333334444444455000000bb00000000000047c0"), 0x012421), - (hex!("0122222222333333334444444455000000bb0000000000006060"), 0x0124c1), - (hex!("0122222222333333334444444455000000bc0000000000000bd0"), 0x012561), - (hex!("0122222222333333334444444455000000bd0000000000000be0"), 0x012601), - (hex!("0122222222333333334444444455000000bd0000000000004e80"), 0x0126a1), - (hex!("0122222222333333334444444455000000be0000000000000bf0"), 0x012741), - (hex!("0122222222333333334444444455000000bf0000000000000c00"), 0x0127e1), - (hex!("0122222222333333334444444455000000bf00000000000047a0"), 0x012881), - (hex!("0122222222333333334444444455000000bf0000000000006da0"), 0x012921), - (hex!("0122222222333333334444444455000000c00000000000000c10"), 0x0129c1), - (hex!("0122222222333333334444444455000000c10000000000000c20"), 0x012a61), - (hex!("0122222222333333334444444455000000c20000000000000c30"), 0x012b01), - (hex!("0122222222333333334444444455000000c20000000000004bd0"), 0x012ba1), - (hex!("0122222222333333334444444455000000c20000000000006ac0"), 0x012c41), - (hex!("0122222222333333334444444455000000c30000000000000c40"), 0x012ce1), - (hex!("0122222222333333334444444455000000c30000000000004660"), 0x012d81), - (hex!("0122222222333333334444444455000000c40000000000000c50"), 0x012e21), - (hex!("0122222222333333334444444455000000c50000000000000c60"), 0x012ec1), - (hex!("0122222222333333334444444455000000c60000000000000c70"), 0x012f61), - (hex!("0122222222333333334444444455000000c60000000000005880"), 0x013001), - (hex!("0122222222333333334444444455000000c60000000000006b70"), 0x0130a1), - (hex!("0122222222333333334444444455000000c70000000000000c80"), 0x013141), - (hex!("0122222222333333334444444455000000c80000000000000c90"), 0x0131e1), - (hex!("0122222222333333334444444455000000c80000000000005310"), 0x013281), - (hex!("0122222222333333334444444455000000c80000000000005db0"), 0x013321), - (hex!("0122222222333333334444444455000000c80000000000007040"), 0x0133c1), - (hex!("0122222222333333334444444455000000c80000000000007290"), 0x013461), - (hex!("0122222222333333334444444455000000c90000000000000ca0"), 0x013501), - (hex!("0122222222333333334444444455000000c90000000000004fe0"), 0x0135a1), - (hex!("0122222222333333334444444455000000ca0000000000000cb0"), 0x013641), - (hex!("0122222222333333334444444455000000ca0000000000006140"), 0x0136e1), - (hex!("0122222222333333334444444455000000ca0000000000007700"), 0x013781), - (hex!("0122222222333333334444444455000000cb0000000000000cc0"), 0x013821), - (hex!("0122222222333333334444444455000000cc0000000000000cd0"), 0x0138c1), - (hex!("0122222222333333334444444455000000cd0000000000000ce0"), 0x013961), - (hex!("0122222222333333334444444455000000cd0000000000003f20"), 0x013a01), - (hex!("0122222222333333334444444455000000cd00000000000040f0"), 0x013aa1), - (hex!("0122222222333333334444444455000000cd0000000000004ec0"), 0x013b41), - (hex!("0122222222333333334444444455000000ce0000000000000cf0"), 0x013be1), - (hex!("0122222222333333334444444455000000ce0000000000007200"), 0x013c81), - (hex!("0122222222333333334444444455000000cf0000000000000d00"), 0x013d21), - (hex!("0122222222333333334444444455000000cf00000000000046a0"), 0x013dc1), - (hex!("0122222222333333334444444455000000cf0000000000005960"), 0x013e61), - (hex!("0122222222333333334444444455000000d00000000000000d10"), 0x013f01), - (hex!("0122222222333333334444444455000000d00000000000005f30"), 0x013fa1), - (hex!("0122222222333333334444444455000000d10000000000000d20"), 0x014041), - (hex!("0122222222333333334444444455000000d10000000000007a00"), 0x0140e1), - (hex!("0122222222333333334444444455000000d20000000000000d30"), 0x014181), - (hex!("0122222222333333334444444455000000d30000000000000d40"), 0x014221), - (hex!("0122222222333333334444444455000000d40000000000000d50"), 0x0142c1), - (hex!("0122222222333333334444444455000000d50000000000000d60"), 0x014361), - (hex!("0122222222333333334444444455000000d50000000000004960"), 0x014401), - (hex!("0122222222333333334444444455000000d500000000000055d0"), 0x0144a1), - (hex!("0122222222333333334444444455000000d500000000000067d0"), 0x014541), - (hex!("0122222222333333334444444455000000d60000000000000d70"), 0x0145e1), - (hex!("0122222222333333334444444455000000d70000000000000d80"), 0x014681), - (hex!("0122222222333333334444444455000000d80000000000000d90"), 0x014721), - (hex!("0122222222333333334444444455000000d800000000000065f0"), 0x0147c1), - (hex!("0122222222333333334444444455000000d90000000000000da0"), 0x014861), - (hex!("0122222222333333334444444455000000d90000000000004980"), 0x014901), - (hex!("0122222222333333334444444455000000da0000000000000db0"), 0x0149a1), - (hex!("0122222222333333334444444455000000da00000000000048c0"), 0x014a41), - (hex!("0122222222333333334444444455000000da00000000000072c0"), 0x014ae1), - (hex!("0122222222333333334444444455000000da00000000000076b0"), 0x014b81), - (hex!("0122222222333333334444444455000000db0000000000000dc0"), 0x014c21), - (hex!("0122222222333333334444444455000000dc0000000000000dd0"), 0x014cc1), - (hex!("0122222222333333334444444455000000dc00000000000040a0"), 0x014d61), - (hex!("0122222222333333334444444455000000dc00000000000074c0"), 0x014e01), - (hex!("0122222222333333334444444455000000dd0000000000000de0"), 0x014ea1), - (hex!("0122222222333333334444444455000000dd0000000000004e50"), 0x014f41), - (hex!("0122222222333333334444444455000000dd0000000000007270"), 0x014fe1), - (hex!("0122222222333333334444444455000000de0000000000000df0"), 0x015081), - (hex!("0122222222333333334444444455000000de00000000000078d0"), 0x015121), - (hex!("0122222222333333334444444455000000df0000000000000e00"), 0x0151c1), - (hex!("0122222222333333334444444455000000df0000000000004d30"), 0x015261), - (hex!("0122222222333333334444444455000000df0000000000006c30"), 0x015301), - (hex!("0122222222333333334444444455000000e00000000000000e10"), 0x0153a1), - (hex!("0122222222333333334444444455000000e00000000000005d30"), 0x015441), - (hex!("0122222222333333334444444455000000e10000000000000e20"), 0x0154e1), - (hex!("0122222222333333334444444455000000e10000000000004610"), 0x015581), - (hex!("0122222222333333334444444455000000e100000000000051d0"), 0x015621), - (hex!("0122222222333333334444444455000000e10000000000005f10"), 0x0156c1), - (hex!("0122222222333333334444444455000000e20000000000000e30"), 0x015761), - (hex!("0122222222333333334444444455000000e20000000000007a90"), 0x015801), - (hex!("0122222222333333334444444455000000e30000000000000e40"), 0x0158a1), - (hex!("0122222222333333334444444455000000e30000000000005ae0"), 0x015941), - (hex!("0122222222333333334444444455000000e40000000000000e50"), 0x0159e1), - (hex!("0122222222333333334444444455000000e50000000000000e60"), 0x015a81), - (hex!("0122222222333333334444444455000000e50000000000004700"), 0x015b21), - (hex!("0122222222333333334444444455000000e500000000000065d0"), 0x015bc1), - (hex!("0122222222333333334444444455000000e60000000000000e70"), 0x015c61), - (hex!("0122222222333333334444444455000000e60000000000004fd0"), 0x015d01), - (hex!("0122222222333333334444444455000000e70000000000000e80"), 0x015da1), - (hex!("0122222222333333334444444455000000e70000000000005150"), 0x015e41), - (hex!("0122222222333333334444444455000000e70000000000005920"), 0x015ee1), - (hex!("0122222222333333334444444455000000e80000000000000e90"), 0x015f81), - (hex!("0122222222333333334444444455000000e80000000000004320"), 0x016021), - (hex!("0122222222333333334444444455000000e80000000000005ec0"), 0x0160c1), - (hex!("0122222222333333334444444455000000e90000000000000ea0"), 0x016161), - (hex!("0122222222333333334444444455000000e900000000000043b0"), 0x016201), - (hex!("0122222222333333334444444455000000ea0000000000000eb0"), 0x0162a1), - (hex!("0122222222333333334444444455000000ea0000000000003ea0"), 0x016341), - (hex!("0122222222333333334444444455000000ea0000000000004f50"), 0x0163e1), - (hex!("0122222222333333334444444455000000ea0000000000007520"), 0x016481), - (hex!("0122222222333333334444444455000000eb0000000000000ec0"), 0x016521), - (hex!("0122222222333333334444444455000000ec0000000000000ed0"), 0x0165c1), - (hex!("0122222222333333334444444455000000ec0000000000006670"), 0x016661), - (hex!("0122222222333333334444444455000000ed0000000000000ee0"), 0x016701), - (hex!("0122222222333333334444444455000000ee0000000000000ef0"), 0x0167a1), - (hex!("0122222222333333334444444455000000ee0000000000004d10"), 0x016841), - (hex!("0122222222333333334444444455000000ef0000000000000f00"), 0x0168e1), - (hex!("0122222222333333334444444455000000f00000000000000f10"), 0x016981), - (hex!("0122222222333333334444444455000000f00000000000007220"), 0x016a21), - (hex!("0122222222333333334444444455000000f00000000000007540"), 0x016ac1), - (hex!("0122222222333333334444444455000000f10000000000000f20"), 0x016b61), - (hex!("0122222222333333334444444455000000f100000000000066f0"), 0x016c01), - (hex!("0122222222333333334444444455000000f20000000000000f30"), 0x016ca1), - (hex!("0122222222333333334444444455000000f20000000000007810"), 0x016d41), - (hex!("0122222222333333334444444455000000f30000000000000f40"), 0x016de1), - (hex!("0122222222333333334444444455000000f30000000000007b70"), 0x016e81), - (hex!("0122222222333333334444444455000000f40000000000000f50"), 0x016f21), - (hex!("0122222222333333334444444455000000f400000000000059c0"), 0x016fc1), - (hex!("0122222222333333334444444455000000f50000000000000f60"), 0x017061), - (hex!("0122222222333333334444444455000000f50000000000003fb0"), 0x017101), - (hex!("0122222222333333334444444455000000f50000000000005740"), 0x0171a1), - (hex!("0122222222333333334444444455000000f500000000000064d0"), 0x017241), - (hex!("0122222222333333334444444455000000f50000000000006960"), 0x0172e1), - (hex!("0122222222333333334444444455000000f60000000000000f70"), 0x017381), - (hex!("0122222222333333334444444455000000f60000000000006d00"), 0x017421), - (hex!("0122222222333333334444444455000000f70000000000000f80"), 0x0174c1), - (hex!("0122222222333333334444444455000000f80000000000000f90"), 0x017561), - (hex!("0122222222333333334444444455000000f90000000000000fa0"), 0x017601), - (hex!("0122222222333333334444444455000000fa0000000000000fb0"), 0x0176a1), - (hex!("0122222222333333334444444455000000fa00000000000067b0"), 0x017741), - (hex!("0122222222333333334444444455000000fb0000000000000fc0"), 0x0177e1), - (hex!("0122222222333333334444444455000000fb0000000000004eb0"), 0x017881), - (hex!("0122222222333333334444444455000000fb0000000000006ef0"), 0x017921), - (hex!("0122222222333333334444444455000000fc0000000000000fd0"), 0x0179c1), - (hex!("0122222222333333334444444455000000fc0000000000004470"), 0x017a61), - (hex!("0122222222333333334444444455000000fc0000000000005940"), 0x017b01), - (hex!("0122222222333333334444444455000000fd0000000000000fe0"), 0x017ba1), - (hex!("0122222222333333334444444455000000fe0000000000000ff0"), 0x017c41), - (hex!("0122222222333333334444444455000000ff0000000000001000"), 0x017ce1), - (hex!("0122222222333333334444444455000000ff0000000000005690"), 0x017d81), - (hex!("0122222222333333334444444455000001000000000000001010"), 0x017e21), - (hex!("0122222222333333334444444455000001000000000000005210"), 0x017ec1), - (hex!("01222222223333333344444444550000010000000000000070a0"), 0x017f61), - (hex!("0122222222333333334444444455000001010000000000001020"), 0x018001), - (hex!("0122222222333333334444444455000001010000000000006b80"), 0x0180a1), - (hex!("0122222222333333334444444455000001020000000000001030"), 0x018141), - (hex!("0122222222333333334444444455000001030000000000001040"), 0x0181e1), - (hex!("0122222222333333334444444455000001030000000000004c80"), 0x018281), - (hex!("0122222222333333334444444455000001040000000000001050"), 0x018321), - (hex!("0122222222333333334444444455000001040000000000004850"), 0x0183c1), - (hex!("01222222223333333344444444550000010400000000000057b0"), 0x018461), - (hex!("0122222222333333334444444455000001050000000000001060"), 0x018501), - (hex!("01222222223333333344444444550000010500000000000048d0"), 0x0185a1), - (hex!("0122222222333333334444444455000001050000000000007870"), 0x018641), - (hex!("0122222222333333334444444455000001060000000000001070"), 0x0186e1), - (hex!("0122222222333333334444444455000001060000000000004f90"), 0x018781), - (hex!("0122222222333333334444444455000001060000000000006270"), 0x018821), - (hex!("0122222222333333334444444455000001070000000000001080"), 0x0188c1), - (hex!("01222222223333333344444444550000010700000000000063b0"), 0x018961), - (hex!("0122222222333333334444444455000001080000000000001090"), 0x018a01), - (hex!("01222222223333333344444444550000010900000000000010a0"), 0x018aa1), - (hex!("0122222222333333334444444455000001090000000000006f40"), 0x018b41), - (hex!("01222222223333333344444444550000010a00000000000010b0"), 0x018be1), - (hex!("01222222223333333344444444550000010a0000000000006640"), 0x018c81), - (hex!("01222222223333333344444444550000010b00000000000010c0"), 0x018d21), - (hex!("01222222223333333344444444550000010c00000000000010d0"), 0x018dc1), - (hex!("01222222223333333344444444550000010d00000000000010e0"), 0x018e61), - (hex!("01222222223333333344444444550000010e00000000000010f0"), 0x018f01), - (hex!("01222222223333333344444444550000010e0000000000005c40"), 0x018fa1), - (hex!("01222222223333333344444444550000010e0000000000007ba0"), 0x019041), - (hex!("01222222223333333344444444550000010f0000000000001100"), 0x0190e1), - (hex!("01222222223333333344444444550000010f0000000000005c30"), 0x019181), - (hex!("0122222222333333334444444455000001100000000000001110"), 0x019221), - (hex!("0122222222333333334444444455000001100000000000007640"), 0x0192c1), - (hex!("0122222222333333334444444455000001110000000000001120"), 0x019361), - (hex!("01222222223333333344444444550000011100000000000052c0"), 0x019401), - (hex!("0122222222333333334444444455000001110000000000005710"), 0x0194a1), - (hex!("0122222222333333334444444455000001110000000000006a00"), 0x019541), - (hex!("0122222222333333334444444455000001120000000000001130"), 0x0195e1), - (hex!("0122222222333333334444444455000001130000000000001140"), 0x019681), - (hex!("0122222222333333334444444455000001140000000000001150"), 0x019721), - (hex!("0122222222333333334444444455000001140000000000003fa0"), 0x0197c1), - (hex!("01222222223333333344444444550000011400000000000054b0"), 0x019861), - (hex!("0122222222333333334444444455000001140000000000006070"), 0x019901), - (hex!("0122222222333333334444444455000001150000000000001160"), 0x0199a1), - (hex!("0122222222333333334444444455000001150000000000005320"), 0x019a41), - (hex!("0122222222333333334444444455000001150000000000006600"), 0x019ae1), - (hex!("0122222222333333334444444455000001150000000000006df0"), 0x019b81), - (hex!("01222222223333333344444444550000011500000000000079c0"), 0x019c21), - (hex!("0122222222333333334444444455000001160000000000001170"), 0x019cc1), - (hex!("0122222222333333334444444455000001170000000000001180"), 0x019d61), - (hex!("0122222222333333334444444455000001170000000000004a60"), 0x019e01), - (hex!("01222222223333333344444444550000011700000000000063c0"), 0x019ea1), - (hex!("0122222222333333334444444455000001180000000000001190"), 0x019f41), - (hex!("0122222222333333334444444455000001180000000000004530"), 0x019fe1), - (hex!("01222222223333333344444444550000011800000000000077c0"), 0x01a081), - (hex!("01222222223333333344444444550000011900000000000011a0"), 0x01a121), - (hex!("01222222223333333344444444550000011a00000000000011b0"), 0x01a1c1), - (hex!("01222222223333333344444444550000011a00000000000041c0"), 0x01a261), - (hex!("01222222223333333344444444550000011a00000000000061e0"), 0x01a301), - (hex!("01222222223333333344444444550000011b00000000000011c0"), 0x01a3a1), - (hex!("01222222223333333344444444550000011c00000000000011d0"), 0x01a441), - (hex!("01222222223333333344444444550000011c0000000000005f90"), 0x01a4e1), - (hex!("01222222223333333344444444550000011d00000000000011e0"), 0x01a581), - (hex!("01222222223333333344444444550000011d0000000000004160"), 0x01a621), - (hex!("01222222223333333344444444550000011e00000000000011f0"), 0x01a6c1), - (hex!("01222222223333333344444444550000011e00000000000056d0"), 0x01a761), - (hex!("01222222223333333344444444550000011f0000000000001200"), 0x01a801), - (hex!("01222222223333333344444444550000011f0000000000004510"), 0x01a8a1), - (hex!("0122222222333333334444444455000001200000000000001210"), 0x01a941), - (hex!("0122222222333333334444444455000001210000000000001220"), 0x01a9e1), - (hex!("0122222222333333334444444455000001210000000000005140"), 0x01aa81), - (hex!("0122222222333333334444444455000001210000000000006710"), 0x01ab21), - (hex!("0122222222333333334444444455000001210000000000006f50"), 0x01abc1), - (hex!("0122222222333333334444444455000001220000000000001230"), 0x01ac61), - (hex!("0122222222333333334444444455000001220000000000005570"), 0x01ad01), - (hex!("0122222222333333334444444455000001220000000000007ac0"), 0x01ada1), - (hex!("0122222222333333334444444455000001230000000000001240"), 0x01ae41), - (hex!("0122222222333333334444444455000001240000000000001250"), 0x01aee1), - (hex!("0122222222333333334444444455000001240000000000006cd0"), 0x01af81), - (hex!("0122222222333333334444444455000001250000000000001260"), 0x01b021), - (hex!("01222222223333333344444444550000012500000000000046b0"), 0x01b0c1), - (hex!("0122222222333333334444444455000001250000000000005eb0"), 0x01b161), - (hex!("0122222222333333334444444455000001260000000000001270"), 0x01b201), - (hex!("0122222222333333334444444455000001260000000000004630"), 0x01b2a1), - (hex!("0122222222333333334444444455000001270000000000001280"), 0x01b341), - (hex!("0122222222333333334444444455000001270000000000004ff0"), 0x01b3e1), - (hex!("0122222222333333334444444455000001270000000000006ec0"), 0x01b481), - (hex!("0122222222333333334444444455000001280000000000001290"), 0x01b521), - (hex!("01222222223333333344444444550000012900000000000012a0"), 0x01b5c1), - (hex!("0122222222333333334444444455000001290000000000005f60"), 0x01b661), - (hex!("01222222223333333344444444550000012a00000000000012b0"), 0x01b701), - (hex!("01222222223333333344444444550000012a0000000000005480"), 0x01b7a1), - (hex!("01222222223333333344444444550000012b00000000000012c0"), 0x01b841), - (hex!("01222222223333333344444444550000012b00000000000065a0"), 0x01b8e1), - (hex!("01222222223333333344444444550000012b00000000000066c0"), 0x01b981), - (hex!("01222222223333333344444444550000012c00000000000012d0"), 0x01ba21), - (hex!("01222222223333333344444444550000012c00000000000064b0"), 0x01bac1), - (hex!("01222222223333333344444444550000012d00000000000012e0"), 0x01bb61), - (hex!("01222222223333333344444444550000012d00000000000049c0"), 0x01bc01), - (hex!("01222222223333333344444444550000012d0000000000004bf0"), 0x01bca1), - (hex!("01222222223333333344444444550000012e00000000000012f0"), 0x01bd41), - (hex!("01222222223333333344444444550000012e0000000000005ed0"), 0x01bde1), - (hex!("01222222223333333344444444550000012f0000000000001300"), 0x01be81), - (hex!("01222222223333333344444444550000012f00000000000049a0"), 0x01bf21), - (hex!("0122222222333333334444444455000001300000000000001310"), 0x01bfc1), - (hex!("0122222222333333334444444455000001300000000000007840"), 0x01c061), - (hex!("0122222222333333334444444455000001310000000000001320"), 0x01c101), - (hex!("0122222222333333334444444455000001310000000000005f70"), 0x01c1a1), - (hex!("0122222222333333334444444455000001320000000000001330"), 0x01c241), - (hex!("0122222222333333334444444455000001320000000000005a00"), 0x01c2e1), - (hex!("0122222222333333334444444455000001330000000000001340"), 0x01c381), - (hex!("0122222222333333334444444455000001330000000000006c70"), 0x01c421), - (hex!("0122222222333333334444444455000001340000000000001350"), 0x01c4c1), - (hex!("0122222222333333334444444455000001340000000000005c60"), 0x01c561), - (hex!("0122222222333333334444444455000001350000000000001360"), 0x01c601), - (hex!("0122222222333333334444444455000001350000000000004f10"), 0x01c6a1), - (hex!("0122222222333333334444444455000001360000000000001370"), 0x01c741), - (hex!("0122222222333333334444444455000001360000000000004c60"), 0x01c7e1), - (hex!("0122222222333333334444444455000001370000000000001380"), 0x01c881), - (hex!("0122222222333333334444444455000001380000000000001390"), 0x01c921), - (hex!("01222222223333333344444444550000013900000000000013a0"), 0x01c9c1), - (hex!("0122222222333333334444444455000001390000000000004ea0"), 0x01ca61), - (hex!("01222222223333333344444444550000013a00000000000013b0"), 0x01cb01), - (hex!("01222222223333333344444444550000013a0000000000007350"), 0x01cba1), - (hex!("01222222223333333344444444550000013b00000000000013c0"), 0x01cc41), - (hex!("01222222223333333344444444550000013c00000000000013d0"), 0x01cce1), - (hex!("01222222223333333344444444550000013c0000000000007050"), 0x01cd81), - (hex!("01222222223333333344444444550000013d00000000000013e0"), 0x01ce21), - (hex!("01222222223333333344444444550000013d0000000000006bd0"), 0x01cec1), - (hex!("01222222223333333344444444550000013e00000000000013f0"), 0x01cf61), - (hex!("01222222223333333344444444550000013e00000000000058e0"), 0x01d001), - (hex!("01222222223333333344444444550000013f0000000000001400"), 0x01d0a1), - (hex!("01222222223333333344444444550000013f0000000000004740"), 0x01d141), - (hex!("0122222222333333334444444455000001400000000000001410"), 0x01d1e1), - (hex!("0122222222333333334444444455000001400000000000003f10"), 0x01d281), - (hex!("0122222222333333334444444455000001400000000000006d40"), 0x01d321), - (hex!("01222222223333333344444444550000014000000000000072d0"), 0x01d3c1), - (hex!("0122222222333333334444444455000001410000000000001420"), 0x01d461), - (hex!("0122222222333333334444444455000001420000000000001430"), 0x01d501), - (hex!("0122222222333333334444444455000001430000000000001440"), 0x01d5a1), - (hex!("0122222222333333334444444455000001440000000000001450"), 0x01d641), - (hex!("0122222222333333334444444455000001450000000000001460"), 0x01d6e1), - (hex!("0122222222333333334444444455000001460000000000001470"), 0x01d781), - (hex!("01222222223333333344444444550000014600000000000055c0"), 0x01d821), - (hex!("0122222222333333334444444455000001470000000000001480"), 0x01d8c1), - (hex!("0122222222333333334444444455000001470000000000004570"), 0x01d961), - (hex!("0122222222333333334444444455000001470000000000004be0"), 0x01da01), - (hex!("0122222222333333334444444455000001480000000000001490"), 0x01daa1), - (hex!("0122222222333333334444444455000001480000000000005360"), 0x01db41), - (hex!("01222222223333333344444444550000014900000000000014a0"), 0x01dbe1), - (hex!("01222222223333333344444444550000014a00000000000014b0"), 0x01dc81), - (hex!("01222222223333333344444444550000014a00000000000053d0"), 0x01dd21), - (hex!("01222222223333333344444444550000014b00000000000014c0"), 0x01ddc1), - (hex!("01222222223333333344444444550000014b0000000000005950"), 0x01de61), - (hex!("01222222223333333344444444550000014c00000000000014d0"), 0x01df01), - (hex!("01222222223333333344444444550000014c0000000000004f60"), 0x01dfa1), - (hex!("01222222223333333344444444550000014d00000000000014e0"), 0x01e041), - (hex!("01222222223333333344444444550000014d0000000000004520"), 0x01e0e1), - (hex!("01222222223333333344444444550000014d0000000000005200"), 0x01e181), - (hex!("01222222223333333344444444550000014e00000000000014f0"), 0x01e221), - (hex!("01222222223333333344444444550000014e0000000000005bd0"), 0x01e2c1), - (hex!("01222222223333333344444444550000014f0000000000001500"), 0x01e361), - (hex!("01222222223333333344444444550000014f00000000000060d0"), 0x01e401), - (hex!("0122222222333333334444444455000001500000000000001510"), 0x01e4a1), - (hex!("01222222223333333344444444550000015000000000000075e0"), 0x01e541), - (hex!("0122222222333333334444444455000001510000000000001520"), 0x01e5e1), - (hex!("0122222222333333334444444455000001510000000000005c00"), 0x01e681), - (hex!("0122222222333333334444444455000001510000000000006af0"), 0x01e721), - (hex!("0122222222333333334444444455000001510000000000007b80"), 0x01e7c1), - (hex!("0122222222333333334444444455000001520000000000001530"), 0x01e861), - (hex!("0122222222333333334444444455000001520000000000004c70"), 0x01e901), - (hex!("0122222222333333334444444455000001530000000000001540"), 0x01e9a1), - (hex!("0122222222333333334444444455000001540000000000001550"), 0x01ea41), - (hex!("0122222222333333334444444455000001540000000000007cd0"), 0x01eae1), - (hex!("0122222222333333334444444455000001550000000000001560"), 0x01eb81), - (hex!("0122222222333333334444444455000001550000000000004ae0"), 0x01ec21), - (hex!("01222222223333333344444444550000015500000000000068c0"), 0x01ecc1), - (hex!("0122222222333333334444444455000001560000000000001570"), 0x01ed61), - (hex!("01222222223333333344444444550000015600000000000064a0"), 0x01ee01), - (hex!("0122222222333333334444444455000001570000000000001580"), 0x01eea1), - (hex!("0122222222333333334444444455000001580000000000001590"), 0x01ef41), - (hex!("0122222222333333334444444455000001580000000000006d30"), 0x01efe1), - (hex!("01222222223333333344444444550000015800000000000074f0"), 0x01f081), - (hex!("01222222223333333344444444550000015900000000000015a0"), 0x01f121), - (hex!("01222222223333333344444444550000015900000000000053a0"), 0x01f1c1), - (hex!("01222222223333333344444444550000015900000000000055e0"), 0x01f261), - (hex!("0122222222333333334444444455000001590000000000006210"), 0x01f301), - (hex!("01222222223333333344444444550000015900000000000067c0"), 0x01f3a1), - (hex!("01222222223333333344444444550000015a00000000000015b0"), 0x01f441), - (hex!("01222222223333333344444444550000015b00000000000015c0"), 0x01f4e1), - (hex!("01222222223333333344444444550000015c00000000000015d0"), 0x01f581), - (hex!("01222222223333333344444444550000015c0000000000004d80"), 0x01f621), - (hex!("01222222223333333344444444550000015c00000000000073f0"), 0x01f6c1), - (hex!("01222222223333333344444444550000015d00000000000015e0"), 0x01f761), - (hex!("01222222223333333344444444550000015e00000000000015f0"), 0x01f801), - (hex!("01222222223333333344444444550000015e0000000000004120"), 0x01f8a1), - (hex!("01222222223333333344444444550000015e0000000000004350"), 0x01f941), - (hex!("01222222223333333344444444550000015e0000000000007c50"), 0x01f9e1), - (hex!("01222222223333333344444444550000015f0000000000001600"), 0x01fa81), - (hex!("0122222222333333334444444455000001600000000000001610"), 0x01fb21), - (hex!("0122222222333333334444444455000001600000000000004840"), 0x01fbc1), - (hex!("0122222222333333334444444455000001600000000000004b10"), 0x01fc61), - (hex!("0122222222333333334444444455000001600000000000007060"), 0x01fd01), - (hex!("0122222222333333334444444455000001610000000000001620"), 0x01fda1), - (hex!("0122222222333333334444444455000001610000000000005300"), 0x01fe41), - (hex!("0122222222333333334444444455000001620000000000001630"), 0x01fee1), - (hex!("0122222222333333334444444455000001620000000000006530"), 0x01ff81), - (hex!("0122222222333333334444444455000001630000000000001640"), 0x020021), - (hex!("0122222222333333334444444455000001640000000000001650"), 0x0200c1), - (hex!("0122222222333333334444444455000001650000000000001660"), 0x020161), - (hex!("0122222222333333334444444455000001660000000000001670"), 0x020201), - (hex!("0122222222333333334444444455000001670000000000001680"), 0x0202a1), - (hex!("0122222222333333334444444455000001670000000000007310"), 0x020341), - (hex!("0122222222333333334444444455000001680000000000001690"), 0x0203e1), - (hex!("0122222222333333334444444455000001680000000000007b50"), 0x020481), - (hex!("01222222223333333344444444550000016900000000000016a0"), 0x020521), - (hex!("01222222223333333344444444550000016900000000000049d0"), 0x0205c1), - (hex!("01222222223333333344444444550000016a00000000000016b0"), 0x020661), - (hex!("01222222223333333344444444550000016a00000000000078b0"), 0x020701), - (hex!("01222222223333333344444444550000016b00000000000016c0"), 0x0207a1), - (hex!("01222222223333333344444444550000016b0000000000004100"), 0x020841), - (hex!("01222222223333333344444444550000016c00000000000016d0"), 0x0208e1), - (hex!("01222222223333333344444444550000016c0000000000006e00"), 0x020981), - (hex!("01222222223333333344444444550000016d00000000000016e0"), 0x020a21), - (hex!("01222222223333333344444444550000016e00000000000016f0"), 0x020ac1), - (hex!("01222222223333333344444444550000016e0000000000004ac0"), 0x020b61), - (hex!("01222222223333333344444444550000016e0000000000007820"), 0x020c01), - (hex!("01222222223333333344444444550000016f0000000000001700"), 0x020ca1), - (hex!("0122222222333333334444444455000001700000000000001710"), 0x020d41), - (hex!("0122222222333333334444444455000001700000000000005830"), 0x020de1), - (hex!("0122222222333333334444444455000001710000000000001720"), 0x020e81), - (hex!("01222222223333333344444444550000017100000000000072f0"), 0x020f21), - (hex!("0122222222333333334444444455000001720000000000001730"), 0x020fc1), - (hex!("0122222222333333334444444455000001720000000000004870"), 0x021061), - (hex!("01222222223333333344444444550000017200000000000070b0"), 0x021101), - (hex!("0122222222333333334444444455000001730000000000001740"), 0x0211a1), - (hex!("0122222222333333334444444455000001740000000000001750"), 0x021241), - (hex!("0122222222333333334444444455000001750000000000001760"), 0x0212e1), - (hex!("0122222222333333334444444455000001750000000000005670"), 0x021381), - (hex!("0122222222333333334444444455000001750000000000005870"), 0x021421), - (hex!("0122222222333333334444444455000001760000000000001770"), 0x0214c1), - (hex!("0122222222333333334444444455000001770000000000001780"), 0x021561), - (hex!("0122222222333333334444444455000001770000000000005000"), 0x021601), - (hex!("0122222222333333334444444455000001770000000000007090"), 0x0216a1), - (hex!("0122222222333333334444444455000001780000000000001790"), 0x021741), - (hex!("01222222223333333344444444550000017800000000000048a0"), 0x0217e1), - (hex!("0122222222333333334444444455000001780000000000006bf0"), 0x021881), - (hex!("01222222223333333344444444550000017900000000000017a0"), 0x021921), - (hex!("01222222223333333344444444550000017900000000000057d0"), 0x0219c1), - (hex!("0122222222333333334444444455000001790000000000006660"), 0x021a61), - (hex!("01222222223333333344444444550000017a00000000000017b0"), 0x021b01), - (hex!("01222222223333333344444444550000017a0000000000004970"), 0x021ba1), - (hex!("01222222223333333344444444550000017a0000000000005dc0"), 0x021c41), - (hex!("01222222223333333344444444550000017b00000000000017c0"), 0x021ce1), - (hex!("01222222223333333344444444550000017b0000000000004ee0"), 0x021d81), - (hex!("01222222223333333344444444550000017b00000000000054c0"), 0x021e21), - (hex!("01222222223333333344444444550000017c00000000000017d0"), 0x021ec1), - (hex!("01222222223333333344444444550000017c0000000000003fc0"), 0x021f61), - (hex!("01222222223333333344444444550000017c00000000000063e0"), 0x022001), - (hex!("01222222223333333344444444550000017c0000000000006520"), 0x0220a1), - (hex!("01222222223333333344444444550000017d00000000000017e0"), 0x022141), - (hex!("01222222223333333344444444550000017d0000000000006220"), 0x0221e1), - (hex!("01222222223333333344444444550000017d0000000000007120"), 0x022281), - (hex!("01222222223333333344444444550000017e00000000000017f0"), 0x022321), - (hex!("01222222223333333344444444550000017f0000000000001800"), 0x0223c1), - (hex!("0122222222333333334444444455000001800000000000001810"), 0x022461), - (hex!("0122222222333333334444444455000001810000000000001820"), 0x022501), - (hex!("01222222223333333344444444550000018100000000000041f0"), 0x0225a1), - (hex!("0122222222333333334444444455000001810000000000007590"), 0x022641), - (hex!("0122222222333333334444444455000001820000000000001830"), 0x0226e1), - (hex!("0122222222333333334444444455000001820000000000004ce0"), 0x022781), - (hex!("0122222222333333334444444455000001830000000000001840"), 0x022821), - (hex!("01222222223333333344444444550000018300000000000042c0"), 0x0228c1), - (hex!("0122222222333333334444444455000001840000000000001850"), 0x022961), - (hex!("0122222222333333334444444455000001840000000000004f70"), 0x022a01), - (hex!("0122222222333333334444444455000001850000000000001860"), 0x022aa1), - (hex!("0122222222333333334444444455000001850000000000006470"), 0x022b41), - (hex!("0122222222333333334444444455000001850000000000007500"), 0x022be1), - (hex!("0122222222333333334444444455000001860000000000001870"), 0x022c81), - (hex!("0122222222333333334444444455000001860000000000004770"), 0x022d21), - (hex!("0122222222333333334444444455000001870000000000001880"), 0x022dc1), - (hex!("0122222222333333334444444455000001870000000000006a30"), 0x022e61), - (hex!("0122222222333333334444444455000001880000000000001890"), 0x022f01), - (hex!("0122222222333333334444444455000001880000000000007410"), 0x022fa1), - (hex!("01222222223333333344444444550000018900000000000018a0"), 0x023041), - (hex!("01222222223333333344444444550000018900000000000044d0"), 0x0230e1), - (hex!("0122222222333333334444444455000001890000000000005ac0"), 0x023181), - (hex!("01222222223333333344444444550000018a00000000000018b0"), 0x023221), - (hex!("01222222223333333344444444550000018a0000000000006260"), 0x0232c1), - (hex!("01222222223333333344444444550000018a0000000000006d70"), 0x023361), - (hex!("01222222223333333344444444550000018b00000000000018c0"), 0x023401), - (hex!("01222222223333333344444444550000018b0000000000004aa0"), 0x0234a1), - (hex!("01222222223333333344444444550000018b0000000000006fd0"), 0x023541), - (hex!("01222222223333333344444444550000018c00000000000018d0"), 0x0235e1), - (hex!("01222222223333333344444444550000018c00000000000051b0"), 0x023681), - (hex!("01222222223333333344444444550000018c0000000000006650"), 0x023721), - (hex!("01222222223333333344444444550000018d00000000000018e0"), 0x0237c1), - (hex!("01222222223333333344444444550000018e00000000000018f0"), 0x023861), - (hex!("01222222223333333344444444550000018e00000000000041d0"), 0x023901), - (hex!("01222222223333333344444444550000018f0000000000001900"), 0x0239a1), - (hex!("01222222223333333344444444550000018f0000000000007600"), 0x023a41), - (hex!("0122222222333333334444444455000001900000000000001910"), 0x023ae1), - (hex!("0122222222333333334444444455000001900000000000005410"), 0x023b81), - (hex!("0122222222333333334444444455000001900000000000006760"), 0x023c21), - (hex!("0122222222333333334444444455000001910000000000001920"), 0x023cc1), - (hex!("0122222222333333334444444455000001920000000000001930"), 0x023d61), - (hex!("0122222222333333334444444455000001920000000000004ca0"), 0x023e01), - (hex!("0122222222333333334444444455000001920000000000005d80"), 0x023ea1), - (hex!("0122222222333333334444444455000001920000000000005fd0"), 0x023f41), - (hex!("01222222223333333344444444550000019200000000000070d0"), 0x023fe1), - (hex!("0122222222333333334444444455000001930000000000001940"), 0x024081), - (hex!("0122222222333333334444444455000001930000000000004010"), 0x024121), - (hex!("0122222222333333334444444455000001930000000000007ca0"), 0x0241c1), - (hex!("0122222222333333334444444455000001940000000000001950"), 0x024261), - (hex!("0122222222333333334444444455000001950000000000001960"), 0x024301), - (hex!("0122222222333333334444444455000001950000000000005380"), 0x0243a1), - (hex!("0122222222333333334444444455000001960000000000001970"), 0x024441), - (hex!("0122222222333333334444444455000001960000000000006de0"), 0x0244e1), - (hex!("0122222222333333334444444455000001970000000000001980"), 0x024581), - (hex!("01222222223333333344444444550000019700000000000048f0"), 0x024621), - (hex!("0122222222333333334444444455000001980000000000001990"), 0x0246c1), - (hex!("0122222222333333334444444455000001980000000000006510"), 0x024761), - (hex!("01222222223333333344444444550000019900000000000019a0"), 0x024801), - (hex!("0122222222333333334444444455000001990000000000007570"), 0x0248a1), - (hex!("0122222222333333334444444455000001990000000000007580"), 0x024941), - (hex!("01222222223333333344444444550000019a00000000000019b0"), 0x0249e1), - (hex!("01222222223333333344444444550000019a0000000000004050"), 0x024a81), - (hex!("01222222223333333344444444550000019a0000000000004ba0"), 0x024b21), - (hex!("01222222223333333344444444550000019a0000000000005540"), 0x024bc1), - (hex!("01222222223333333344444444550000019a00000000000061c0"), 0x024c61), - (hex!("01222222223333333344444444550000019a0000000000007c60"), 0x024d01), - (hex!("01222222223333333344444444550000019b00000000000019c0"), 0x024da1), - (hex!("01222222223333333344444444550000019b0000000000006240"), 0x024e41), - (hex!("01222222223333333344444444550000019c00000000000019d0"), 0x024ee1), - (hex!("01222222223333333344444444550000019d00000000000019e0"), 0x024f81), - (hex!("01222222223333333344444444550000019d0000000000004640"), 0x025021), - (hex!("01222222223333333344444444550000019d00000000000052a0"), 0x0250c1), - (hex!("01222222223333333344444444550000019d00000000000052b0"), 0x025161), - (hex!("01222222223333333344444444550000019e00000000000019f0"), 0x025201), - (hex!("01222222223333333344444444550000019f0000000000001a00"), 0x0252a1), - (hex!("01222222223333333344444444550000019f0000000000006b20"), 0x025341), - (hex!("0122222222333333334444444455000001a00000000000001a10"), 0x0253e1), - (hex!("0122222222333333334444444455000001a10000000000001a20"), 0x025481), - (hex!("0122222222333333334444444455000001a10000000000005460"), 0x025521), - (hex!("0122222222333333334444444455000001a10000000000005d20"), 0x0255c1), - (hex!("0122222222333333334444444455000001a100000000000068f0"), 0x025661), - (hex!("0122222222333333334444444455000001a20000000000001a30"), 0x025701), - (hex!("0122222222333333334444444455000001a20000000000007170"), 0x0257a1), - (hex!("0122222222333333334444444455000001a30000000000001a40"), 0x025841), - (hex!("0122222222333333334444444455000001a40000000000001a50"), 0x0258e1), - (hex!("0122222222333333334444444455000001a50000000000001a60"), 0x025981), - (hex!("0122222222333333334444444455000001a60000000000001a70"), 0x025a21), - (hex!("0122222222333333334444444455000001a70000000000001a80"), 0x025ac1), - (hex!("0122222222333333334444444455000001a70000000000005a90"), 0x025b61), - (hex!("0122222222333333334444444455000001a70000000000006440"), 0x025c01), - (hex!("0122222222333333334444444455000001a80000000000001a90"), 0x025ca1), - (hex!("0122222222333333334444444455000001a80000000000004800"), 0x025d41), - (hex!("0122222222333333334444444455000001a90000000000001aa0"), 0x025de1), - (hex!("0122222222333333334444444455000001aa0000000000001ab0"), 0x025e81), - (hex!("0122222222333333334444444455000001aa0000000000005b60"), 0x025f21), - (hex!("0122222222333333334444444455000001ab0000000000001ac0"), 0x025fc1), - (hex!("0122222222333333334444444455000001ab0000000000006700"), 0x026061), - (hex!("0122222222333333334444444455000001ab00000000000071d0"), 0x026101), - (hex!("0122222222333333334444444455000001ac0000000000001ad0"), 0x0261a1), - (hex!("0122222222333333334444444455000001ac0000000000007380"), 0x026241), - (hex!("0122222222333333334444444455000001ad0000000000001ae0"), 0x0262e1), - (hex!("0122222222333333334444444455000001ad0000000000006350"), 0x026381), - (hex!("0122222222333333334444444455000001ae0000000000001af0"), 0x026421), - (hex!("0122222222333333334444444455000001af0000000000001b00"), 0x0264c1), - (hex!("0122222222333333334444444455000001af0000000000007390"), 0x026561), - (hex!("0122222222333333334444444455000001b00000000000001b10"), 0x026601), - (hex!("0122222222333333334444444455000001b10000000000001b20"), 0x0266a1), - (hex!("0122222222333333334444444455000001b10000000000005cc0"), 0x026741), - (hex!("0122222222333333334444444455000001b20000000000001b30"), 0x0267e1), - (hex!("0122222222333333334444444455000001b20000000000004fb0"), 0x026881), - (hex!("0122222222333333334444444455000001b30000000000001b40"), 0x026921), - (hex!("0122222222333333334444444455000001b40000000000001b50"), 0x0269c1), - (hex!("0122222222333333334444444455000001b50000000000001b60"), 0x026a61), - (hex!("0122222222333333334444444455000001b60000000000001b70"), 0x026b01), - (hex!("0122222222333333334444444455000001b600000000000048e0"), 0x026ba1), - (hex!("0122222222333333334444444455000001b70000000000001b80"), 0x026c41), - (hex!("0122222222333333334444444455000001b70000000000005ca0"), 0x026ce1), - (hex!("0122222222333333334444444455000001b70000000000007900"), 0x026d81), - (hex!("0122222222333333334444444455000001b80000000000001b90"), 0x026e21), - (hex!("0122222222333333334444444455000001b80000000000004d90"), 0x026ec1), - (hex!("0122222222333333334444444455000001b90000000000001ba0"), 0x026f61), - (hex!("0122222222333333334444444455000001b90000000000003f40"), 0x027001), - (hex!("0122222222333333334444444455000001ba0000000000001bb0"), 0x0270a1), - (hex!("0122222222333333334444444455000001ba00000000000042a0"), 0x027141), - (hex!("0122222222333333334444444455000001ba00000000000067f0"), 0x0271e1), - (hex!("0122222222333333334444444455000001ba00000000000073a0"), 0x027281), - (hex!("0122222222333333334444444455000001bb0000000000001bc0"), 0x027321), - (hex!("0122222222333333334444444455000001bb0000000000004a00"), 0x0273c1), - (hex!("0122222222333333334444444455000001bb0000000000005e00"), 0x027461), - (hex!("0122222222333333334444444455000001bc0000000000001bd0"), 0x027501), - (hex!("0122222222333333334444444455000001bc0000000000004230"), 0x0275a1), - (hex!("0122222222333333334444444455000001bc0000000000005860"), 0x027641), - (hex!("0122222222333333334444444455000001bd0000000000001be0"), 0x0276e1), - (hex!("0122222222333333334444444455000001bd0000000000007c70"), 0x027781), - (hex!("0122222222333333334444444455000001be0000000000001bf0"), 0x027821), - (hex!("0122222222333333334444444455000001be0000000000007770"), 0x0278c1), - (hex!("0122222222333333334444444455000001be0000000000007cf0"), 0x027961), - (hex!("0122222222333333334444444455000001bf0000000000001c00"), 0x027a01), - (hex!("0122222222333333334444444455000001bf0000000000006490"), 0x027aa1), - (hex!("0122222222333333334444444455000001c00000000000001c10"), 0x027b41), - (hex!("0122222222333333334444444455000001c10000000000001c20"), 0x027be1), - (hex!("0122222222333333334444444455000001c10000000000004600"), 0x027c81), - (hex!("0122222222333333334444444455000001c20000000000001c30"), 0x027d21), - (hex!("0122222222333333334444444455000001c20000000000006e30"), 0x027dc1), - (hex!("0122222222333333334444444455000001c30000000000001c40"), 0x027e61), - (hex!("0122222222333333334444444455000001c40000000000001c50"), 0x027f01), - (hex!("0122222222333333334444444455000001c50000000000001c60"), 0x027fa1), - (hex!("0122222222333333334444444455000001c60000000000001c70"), 0x028041), - (hex!("0122222222333333334444444455000001c60000000000004240"), 0x0280e1), - (hex!("0122222222333333334444444455000001c60000000000005bb0"), 0x028181), - (hex!("0122222222333333334444444455000001c70000000000001c80"), 0x028221), - (hex!("0122222222333333334444444455000001c80000000000001c90"), 0x0282c1), - (hex!("0122222222333333334444444455000001c90000000000001ca0"), 0x028361), - (hex!("0122222222333333334444444455000001c90000000000006730"), 0x028401), - (hex!("0122222222333333334444444455000001ca0000000000001cb0"), 0x0284a1), - (hex!("0122222222333333334444444455000001ca00000000000070f0"), 0x028541), - (hex!("0122222222333333334444444455000001cb0000000000001cc0"), 0x0285e1), - (hex!("0122222222333333334444444455000001cb00000000000071a0"), 0x028681), - (hex!("0122222222333333334444444455000001cc0000000000001cd0"), 0x028721), - (hex!("0122222222333333334444444455000001cc0000000000005280"), 0x0287c1), - (hex!("0122222222333333334444444455000001cc0000000000005d90"), 0x028861), - (hex!("0122222222333333334444444455000001cd0000000000001ce0"), 0x028901), - (hex!("0122222222333333334444444455000001cd00000000000069b0"), 0x0289a1), - (hex!("0122222222333333334444444455000001ce0000000000001cf0"), 0x028a41), - (hex!("0122222222333333334444444455000001ce0000000000004540"), 0x028ae1), - (hex!("0122222222333333334444444455000001cf0000000000001d00"), 0x028b81), - (hex!("0122222222333333334444444455000001cf00000000000076a0"), 0x028c21), - (hex!("0122222222333333334444444455000001d00000000000001d10"), 0x028cc1), - (hex!("0122222222333333334444444455000001d000000000000060a0"), 0x028d61), - (hex!("0122222222333333334444444455000001d10000000000001d20"), 0x028e01), - (hex!("0122222222333333334444444455000001d20000000000001d30"), 0x028ea1), - (hex!("0122222222333333334444444455000001d30000000000001d40"), 0x028f41), - (hex!("0122222222333333334444444455000001d30000000000004000"), 0x028fe1), - (hex!("0122222222333333334444444455000001d30000000000004140"), 0x029081), - (hex!("0122222222333333334444444455000001d30000000000006790"), 0x029121), - (hex!("0122222222333333334444444455000001d40000000000001d50"), 0x0291c1), - (hex!("0122222222333333334444444455000001d50000000000001d60"), 0x029261), - (hex!("0122222222333333334444444455000001d60000000000001d70"), 0x029301), - (hex!("0122222222333333334444444455000001d60000000000004b50"), 0x0293a1), - (hex!("0122222222333333334444444455000001d60000000000007430"), 0x029441), - (hex!("0122222222333333334444444455000001d70000000000001d80"), 0x0294e1), - (hex!("0122222222333333334444444455000001d70000000000006920"), 0x029581), - (hex!("0122222222333333334444444455000001d80000000000001d90"), 0x029621), - (hex!("0122222222333333334444444455000001d80000000000005b30"), 0x0296c1), - (hex!("0122222222333333334444444455000001d90000000000001da0"), 0x029761), - (hex!("0122222222333333334444444455000001da0000000000001db0"), 0x029801), - (hex!("0122222222333333334444444455000001da0000000000004af0"), 0x0298a1), - (hex!("0122222222333333334444444455000001da0000000000007240"), 0x029941), - (hex!("0122222222333333334444444455000001da0000000000007470"), 0x0299e1), - (hex!("0122222222333333334444444455000001db0000000000001dc0"), 0x029a81), - (hex!("0122222222333333334444444455000001db00000000000045d0"), 0x029b21), - (hex!("0122222222333333334444444455000001dc0000000000001dd0"), 0x029bc1), - (hex!("0122222222333333334444444455000001dd0000000000001de0"), 0x029c61), - (hex!("0122222222333333334444444455000001dd0000000000004bb0"), 0x029d01), - (hex!("0122222222333333334444444455000001dd0000000000004cd0"), 0x029da1), - (hex!("0122222222333333334444444455000001dd0000000000006100"), 0x029e41), - (hex!("0122222222333333334444444455000001dd0000000000007bb0"), 0x029ee1), - (hex!("0122222222333333334444444455000001de0000000000001df0"), 0x029f81), - (hex!("0122222222333333334444444455000001de0000000000004260"), 0x02a021), - (hex!("0122222222333333334444444455000001de0000000000006040"), 0x02a0c1), - (hex!("0122222222333333334444444455000001df0000000000001e00"), 0x02a161), - (hex!("0122222222333333334444444455000001df0000000000005fa0"), 0x02a201), - (hex!("0122222222333333334444444455000001df0000000000006a70"), 0x02a2a1), - (hex!("0122222222333333334444444455000001df0000000000006dc0"), 0x02a341), - (hex!("0122222222333333334444444455000001e00000000000001e10"), 0x02a3e1), - (hex!("0122222222333333334444444455000001e00000000000007010"), 0x02a481), - (hex!("0122222222333333334444444455000001e10000000000001e20"), 0x02a521), - (hex!("0122222222333333334444444455000001e10000000000005720"), 0x02a5c1), - (hex!("0122222222333333334444444455000001e10000000000006830"), 0x02a661), - (hex!("0122222222333333334444444455000001e20000000000001e30"), 0x02a701), - (hex!("0122222222333333334444444455000001e20000000000005100"), 0x02a7a1), - (hex!("0122222222333333334444444455000001e30000000000001e40"), 0x02a841), - (hex!("0122222222333333334444444455000001e40000000000001e50"), 0x02a8e1), - (hex!("0122222222333333334444444455000001e40000000000003f30"), 0x02a981), - (hex!("0122222222333333334444444455000001e40000000000005220"), 0x02aa21), - (hex!("0122222222333333334444444455000001e50000000000001e60"), 0x02aac1), - (hex!("0122222222333333334444444455000001e50000000000006f60"), 0x02ab61), - (hex!("0122222222333333334444444455000001e60000000000001e70"), 0x02ac01), - (hex!("0122222222333333334444444455000001e60000000000006c80"), 0x02aca1), - (hex!("0122222222333333334444444455000001e70000000000001e80"), 0x02ad41), - (hex!("0122222222333333334444444455000001e80000000000001e90"), 0x02ade1), - (hex!("0122222222333333334444444455000001e80000000000004e30"), 0x02ae81), - (hex!("0122222222333333334444444455000001e90000000000001ea0"), 0x02af21), - (hex!("0122222222333333334444444455000001e90000000000005470"), 0x02afc1), - (hex!("0122222222333333334444444455000001ea0000000000001eb0"), 0x02b061), - (hex!("0122222222333333334444444455000001ea0000000000007980"), 0x02b101), - (hex!("0122222222333333334444444455000001eb0000000000001ec0"), 0x02b1a1), - (hex!("0122222222333333334444444455000001eb0000000000004390"), 0x02b241), - (hex!("0122222222333333334444444455000001eb0000000000005970"), 0x02b2e1), - (hex!("0122222222333333334444444455000001ec0000000000001ed0"), 0x02b381), - (hex!("0122222222333333334444444455000001ec0000000000005d50"), 0x02b421), - (hex!("0122222222333333334444444455000001ec00000000000076e0"), 0x02b4c1), - (hex!("0122222222333333334444444455000001ed0000000000001ee0"), 0x02b561), - (hex!("0122222222333333334444444455000001ed0000000000006190"), 0x02b601), - (hex!("0122222222333333334444444455000001ee0000000000001ef0"), 0x02b6a1), - (hex!("0122222222333333334444444455000001ee0000000000004900"), 0x02b741), - (hex!("0122222222333333334444444455000001ef0000000000001f00"), 0x02b7e1), - (hex!("0122222222333333334444444455000001ef0000000000006c60"), 0x02b881), - (hex!("0122222222333333334444444455000001f00000000000001f10"), 0x02b921), - (hex!("0122222222333333334444444455000001f00000000000006950"), 0x02b9c1), - (hex!("0122222222333333334444444455000001f10000000000001f20"), 0x02ba61), - (hex!("0122222222333333334444444455000001f10000000000006400"), 0x02bb01), - (hex!("0122222222333333334444444455000001f20000000000001f30"), 0x02bba1), - (hex!("0122222222333333334444444455000001f20000000000006f00"), 0x02bc41), - (hex!("0122222222333333334444444455000001f20000000000007b10"), 0x02bce1), - (hex!("0122222222333333334444444455000001f30000000000001f40"), 0x02bd81), - (hex!("0122222222333333334444444455000001f40000000000001f50"), 0x02be21), - (hex!("0122222222333333334444444455000001f50000000000001f60"), 0x02bec1), - (hex!("0122222222333333334444444455000001f500000000000044f0"), 0x02bf61), - (hex!("0122222222333333334444444455000001f60000000000001f70"), 0x02c001), - (hex!("0122222222333333334444444455000001f70000000000001f80"), 0x02c0a1), - (hex!("0122222222333333334444444455000001f70000000000004ad0"), 0x02c141), - (hex!("0122222222333333334444444455000001f80000000000001f90"), 0x02c1e1), - (hex!("0122222222333333334444444455000001f90000000000001fa0"), 0x02c281), - (hex!("0122222222333333334444444455000001f90000000000003f60"), 0x02c321), - (hex!("0122222222333333334444444455000001f90000000000004a80"), 0x02c3c1), - (hex!("0122222222333333334444444455000001fa0000000000001fb0"), 0x02c461), - (hex!("0122222222333333334444444455000001fa0000000000006f90"), 0x02c501), - (hex!("0122222222333333334444444455000001fb0000000000001fc0"), 0x02c5a1), - (hex!("0122222222333333334444444455000001fc0000000000001fd0"), 0x02c641), - (hex!("0122222222333333334444444455000001fc0000000000004a90"), 0x02c6e1), - (hex!("0122222222333333334444444455000001fd0000000000001fe0"), 0x02c781), - (hex!("0122222222333333334444444455000001fd0000000000005f50"), 0x02c821), - (hex!("0122222222333333334444444455000001fe0000000000001ff0"), 0x02c8c1), - (hex!("0122222222333333334444444455000001ff0000000000002000"), 0x02c961), - (hex!("0122222222333333334444444455000002000000000000002010"), 0x02ca01), - (hex!("0122222222333333334444444455000002000000000000005f00"), 0x02caa1), - (hex!("0122222222333333334444444455000002000000000000006840"), 0x02cb41), - (hex!("0122222222333333334444444455000002010000000000002020"), 0x02cbe1), - (hex!("0122222222333333334444444455000002020000000000002030"), 0x02cc81), - (hex!("0122222222333333334444444455000002030000000000002040"), 0x02cd21), - (hex!("0122222222333333334444444455000002040000000000002050"), 0x02cdc1), - (hex!("01222222223333333344444444550000020400000000000051f0"), 0x02ce61), - (hex!("0122222222333333334444444455000002050000000000002060"), 0x02cf01), - (hex!("0122222222333333334444444455000002060000000000002070"), 0x02cfa1), - (hex!("0122222222333333334444444455000002060000000000005c80"), 0x02d041), - (hex!("01222222223333333344444444550000020600000000000061d0"), 0x02d0e1), - (hex!("01222222223333333344444444550000020600000000000078c0"), 0x02d181), - (hex!("0122222222333333334444444455000002070000000000002080"), 0x02d221), - (hex!("0122222222333333334444444455000002070000000000006ba0"), 0x02d2c1), - (hex!("0122222222333333334444444455000002080000000000002090"), 0x02d361), - (hex!("01222222223333333344444444550000020900000000000020a0"), 0x02d401), - (hex!("01222222223333333344444444550000020900000000000067a0"), 0x02d4a1), - (hex!("01222222223333333344444444550000020a00000000000020b0"), 0x02d541), - (hex!("01222222223333333344444444550000020a0000000000004950"), 0x02d5e1), - (hex!("01222222223333333344444444550000020a0000000000004de0"), 0x02d681), - (hex!("01222222223333333344444444550000020b00000000000020c0"), 0x02d721), - (hex!("01222222223333333344444444550000020b0000000000004b00"), 0x02d7c1), - (hex!("01222222223333333344444444550000020c00000000000020d0"), 0x02d861), - (hex!("01222222223333333344444444550000020d00000000000020e0"), 0x02d901), - (hex!("01222222223333333344444444550000020e00000000000020f0"), 0x02d9a1), - (hex!("01222222223333333344444444550000020f0000000000002100"), 0x02da41), - (hex!("0122222222333333334444444455000002100000000000002110"), 0x02dae1), - (hex!("0122222222333333334444444455000002110000000000002120"), 0x02db81), - (hex!("0122222222333333334444444455000002110000000000004490"), 0x02dc21), - (hex!("0122222222333333334444444455000002120000000000002130"), 0x02dcc1), - (hex!("0122222222333333334444444455000002130000000000002140"), 0x02dd61), - (hex!("01222222223333333344444444550000021300000000000046d0"), 0x02de01), - (hex!("01222222223333333344444444550000021300000000000046e0"), 0x02dea1), - (hex!("0122222222333333334444444455000002130000000000004b70"), 0x02df41), - (hex!("0122222222333333334444444455000002140000000000002150"), 0x02dfe1), - (hex!("0122222222333333334444444455000002140000000000006c50"), 0x02e081), - (hex!("0122222222333333334444444455000002150000000000002160"), 0x02e121), - (hex!("01222222223333333344444444550000021500000000000043c0"), 0x02e1c1), - (hex!("0122222222333333334444444455000002160000000000002170"), 0x02e261), - (hex!("01222222223333333344444444550000021600000000000055b0"), 0x02e301), - (hex!("0122222222333333334444444455000002160000000000006150"), 0x02e3a1), - (hex!("0122222222333333334444444455000002170000000000002180"), 0x02e441), - (hex!("01222222223333333344444444550000021700000000000053b0"), 0x02e4e1), - (hex!("0122222222333333334444444455000002170000000000007460"), 0x02e581), - (hex!("0122222222333333334444444455000002180000000000002190"), 0x02e621), - (hex!("01222222223333333344444444550000021900000000000021a0"), 0x02e6c1), - (hex!("01222222223333333344444444550000021a00000000000021b0"), 0x02e761), - (hex!("01222222223333333344444444550000021a0000000000007650"), 0x02e801), - (hex!("01222222223333333344444444550000021b00000000000021c0"), 0x02e8a1), - (hex!("01222222223333333344444444550000021b0000000000004b20"), 0x02e941), - (hex!("01222222223333333344444444550000021c00000000000021d0"), 0x02e9e1), - (hex!("01222222223333333344444444550000021c0000000000007610"), 0x02ea81), - (hex!("01222222223333333344444444550000021d00000000000021e0"), 0x02eb21), - (hex!("01222222223333333344444444550000021d0000000000005f40"), 0x02ebc1), - (hex!("01222222223333333344444444550000021e00000000000021f0"), 0x02ec61), - (hex!("01222222223333333344444444550000021e0000000000005a50"), 0x02ed01), - (hex!("01222222223333333344444444550000021e0000000000005ff0"), 0x02eda1), - (hex!("01222222223333333344444444550000021f0000000000002200"), 0x02ee41), - (hex!("01222222223333333344444444550000021f00000000000043a0"), 0x02eee1), - (hex!("01222222223333333344444444550000021f0000000000004cb0"), 0x02ef81), - (hex!("01222222223333333344444444550000021f0000000000004e00"), 0x02f021), - (hex!("0122222222333333334444444455000002200000000000002210"), 0x02f0c1), - (hex!("0122222222333333334444444455000002210000000000002220"), 0x02f161), - (hex!("0122222222333333334444444455000002210000000000006290"), 0x02f201), - (hex!("0122222222333333334444444455000002210000000000007230"), 0x02f2a1), - (hex!("0122222222333333334444444455000002220000000000002230"), 0x02f341), - (hex!("0122222222333333334444444455000002220000000000006ea0"), 0x02f3e1), - (hex!("0122222222333333334444444455000002230000000000002240"), 0x02f481), - (hex!("0122222222333333334444444455000002230000000000004710"), 0x02f521), - (hex!("0122222222333333334444444455000002240000000000002250"), 0x02f5c1), - (hex!("0122222222333333334444444455000002250000000000002260"), 0x02f661), - (hex!("0122222222333333334444444455000002260000000000002270"), 0x02f701), - (hex!("0122222222333333334444444455000002260000000000005b40"), 0x02f7a1), - (hex!("0122222222333333334444444455000002260000000000006300"), 0x02f841), - (hex!("0122222222333333334444444455000002270000000000002280"), 0x02f8e1), - (hex!("0122222222333333334444444455000002270000000000005b80"), 0x02f981), - (hex!("0122222222333333334444444455000002280000000000002290"), 0x02fa21), - (hex!("0122222222333333334444444455000002280000000000003ed0"), 0x02fac1), - (hex!("0122222222333333334444444455000002280000000000004550"), 0x02fb61), - (hex!("01222222223333333344444444550000022800000000000077d0"), 0x02fc01), - (hex!("01222222223333333344444444550000022900000000000022a0"), 0x02fca1), - (hex!("0122222222333333334444444455000002290000000000006480"), 0x02fd41), - (hex!("01222222223333333344444444550000022a00000000000022b0"), 0x02fde1), - (hex!("01222222223333333344444444550000022a0000000000005450"), 0x02fe81), - (hex!("01222222223333333344444444550000022b00000000000022c0"), 0x02ff21), - (hex!("01222222223333333344444444550000022b0000000000006dd0"), 0x02ffc1), - (hex!("01222222223333333344444444550000022c00000000000022d0"), 0x030061), - (hex!("01222222223333333344444444550000022c0000000000006890"), 0x030101), - (hex!("01222222223333333344444444550000022d00000000000022e0"), 0x0301a1), - (hex!("01222222223333333344444444550000022e00000000000022f0"), 0x030241), - (hex!("01222222223333333344444444550000022e0000000000004f20"), 0x0302e1), - (hex!("01222222223333333344444444550000022f0000000000002300"), 0x030381), - (hex!("01222222223333333344444444550000022f0000000000005260"), 0x030421), - (hex!("01222222223333333344444444550000022f00000000000053f0"), 0x0304c1), - (hex!("0122222222333333334444444455000002300000000000002310"), 0x030561), - (hex!("01222222223333333344444444550000023000000000000050e0"), 0x030601), - (hex!("0122222222333333334444444455000002310000000000002320"), 0x0306a1), - (hex!("0122222222333333334444444455000002310000000000007800"), 0x030741), - (hex!("0122222222333333334444444455000002320000000000002330"), 0x0307e1), - (hex!("0122222222333333334444444455000002330000000000002340"), 0x030881), - (hex!("0122222222333333334444444455000002330000000000004d70"), 0x030921), - (hex!("0122222222333333334444444455000002330000000000005cf0"), 0x0309c1), - (hex!("0122222222333333334444444455000002340000000000002350"), 0x030a61), - (hex!("0122222222333333334444444455000002350000000000002360"), 0x030b01), - (hex!("0122222222333333334444444455000002350000000000006970"), 0x030ba1), - (hex!("0122222222333333334444444455000002360000000000002370"), 0x030c41), - (hex!("0122222222333333334444444455000002360000000000005270"), 0x030ce1), - (hex!("0122222222333333334444444455000002370000000000002380"), 0x030d81), - (hex!("0122222222333333334444444455000002370000000000005d70"), 0x030e21), - (hex!("0122222222333333334444444455000002380000000000002390"), 0x030ec1), - (hex!("01222222223333333344444444550000023800000000000069a0"), 0x030f61), - (hex!("01222222223333333344444444550000023900000000000023a0"), 0x031001), - (hex!("01222222223333333344444444550000023900000000000052e0"), 0x0310a1), - (hex!("0122222222333333334444444455000002390000000000005a10"), 0x031141), - (hex!("0122222222333333334444444455000002390000000000007440"), 0x0311e1), - (hex!("01222222223333333344444444550000023a00000000000023b0"), 0x031281), - (hex!("01222222223333333344444444550000023a0000000000003f00"), 0x031321), - (hex!("01222222223333333344444444550000023a0000000000004430"), 0x0313c1), - (hex!("01222222223333333344444444550000023a0000000000007070"), 0x031461), - (hex!("01222222223333333344444444550000023a00000000000074a0"), 0x031501), - (hex!("01222222223333333344444444550000023b00000000000023c0"), 0x0315a1), - (hex!("01222222223333333344444444550000023b0000000000004730"), 0x031641), - (hex!("01222222223333333344444444550000023b00000000000068b0"), 0x0316e1), - (hex!("01222222223333333344444444550000023c00000000000023d0"), 0x031781), - (hex!("01222222223333333344444444550000023c0000000000004680"), 0x031821), - (hex!("01222222223333333344444444550000023d00000000000023e0"), 0x0318c1), - (hex!("01222222223333333344444444550000023d00000000000059a0"), 0x031961), - (hex!("01222222223333333344444444550000023e00000000000023f0"), 0x031a01), - (hex!("01222222223333333344444444550000023f0000000000002400"), 0x031aa1), - (hex!("0122222222333333334444444455000002400000000000002410"), 0x031b41), - (hex!("0122222222333333334444444455000002400000000000004920"), 0x031be1), - (hex!("01222222223333333344444444550000024000000000000066e0"), 0x031c81), - (hex!("01222222223333333344444444550000024000000000000076f0"), 0x031d21), - (hex!("01222222223333333344444444550000024000000000000078e0"), 0x031dc1), - (hex!("0122222222333333334444444455000002410000000000002420"), 0x031e61), - (hex!("0122222222333333334444444455000002420000000000002430"), 0x031f01), - (hex!("0122222222333333334444444455000002420000000000006590"), 0x031fa1), - (hex!("0122222222333333334444444455000002430000000000002440"), 0x032041), - (hex!("0122222222333333334444444455000002430000000000004d00"), 0x0320e1), - (hex!("0122222222333333334444444455000002440000000000002450"), 0x032181), - (hex!("0122222222333333334444444455000002440000000000005f80"), 0x032221), - (hex!("0122222222333333334444444455000002450000000000002460"), 0x0322c1), - (hex!("0122222222333333334444444455000002450000000000004940"), 0x032361), - (hex!("0122222222333333334444444455000002460000000000002470"), 0x032401), - (hex!("0122222222333333334444444455000002470000000000002480"), 0x0324a1), - (hex!("0122222222333333334444444455000002470000000000004dd0"), 0x032541), - (hex!("0122222222333333334444444455000002470000000000005930"), 0x0325e1), - (hex!("01222222223333333344444444550000024700000000000061b0"), 0x032681), - (hex!("0122222222333333334444444455000002470000000000007740"), 0x032721), - (hex!("0122222222333333334444444455000002480000000000002490"), 0x0327c1), - (hex!("0122222222333333334444444455000002480000000000004890"), 0x032861), - (hex!("01222222223333333344444444550000024900000000000024a0"), 0x032901), - (hex!("01222222223333333344444444550000024a00000000000024b0"), 0x0329a1), - (hex!("01222222223333333344444444550000024b00000000000024c0"), 0x032a41), - (hex!("01222222223333333344444444550000024c00000000000024d0"), 0x032ae1), - (hex!("01222222223333333344444444550000024d00000000000024e0"), 0x032b81), - (hex!("01222222223333333344444444550000024d0000000000004070"), 0x032c21), - (hex!("01222222223333333344444444550000024e00000000000024f0"), 0x032cc1), - (hex!("01222222223333333344444444550000024e00000000000066a0"), 0x032d61), - (hex!("01222222223333333344444444550000024e0000000000006ab0"), 0x032e01), - (hex!("01222222223333333344444444550000024f0000000000002500"), 0x032ea1), - (hex!("0122222222333333334444444455000002500000000000002510"), 0x032f41), - (hex!("0122222222333333334444444455000002510000000000002520"), 0x032fe1), - (hex!("0122222222333333334444444455000002510000000000007320"), 0x033081), - (hex!("0122222222333333334444444455000002520000000000002530"), 0x033121), - (hex!("0122222222333333334444444455000002520000000000006410"), 0x0331c1), - (hex!("0122222222333333334444444455000002530000000000002540"), 0x033261), - (hex!("0122222222333333334444444455000002530000000000005110"), 0x033301), - (hex!("0122222222333333334444444455000002540000000000002550"), 0x0333a1), - (hex!("01222222223333333344444444550000025400000000000040c0"), 0x033441), - (hex!("0122222222333333334444444455000002540000000000006a40"), 0x0334e1), - (hex!("0122222222333333334444444455000002550000000000002560"), 0x033581), - (hex!("0122222222333333334444444455000002550000000000005190"), 0x033621), - (hex!("0122222222333333334444444455000002560000000000002570"), 0x0336c1), - (hex!("01222222223333333344444444550000025600000000000061f0"), 0x033761), - (hex!("0122222222333333334444444455000002570000000000002580"), 0x033801), - (hex!("0122222222333333334444444455000002580000000000002590"), 0x0338a1), - (hex!("01222222223333333344444444550000025800000000000043d0"), 0x033941), - (hex!("01222222223333333344444444550000025900000000000025a0"), 0x0339e1), - (hex!("0122222222333333334444444455000002590000000000006bb0"), 0x033a81), - (hex!("01222222223333333344444444550000025a00000000000025b0"), 0x033b21), - (hex!("01222222223333333344444444550000025a0000000000005fb0"), 0x033bc1), - (hex!("01222222223333333344444444550000025a00000000000064c0"), 0x033c61), - (hex!("01222222223333333344444444550000025b00000000000025c0"), 0x033d01), - (hex!("01222222223333333344444444550000025b0000000000005c10"), 0x033da1), - (hex!("01222222223333333344444444550000025c00000000000025d0"), 0x033e41), - (hex!("01222222223333333344444444550000025c0000000000007d00"), 0x033ee1), - (hex!("01222222223333333344444444550000025d00000000000025e0"), 0x033f81), - (hex!("01222222223333333344444444550000025e00000000000025f0"), 0x034021), - (hex!("01222222223333333344444444550000025e00000000000045e0"), 0x0340c1), - (hex!("01222222223333333344444444550000025e0000000000006ee0"), 0x034161), - (hex!("01222222223333333344444444550000025f0000000000002600"), 0x034201), - (hex!("01222222223333333344444444550000025f00000000000050b0"), 0x0342a1), - (hex!("01222222223333333344444444550000025f0000000000007690"), 0x034341), - (hex!("0122222222333333334444444455000002600000000000002610"), 0x0343e1), - (hex!("0122222222333333334444444455000002600000000000007b60"), 0x034481), - (hex!("0122222222333333334444444455000002610000000000002620"), 0x034521), - (hex!("0122222222333333334444444455000002620000000000002630"), 0x0345c1), - (hex!("0122222222333333334444444455000002630000000000002640"), 0x034661), - (hex!("0122222222333333334444444455000002640000000000002650"), 0x034701), - (hex!("0122222222333333334444444455000002650000000000002660"), 0x0347a1), - (hex!("0122222222333333334444444455000002650000000000006180"), 0x034841), - (hex!("0122222222333333334444444455000002660000000000002670"), 0x0348e1), - (hex!("0122222222333333334444444455000002660000000000005430"), 0x034981), - (hex!("0122222222333333334444444455000002660000000000007a60"), 0x034a21), - (hex!("0122222222333333334444444455000002670000000000002680"), 0x034ac1), - (hex!("01222222223333333344444444550000026700000000000077f0"), 0x034b61), - (hex!("0122222222333333334444444455000002680000000000002690"), 0x034c01), - (hex!("01222222223333333344444444550000026900000000000026a0"), 0x034ca1), - (hex!("01222222223333333344444444550000026a00000000000026b0"), 0x034d41), - (hex!("01222222223333333344444444550000026a0000000000007530"), 0x034de1), - (hex!("01222222223333333344444444550000026b00000000000026c0"), 0x034e81), - (hex!("01222222223333333344444444550000026b00000000000058b0"), 0x034f21), - (hex!("01222222223333333344444444550000026b00000000000066b0"), 0x034fc1), - (hex!("01222222223333333344444444550000026b0000000000006b10"), 0x035061), - (hex!("01222222223333333344444444550000026c00000000000026d0"), 0x035101), - (hex!("01222222223333333344444444550000026d00000000000026e0"), 0x0351a1), - (hex!("01222222223333333344444444550000026d0000000000004210"), 0x035241), - (hex!("01222222223333333344444444550000026d0000000000005490"), 0x0352e1), - (hex!("01222222223333333344444444550000026d0000000000005e60"), 0x035381), - (hex!("01222222223333333344444444550000026d00000000000068e0"), 0x035421), - (hex!("01222222223333333344444444550000026d0000000000007020"), 0x0354c1), - (hex!("01222222223333333344444444550000026d0000000000007300"), 0x035561), - (hex!("01222222223333333344444444550000026e00000000000026f0"), 0x035601), - (hex!("01222222223333333344444444550000026f0000000000002700"), 0x0356a1), - (hex!("01222222223333333344444444550000026f0000000000004910"), 0x035741), - (hex!("0122222222333333334444444455000002700000000000002710"), 0x0357e1), - (hex!("0122222222333333334444444455000002710000000000002720"), 0x035881), - (hex!("01222222223333333344444444550000027100000000000050c0"), 0x035921), - (hex!("0122222222333333334444444455000002720000000000002730"), 0x0359c1), - (hex!("0122222222333333334444444455000002730000000000002740"), 0x035a61), - (hex!("0122222222333333334444444455000002740000000000002750"), 0x035b01), - (hex!("0122222222333333334444444455000002740000000000007490"), 0x035ba1), - (hex!("0122222222333333334444444455000002750000000000002760"), 0x035c41), - (hex!("0122222222333333334444444455000002760000000000002770"), 0x035ce1), - (hex!("0122222222333333334444444455000002760000000000004790"), 0x035d81), - (hex!("0122222222333333334444444455000002770000000000002780"), 0x035e21), - (hex!("01222222223333333344444444550000027700000000000050a0"), 0x035ec1), - (hex!("0122222222333333334444444455000002780000000000002790"), 0x035f61), - (hex!("0122222222333333334444444455000002780000000000004330"), 0x036001), - (hex!("0122222222333333334444444455000002780000000000006b00"), 0x0360a1), - (hex!("01222222223333333344444444550000027900000000000027a0"), 0x036141), - (hex!("01222222223333333344444444550000027a00000000000027b0"), 0x0361e1), - (hex!("01222222223333333344444444550000027b00000000000027c0"), 0x036281), - (hex!("01222222223333333344444444550000027b0000000000004930"), 0x036321), - (hex!("01222222223333333344444444550000027b0000000000006250"), 0x0363c1), - (hex!("01222222223333333344444444550000027c00000000000027d0"), 0x036461), - (hex!("01222222223333333344444444550000027d00000000000027e0"), 0x036501), - (hex!("01222222223333333344444444550000027d0000000000005ce0"), 0x0365a1), - (hex!("01222222223333333344444444550000027d0000000000005fe0"), 0x036641), - (hex!("01222222223333333344444444550000027e00000000000027f0"), 0x0366e1), - (hex!("01222222223333333344444444550000027f0000000000002800"), 0x036781), - (hex!("01222222223333333344444444550000027f0000000000003e90"), 0x036821), - (hex!("01222222223333333344444444550000027f0000000000007910"), 0x0368c1), - (hex!("0122222222333333334444444455000002800000000000002810"), 0x036961), - (hex!("0122222222333333334444444455000002800000000000004990"), 0x036a01), - (hex!("0122222222333333334444444455000002800000000000006160"), 0x036aa1), - (hex!("0122222222333333334444444455000002800000000000006740"), 0x036b41), - (hex!("0122222222333333334444444455000002810000000000002820"), 0x036be1), - (hex!("0122222222333333334444444455000002820000000000002830"), 0x036c81), - (hex!("0122222222333333334444444455000002820000000000005170"), 0x036d21), - (hex!("0122222222333333334444444455000002830000000000002840"), 0x036dc1), - (hex!("0122222222333333334444444455000002840000000000002850"), 0x036e61), - (hex!("0122222222333333334444444455000002840000000000004810"), 0x036f01), - (hex!("0122222222333333334444444455000002840000000000006aa0"), 0x036fa1), - (hex!("0122222222333333334444444455000002850000000000002860"), 0x037041), - (hex!("0122222222333333334444444455000002860000000000002870"), 0x0370e1), - (hex!("0122222222333333334444444455000002860000000000005080"), 0x037181), - (hex!("0122222222333333334444444455000002870000000000002880"), 0x037221), - (hex!("0122222222333333334444444455000002870000000000004e60"), 0x0372c1), - (hex!("0122222222333333334444444455000002880000000000002890"), 0x037361), - (hex!("0122222222333333334444444455000002880000000000005060"), 0x037401), - (hex!("0122222222333333334444444455000002880000000000006f20"), 0x0374a1), - (hex!("01222222223333333344444444550000028900000000000028a0"), 0x037541), - (hex!("01222222223333333344444444550000028900000000000047e0"), 0x0375e1), - (hex!("01222222223333333344444444550000028a00000000000028b0"), 0x037681), - (hex!("01222222223333333344444444550000028a0000000000005ab0"), 0x037721), - (hex!("01222222223333333344444444550000028a0000000000007130"), 0x0377c1), - (hex!("01222222223333333344444444550000028a0000000000007660"), 0x037861), - (hex!("01222222223333333344444444550000028b00000000000028c0"), 0x037901), - (hex!("01222222223333333344444444550000028b00000000000054e0"), 0x0379a1), - (hex!("01222222223333333344444444550000028c00000000000028d0"), 0x037a41), - (hex!("01222222223333333344444444550000028c00000000000046f0"), 0x037ae1), - (hex!("01222222223333333344444444550000028c00000000000061a0"), 0x037b81), - (hex!("01222222223333333344444444550000028d00000000000028e0"), 0x037c21), - (hex!("01222222223333333344444444550000028e00000000000028f0"), 0x037cc1), - (hex!("01222222223333333344444444550000028e0000000000004130"), 0x037d61), - (hex!("01222222223333333344444444550000028f0000000000002900"), 0x037e01), - (hex!("01222222223333333344444444550000028f0000000000007510"), 0x037ea1), - (hex!("0122222222333333334444444455000002900000000000002910"), 0x037f41), - (hex!("0122222222333333334444444455000002900000000000004a40"), 0x037fe1), - (hex!("0122222222333333334444444455000002910000000000002920"), 0x038081), - (hex!("0122222222333333334444444455000002920000000000002930"), 0x038121), - (hex!("0122222222333333334444444455000002920000000000004e90"), 0x0381c1), - (hex!("0122222222333333334444444455000002930000000000002940"), 0x038261), - (hex!("0122222222333333334444444455000002930000000000006880"), 0x038301), - (hex!("0122222222333333334444444455000002940000000000002950"), 0x0383a1), - (hex!("0122222222333333334444444455000002940000000000007bc0"), 0x038441), - (hex!("0122222222333333334444444455000002950000000000002960"), 0x0384e1), - (hex!("0122222222333333334444444455000002960000000000002970"), 0x038581), - (hex!("01222222223333333344444444550000029600000000000059d0"), 0x038621), - (hex!("0122222222333333334444444455000002970000000000002980"), 0x0386c1), - (hex!("0122222222333333334444444455000002970000000000004a50"), 0x038761), - (hex!("0122222222333333334444444455000002970000000000005f20"), 0x038801), - (hex!("01222222223333333344444444550000029700000000000068d0"), 0x0388a1), - (hex!("0122222222333333334444444455000002980000000000002990"), 0x038941), - (hex!("0122222222333333334444444455000002980000000000004370"), 0x0389e1), - (hex!("0122222222333333334444444455000002980000000000004420"), 0x038a81), - (hex!("01222222223333333344444444550000029900000000000029a0"), 0x038b21), - (hex!("01222222223333333344444444550000029a00000000000029b0"), 0x038bc1), - (hex!("01222222223333333344444444550000029a0000000000006010"), 0x038c61), - (hex!("01222222223333333344444444550000029a0000000000006980"), 0x038d01), - (hex!("01222222223333333344444444550000029b00000000000029c0"), 0x038da1), - (hex!("01222222223333333344444444550000029c00000000000029d0"), 0x038e41), - (hex!("01222222223333333344444444550000029c0000000000007480"), 0x038ee1), - (hex!("01222222223333333344444444550000029d00000000000029e0"), 0x038f81), - (hex!("01222222223333333344444444550000029d0000000000005030"), 0x039021), - (hex!("01222222223333333344444444550000029d0000000000007780"), 0x0390c1), - (hex!("01222222223333333344444444550000029d0000000000007a50"), 0x039161), - (hex!("01222222223333333344444444550000029e00000000000029f0"), 0x039201), - (hex!("01222222223333333344444444550000029e00000000000074b0"), 0x0392a1), - (hex!("01222222223333333344444444550000029f0000000000002a00"), 0x039341), - (hex!("0122222222333333334444444455000002a00000000000002a10"), 0x0393e1), - (hex!("0122222222333333334444444455000002a10000000000002a20"), 0x039481), - (hex!("0122222222333333334444444455000002a20000000000002a30"), 0x039521), - (hex!("0122222222333333334444444455000002a20000000000004c50"), 0x0395c1), - (hex!("0122222222333333334444444455000002a20000000000006f10"), 0x039661), - (hex!("0122222222333333334444444455000002a30000000000002a40"), 0x039701), - (hex!("0122222222333333334444444455000002a40000000000002a50"), 0x0397a1), - (hex!("0122222222333333334444444455000002a40000000000005d60"), 0x039841), - (hex!("0122222222333333334444444455000002a50000000000002a60"), 0x0398e1), - (hex!("0122222222333333334444444455000002a50000000000005440"), 0x039981), - (hex!("0122222222333333334444444455000002a50000000000005890"), 0x039a21), - (hex!("0122222222333333334444444455000002a60000000000002a70"), 0x039ac1), - (hex!("0122222222333333334444444455000002a70000000000002a80"), 0x039b61), - (hex!("0122222222333333334444444455000002a700000000000054a0"), 0x039c01), - (hex!("0122222222333333334444444455000002a70000000000007280"), 0x039ca1), - (hex!("0122222222333333334444444455000002a80000000000002a90"), 0x039d41), - (hex!("0122222222333333334444444455000002a90000000000002aa0"), 0x039de1), - (hex!("0122222222333333334444444455000002aa0000000000002ab0"), 0x039e81), - (hex!("0122222222333333334444444455000002ab0000000000002ac0"), 0x039f21), - (hex!("0122222222333333334444444455000002ab0000000000006c90"), 0x039fc1), - (hex!("0122222222333333334444444455000002ac0000000000002ad0"), 0x03a061), - (hex!("0122222222333333334444444455000002ac0000000000006db0"), 0x03a101), - (hex!("0122222222333333334444444455000002ad0000000000002ae0"), 0x03a1a1), - (hex!("0122222222333333334444444455000002ad00000000000065e0"), 0x03a241), - (hex!("0122222222333333334444444455000002ad0000000000007b40"), 0x03a2e1), - (hex!("0122222222333333334444444455000002ae0000000000002af0"), 0x03a381), - (hex!("0122222222333333334444444455000002ae0000000000004d20"), 0x03a421), - (hex!("0122222222333333334444444455000002ae0000000000006f30"), 0x03a4c1), - (hex!("0122222222333333334444444455000002af0000000000002b00"), 0x03a561), - (hex!("0122222222333333334444444455000002b00000000000002b10"), 0x03a601), - (hex!("0122222222333333334444444455000002b00000000000004560"), 0x03a6a1), - (hex!("0122222222333333334444444455000002b00000000000005800"), 0x03a741), - (hex!("0122222222333333334444444455000002b00000000000005a60"), 0x03a7e1), - (hex!("0122222222333333334444444455000002b10000000000002b20"), 0x03a881), - (hex!("0122222222333333334444444455000002b10000000000007b30"), 0x03a921), - (hex!("0122222222333333334444444455000002b20000000000002b30"), 0x03a9c1), - (hex!("0122222222333333334444444455000002b20000000000004440"), 0x03aa61), - (hex!("0122222222333333334444444455000002b20000000000004f80"), 0x03ab01), - (hex!("0122222222333333334444444455000002b20000000000005020"), 0x03aba1), - (hex!("0122222222333333334444444455000002b30000000000002b40"), 0x03ac41), - (hex!("0122222222333333334444444455000002b40000000000002b50"), 0x03ace1), - (hex!("0122222222333333334444444455000002b50000000000002b60"), 0x03ad81), - (hex!("0122222222333333334444444455000002b500000000000059e0"), 0x03ae21), - (hex!("0122222222333333334444444455000002b60000000000002b70"), 0x03aec1), - (hex!("0122222222333333334444444455000002b70000000000002b80"), 0x03af61), - (hex!("0122222222333333334444444455000002b80000000000002b90"), 0x03b001), - (hex!("0122222222333333334444444455000002b80000000000004590"), 0x03b0a1), - (hex!("0122222222333333334444444455000002b800000000000047d0"), 0x03b141), - (hex!("0122222222333333334444444455000002b80000000000006030"), 0x03b1e1), - (hex!("0122222222333333334444444455000002b80000000000006a20"), 0x03b281), - (hex!("0122222222333333334444444455000002b80000000000006a90"), 0x03b321), - (hex!("0122222222333333334444444455000002b90000000000002ba0"), 0x03b3c1), - (hex!("0122222222333333334444444455000002ba0000000000002bb0"), 0x03b461), - (hex!("0122222222333333334444444455000002ba0000000000006e80"), 0x03b501), - (hex!("0122222222333333334444444455000002bb0000000000002bc0"), 0x03b5a1), - (hex!("0122222222333333334444444455000002bc0000000000002bd0"), 0x03b641), - (hex!("0122222222333333334444444455000002bc0000000000004b30"), 0x03b6e1), - (hex!("0122222222333333334444444455000002bd0000000000002be0"), 0x03b781), - (hex!("0122222222333333334444444455000002bd0000000000005e10"), 0x03b821), - (hex!("0122222222333333334444444455000002be0000000000002bf0"), 0x03b8c1), - (hex!("0122222222333333334444444455000002bf0000000000002c00"), 0x03b961), - (hex!("0122222222333333334444444455000002c00000000000002c10"), 0x03ba01), - (hex!("0122222222333333334444444455000002c10000000000002c20"), 0x03baa1), - (hex!("0122222222333333334444444455000002c10000000000003ef0"), 0x03bb41), - (hex!("0122222222333333334444444455000002c20000000000002c30"), 0x03bbe1), - (hex!("0122222222333333334444444455000002c200000000000056e0"), 0x03bc81), - (hex!("0122222222333333334444444455000002c30000000000002c40"), 0x03bd21), - (hex!("0122222222333333334444444455000002c30000000000004b60"), 0x03bdc1), - (hex!("0122222222333333334444444455000002c40000000000002c50"), 0x03be61), - (hex!("0122222222333333334444444455000002c400000000000045f0"), 0x03bf01), - (hex!("0122222222333333334444444455000002c40000000000005290"), 0x03bfa1), - (hex!("0122222222333333334444444455000002c50000000000002c60"), 0x03c041), - (hex!("0122222222333333334444444455000002c60000000000002c70"), 0x03c0e1), - (hex!("0122222222333333334444444455000002c60000000000006ae0"), 0x03c181), - (hex!("0122222222333333334444444455000002c70000000000002c80"), 0x03c221), - (hex!("0122222222333333334444444455000002c70000000000005680"), 0x03c2c1), - (hex!("0122222222333333334444444455000002c70000000000006e10"), 0x03c361), - (hex!("0122222222333333334444444455000002c80000000000002c90"), 0x03c401), - (hex!("0122222222333333334444444455000002c90000000000002ca0"), 0x03c4a1), - (hex!("0122222222333333334444444455000002ca0000000000002cb0"), 0x03c541), - (hex!("0122222222333333334444444455000002cb0000000000002cc0"), 0x03c5e1), - (hex!("0122222222333333334444444455000002cc0000000000002cd0"), 0x03c681), - (hex!("0122222222333333334444444455000002cc0000000000005b50"), 0x03c721), - (hex!("0122222222333333334444444455000002cd0000000000002ce0"), 0x03c7c1), - (hex!("0122222222333333334444444455000002ce0000000000002cf0"), 0x03c861), - (hex!("0122222222333333334444444455000002ce00000000000043f0"), 0x03c901), - (hex!("0122222222333333334444444455000002ce0000000000006420"), 0x03c9a1), - (hex!("0122222222333333334444444455000002cf0000000000002d00"), 0x03ca41), - (hex!("0122222222333333334444444455000002d00000000000002d10"), 0x03cae1), - (hex!("0122222222333333334444444455000002d10000000000002d20"), 0x03cb81), - (hex!("0122222222333333334444444455000002d10000000000005370"), 0x03cc21), - (hex!("0122222222333333334444444455000002d20000000000002d30"), 0x03ccc1), - (hex!("0122222222333333334444444455000002d20000000000005ef0"), 0x03cd61), - (hex!("0122222222333333334444444455000002d20000000000006570"), 0x03ce01), - (hex!("0122222222333333334444444455000002d30000000000002d40"), 0x03cea1), - (hex!("0122222222333333334444444455000002d30000000000007360"), 0x03cf41), - (hex!("0122222222333333334444444455000002d40000000000002d50"), 0x03cfe1), - (hex!("0122222222333333334444444455000002d400000000000079a0"), 0x03d081), - (hex!("0122222222333333334444444455000002d50000000000002d60"), 0x03d121), - (hex!("0122222222333333334444444455000002d50000000000004250"), 0x03d1c1), - (hex!("0122222222333333334444444455000002d50000000000006050"), 0x03d261), - (hex!("0122222222333333334444444455000002d60000000000002d70"), 0x03d301), - (hex!("0122222222333333334444444455000002d60000000000007080"), 0x03d3a1), - (hex!("0122222222333333334444444455000002d70000000000002d80"), 0x03d441), - (hex!("0122222222333333334444444455000002d80000000000002d90"), 0x03d4e1), - (hex!("0122222222333333334444444455000002d80000000000007110"), 0x03d581), - (hex!("0122222222333333334444444455000002d800000000000073c0"), 0x03d621), - (hex!("0122222222333333334444444455000002d800000000000075a0"), 0x03d6c1), - (hex!("0122222222333333334444444455000002d90000000000002da0"), 0x03d761), - (hex!("0122222222333333334444444455000002d90000000000004860"), 0x03d801), - (hex!("0122222222333333334444444455000002d90000000000006b60"), 0x03d8a1), - (hex!("0122222222333333334444444455000002da0000000000002db0"), 0x03d941), - (hex!("0122222222333333334444444455000002da0000000000006630"), 0x03d9e1), - (hex!("0122222222333333334444444455000002db0000000000002dc0"), 0x03da81), - (hex!("0122222222333333334444444455000002dc0000000000002dd0"), 0x03db21), - (hex!("0122222222333333334444444455000002dc0000000000004830"), 0x03dbc1), - (hex!("0122222222333333334444444455000002dd0000000000002de0"), 0x03dc61), - (hex!("0122222222333333334444444455000002de0000000000002df0"), 0x03dd01), - (hex!("0122222222333333334444444455000002de0000000000004f00"), 0x03dda1), - (hex!("0122222222333333334444444455000002df0000000000002e00"), 0x03de41), - (hex!("0122222222333333334444444455000002e00000000000002e10"), 0x03dee1), - (hex!("0122222222333333334444444455000002e10000000000002e20"), 0x03df81), - (hex!("0122222222333333334444444455000002e10000000000006e90"), 0x03e021), - (hex!("0122222222333333334444444455000002e20000000000002e30"), 0x03e0c1), - (hex!("0122222222333333334444444455000002e200000000000053e0"), 0x03e161), - (hex!("0122222222333333334444444455000002e30000000000002e40"), 0x03e201), - (hex!("0122222222333333334444444455000002e30000000000006020"), 0x03e2a1), - (hex!("0122222222333333334444444455000002e30000000000006540"), 0x03e341), - (hex!("0122222222333333334444444455000002e40000000000002e50"), 0x03e3e1), - (hex!("0122222222333333334444444455000002e50000000000002e60"), 0x03e481), - (hex!("0122222222333333334444444455000002e50000000000005180"), 0x03e521), - (hex!("0122222222333333334444444455000002e50000000000007bf0"), 0x03e5c1), - (hex!("0122222222333333334444444455000002e60000000000002e70"), 0x03e661), - (hex!("0122222222333333334444444455000002e60000000000005350"), 0x03e701), - (hex!("0122222222333333334444444455000002e60000000000007960"), 0x03e7a1), - (hex!("0122222222333333334444444455000002e70000000000002e80"), 0x03e841), - (hex!("0122222222333333334444444455000002e80000000000002e90"), 0x03e8e1), - (hex!("0122222222333333334444444455000002e90000000000002ea0"), 0x03e981), - (hex!("0122222222333333334444444455000002ea0000000000002eb0"), 0x03ea21), - (hex!("0122222222333333334444444455000002eb0000000000002ec0"), 0x03eac1), - (hex!("0122222222333333334444444455000002ec0000000000002ed0"), 0x03eb61), - (hex!("0122222222333333334444444455000002ec0000000000006c10"), 0x03ec01), - (hex!("0122222222333333334444444455000002ed0000000000002ee0"), 0x03eca1), - (hex!("0122222222333333334444444455000002ed0000000000005590"), 0x03ed41), - (hex!("0122222222333333334444444455000002ed0000000000005cd0"), 0x03ede1), - (hex!("0122222222333333334444444455000002ed0000000000006910"), 0x03ee81), - (hex!("0122222222333333334444444455000002ee0000000000002ef0"), 0x03ef21), - (hex!("0122222222333333334444444455000002ef0000000000002f00"), 0x03efc1), - (hex!("0122222222333333334444444455000002ef0000000000004ed0"), 0x03f061), - (hex!("0122222222333333334444444455000002f00000000000002f10"), 0x03f101), - (hex!("0122222222333333334444444455000002f00000000000004cf0"), 0x03f1a1), - (hex!("0122222222333333334444444455000002f00000000000005d10"), 0x03f241), - (hex!("0122222222333333334444444455000002f00000000000006860"), 0x03f2e1), - (hex!("0122222222333333334444444455000002f00000000000006b50"), 0x03f381), - (hex!("0122222222333333334444444455000002f00000000000007100"), 0x03f421), - (hex!("0122222222333333334444444455000002f00000000000007aa0"), 0x03f4c1), - (hex!("0122222222333333334444444455000002f10000000000002f20"), 0x03f561), - (hex!("0122222222333333334444444455000002f20000000000002f30"), 0x03f601), - (hex!("0122222222333333334444444455000002f200000000000044b0"), 0x03f6a1), - (hex!("0122222222333333334444444455000002f30000000000002f40"), 0x03f741), - (hex!("0122222222333333334444444455000002f300000000000075b0"), 0x03f7e1), - (hex!("0122222222333333334444444455000002f40000000000002f50"), 0x03f881), - (hex!("0122222222333333334444444455000002f400000000000060f0"), 0x03f921), - (hex!("0122222222333333334444444455000002f50000000000002f60"), 0x03f9c1), - (hex!("0122222222333333334444444455000002f50000000000007210"), 0x03fa61), - (hex!("0122222222333333334444444455000002f60000000000002f70"), 0x03fb01), - (hex!("0122222222333333334444444455000002f60000000000006610"), 0x03fba1), - (hex!("0122222222333333334444444455000002f70000000000002f80"), 0x03fc41), - (hex!("0122222222333333334444444455000002f70000000000007560"), 0x03fce1), - (hex!("0122222222333333334444444455000002f80000000000002f90"), 0x03fd81), - (hex!("0122222222333333334444444455000002f80000000000006320"), 0x03fe21), - (hex!("0122222222333333334444444455000002f90000000000002fa0"), 0x03fec1), - (hex!("0122222222333333334444444455000002f90000000000006e50"), 0x03ff61), - (hex!("0122222222333333334444444455000002fa0000000000002fb0"), 0x040001), - (hex!("0122222222333333334444444455000002fb0000000000002fc0"), 0x0400a1), - (hex!("0122222222333333334444444455000002fb0000000000004780"), 0x040141), - (hex!("0122222222333333334444444455000002fc0000000000002fd0"), 0x0401e1), - (hex!("0122222222333333334444444455000002fd0000000000002fe0"), 0x040281), - (hex!("0122222222333333334444444455000002fd0000000000005600"), 0x040321), - (hex!("0122222222333333334444444455000002fd0000000000006c00"), 0x0403c1), - (hex!("0122222222333333334444444455000002fe0000000000002ff0"), 0x040461), - (hex!("0122222222333333334444444455000002ff0000000000003000"), 0x040501), - (hex!("0122222222333333334444444455000003000000000000003010"), 0x0405a1), - (hex!("0122222222333333334444444455000003000000000000004080"), 0x040641), - (hex!("0122222222333333334444444455000003010000000000003020"), 0x0406e1), - (hex!("0122222222333333334444444455000003010000000000006340"), 0x040781), - (hex!("0122222222333333334444444455000003020000000000003030"), 0x040821), - (hex!("0122222222333333334444444455000003020000000000005b00"), 0x0408c1), - (hex!("0122222222333333334444444455000003020000000000007b20"), 0x040961), - (hex!("0122222222333333334444444455000003030000000000003040"), 0x040a01), - (hex!("01222222223333333344444444550000030300000000000056b0"), 0x040aa1), - (hex!("0122222222333333334444444455000003030000000000006280"), 0x040b41), - (hex!("0122222222333333334444444455000003030000000000007ad0"), 0x040be1), - (hex!("0122222222333333334444444455000003040000000000003050"), 0x040c81), - (hex!("0122222222333333334444444455000003040000000000005c50"), 0x040d21), - (hex!("0122222222333333334444444455000003050000000000003060"), 0x040dc1), - (hex!("01222222223333333344444444550000030500000000000072e0"), 0x040e61), - (hex!("0122222222333333334444444455000003060000000000003070"), 0x040f01), - (hex!("0122222222333333334444444455000003060000000000004360"), 0x040fa1), - (hex!("0122222222333333334444444455000003060000000000004380"), 0x041041), - (hex!("0122222222333333334444444455000003060000000000004820"), 0x0410e1), - (hex!("0122222222333333334444444455000003060000000000006d10"), 0x041181), - (hex!("0122222222333333334444444455000003070000000000003080"), 0x041221), - (hex!("0122222222333333334444444455000003070000000000004450"), 0x0412c1), - (hex!("0122222222333333334444444455000003080000000000003090"), 0x041361), - (hex!("0122222222333333334444444455000003080000000000005ad0"), 0x041401), - (hex!("01222222223333333344444444550000030900000000000030a0"), 0x0414a1), - (hex!("01222222223333333344444444550000030a00000000000030b0"), 0x041541), - (hex!("01222222223333333344444444550000030a0000000000007760"), 0x0415e1), - (hex!("01222222223333333344444444550000030b00000000000030c0"), 0x041681), - (hex!("01222222223333333344444444550000030b0000000000007a80"), 0x041721), - (hex!("01222222223333333344444444550000030c00000000000030d0"), 0x0417c1), - (hex!("01222222223333333344444444550000030d00000000000030e0"), 0x041861), - (hex!("01222222223333333344444444550000030d0000000000003eb0"), 0x041901), - (hex!("01222222223333333344444444550000030e00000000000030f0"), 0x0419a1), - (hex!("01222222223333333344444444550000030f0000000000003100"), 0x041a41), - (hex!("01222222223333333344444444550000030f0000000000004690"), 0x041ae1), - (hex!("01222222223333333344444444550000030f0000000000006900"), 0x041b81), - (hex!("0122222222333333334444444455000003100000000000003110"), 0x041c21), - (hex!("01222222223333333344444444550000031000000000000058a0"), 0x041cc1), - (hex!("0122222222333333334444444455000003110000000000003120"), 0x041d61), - (hex!("0122222222333333334444444455000003110000000000004200"), 0x041e01), - (hex!("0122222222333333334444444455000003120000000000003130"), 0x041ea1), - (hex!("0122222222333333334444444455000003130000000000003140"), 0x041f41), - (hex!("0122222222333333334444444455000003130000000000004d50"), 0x041fe1), - (hex!("0122222222333333334444444455000003130000000000005400"), 0x042081), - (hex!("0122222222333333334444444455000003130000000000005520"), 0x042121), - (hex!("0122222222333333334444444455000003140000000000003150"), 0x0421c1), - (hex!("0122222222333333334444444455000003140000000000006450"), 0x042261), - (hex!("0122222222333333334444444455000003150000000000003160"), 0x042301), - (hex!("01222222223333333344444444550000031500000000000062d0"), 0x0423a1), - (hex!("0122222222333333334444444455000003160000000000003170"), 0x042441), - (hex!("0122222222333333334444444455000003160000000000004c40"), 0x0424e1), - (hex!("0122222222333333334444444455000003160000000000007c80"), 0x042581), - (hex!("0122222222333333334444444455000003170000000000003180"), 0x042621), - (hex!("0122222222333333334444444455000003170000000000004400"), 0x0426c1), - (hex!("0122222222333333334444444455000003170000000000005090"), 0x042761), - (hex!("0122222222333333334444444455000003170000000000006cb0"), 0x042801), - (hex!("0122222222333333334444444455000003180000000000003190"), 0x0428a1), - (hex!("0122222222333333334444444455000003180000000000006560"), 0x042941), - (hex!("01222222223333333344444444550000031900000000000031a0"), 0x0429e1), - (hex!("01222222223333333344444444550000031900000000000052d0"), 0x042a81), - (hex!("01222222223333333344444444550000031900000000000057e0"), 0x042b21), - (hex!("01222222223333333344444444550000031a00000000000031b0"), 0x042bc1), - (hex!("01222222223333333344444444550000031a00000000000071e0"), 0x042c61), - (hex!("01222222223333333344444444550000031b00000000000031c0"), 0x042d01), - (hex!("01222222223333333344444444550000031c00000000000031d0"), 0x042da1), - (hex!("01222222223333333344444444550000031c0000000000004480"), 0x042e41), - (hex!("01222222223333333344444444550000031c0000000000005790"), 0x042ee1), - (hex!("01222222223333333344444444550000031c0000000000007be0"), 0x042f81), - (hex!("01222222223333333344444444550000031d00000000000031e0"), 0x043021), - (hex!("01222222223333333344444444550000031d0000000000005560"), 0x0430c1), - (hex!("01222222223333333344444444550000031e00000000000031f0"), 0x043161), - (hex!("01222222223333333344444444550000031f0000000000003200"), 0x043201), - (hex!("01222222223333333344444444550000031f0000000000004190"), 0x0432a1), - (hex!("0122222222333333334444444455000003200000000000003210"), 0x043341), - (hex!("0122222222333333334444444455000003210000000000003220"), 0x0433e1), - (hex!("0122222222333333334444444455000003220000000000003230"), 0x043481), - (hex!("0122222222333333334444444455000003230000000000003240"), 0x043521), - (hex!("01222222223333333344444444550000032300000000000069d0"), 0x0435c1), - (hex!("0122222222333333334444444455000003240000000000003250"), 0x043661), - (hex!("0122222222333333334444444455000003250000000000003260"), 0x043701), - (hex!("01222222223333333344444444550000032500000000000042b0"), 0x0437a1), - (hex!("01222222223333333344444444550000032500000000000064e0"), 0x043841), - (hex!("0122222222333333334444444455000003260000000000003270"), 0x0438e1), - (hex!("0122222222333333334444444455000003270000000000003280"), 0x043981), - (hex!("0122222222333333334444444455000003270000000000005b20"), 0x043a21), - (hex!("0122222222333333334444444455000003270000000000006330"), 0x043ac1), - (hex!("0122222222333333334444444455000003270000000000006810"), 0x043b61), - (hex!("0122222222333333334444444455000003280000000000003290"), 0x043c01), - (hex!("01222222223333333344444444550000032900000000000032a0"), 0x043ca1), - (hex!("01222222223333333344444444550000032900000000000056f0"), 0x043d41), - (hex!("0122222222333333334444444455000003290000000000005e20"), 0x043de1), - (hex!("0122222222333333334444444455000003290000000000005e70"), 0x043e81), - (hex!("01222222223333333344444444550000032a00000000000032b0"), 0x043f21), - (hex!("01222222223333333344444444550000032b00000000000032c0"), 0x043fc1), - (hex!("01222222223333333344444444550000032b0000000000005500"), 0x044061), - (hex!("01222222223333333344444444550000032b0000000000005a20"), 0x044101), - (hex!("01222222223333333344444444550000032c00000000000032d0"), 0x0441a1), - (hex!("01222222223333333344444444550000032c0000000000004060"), 0x044241), - (hex!("01222222223333333344444444550000032c0000000000004760"), 0x0442e1), - (hex!("01222222223333333344444444550000032d00000000000032e0"), 0x044381), - (hex!("01222222223333333344444444550000032d00000000000068a0"), 0x044421), - (hex!("01222222223333333344444444550000032e00000000000032f0"), 0x0444c1), - (hex!("01222222223333333344444444550000032f0000000000003300"), 0x044561), - (hex!("0122222222333333334444444455000003300000000000003310"), 0x044601), - (hex!("0122222222333333334444444455000003300000000000006e40"), 0x0446a1), - (hex!("0122222222333333334444444455000003310000000000003320"), 0x044741), - (hex!("0122222222333333334444444455000003310000000000004620"), 0x0447e1), - (hex!("0122222222333333334444444455000003320000000000003330"), 0x044881), - (hex!("0122222222333333334444444455000003330000000000003340"), 0x044921), - (hex!("0122222222333333334444444455000003330000000000004b80"), 0x0449c1), - (hex!("0122222222333333334444444455000003340000000000003350"), 0x044a61), - (hex!("0122222222333333334444444455000003350000000000003360"), 0x044b01), - (hex!("0122222222333333334444444455000003360000000000003370"), 0x044ba1), - (hex!("0122222222333333334444444455000003370000000000003380"), 0x044c41), - (hex!("0122222222333333334444444455000003380000000000003390"), 0x044ce1), - (hex!("01222222223333333344444444550000033900000000000033a0"), 0x044d81), - (hex!("0122222222333333334444444455000003390000000000006b90"), 0x044e21), - (hex!("01222222223333333344444444550000033a00000000000033b0"), 0x044ec1), - (hex!("01222222223333333344444444550000033a0000000000007420"), 0x044f61), - (hex!("01222222223333333344444444550000033b00000000000033c0"), 0x045001), - (hex!("01222222223333333344444444550000033b0000000000007620"), 0x0450a1), - (hex!("01222222223333333344444444550000033c00000000000033d0"), 0x045141), - (hex!("01222222223333333344444444550000033c0000000000006b30"), 0x0451e1), - (hex!("01222222223333333344444444550000033d00000000000033e0"), 0x045281), - (hex!("01222222223333333344444444550000033e00000000000033f0"), 0x045321), - (hex!("01222222223333333344444444550000033e00000000000048b0"), 0x0453c1), - (hex!("01222222223333333344444444550000033e0000000000004e70"), 0x045461), - (hex!("01222222223333333344444444550000033f0000000000003400"), 0x045501), - (hex!("01222222223333333344444444550000033f0000000000006380"), 0x0455a1), - (hex!("0122222222333333334444444455000003400000000000003410"), 0x045641), - (hex!("0122222222333333334444444455000003410000000000003420"), 0x0456e1), - (hex!("0122222222333333334444444455000003410000000000006090"), 0x045781), - (hex!("0122222222333333334444444455000003420000000000003430"), 0x045821), - (hex!("01222222223333333344444444550000034200000000000073d0"), 0x0458c1), - (hex!("0122222222333333334444444455000003430000000000003440"), 0x045961), - (hex!("0122222222333333334444444455000003430000000000006370"), 0x045a01), - (hex!("01222222223333333344444444550000034300000000000075c0"), 0x045aa1), - (hex!("0122222222333333334444444455000003440000000000003450"), 0x045b41), - (hex!("0122222222333333334444444455000003450000000000003460"), 0x045be1), - (hex!("0122222222333333334444444455000003460000000000003470"), 0x045c81), - (hex!("01222222223333333344444444550000034600000000000055f0"), 0x045d21), - (hex!("0122222222333333334444444455000003470000000000003480"), 0x045dc1), - (hex!("0122222222333333334444444455000003470000000000003fe0"), 0x045e61), - (hex!("0122222222333333334444444455000003480000000000003490"), 0x045f01), - (hex!("0122222222333333334444444455000003480000000000007990"), 0x045fa1), - (hex!("01222222223333333344444444550000034900000000000034a0"), 0x046041), - (hex!("0122222222333333334444444455000003490000000000004410"), 0x0460e1), - (hex!("01222222223333333344444444550000034a00000000000034b0"), 0x046181), - (hex!("01222222223333333344444444550000034a00000000000062a0"), 0x046221), - (hex!("01222222223333333344444444550000034a0000000000007260"), 0x0462c1), - (hex!("01222222223333333344444444550000034b00000000000034c0"), 0x046361), - (hex!("01222222223333333344444444550000034b0000000000005760"), 0x046401), - (hex!("01222222223333333344444444550000034b0000000000006200"), 0x0464a1), - (hex!("01222222223333333344444444550000034c00000000000034d0"), 0x046541), - (hex!("01222222223333333344444444550000034d00000000000034e0"), 0x0465e1), - (hex!("01222222223333333344444444550000034e00000000000034f0"), 0x046681), - (hex!("01222222223333333344444444550000034e0000000000007790"), 0x046721), - (hex!("01222222223333333344444444550000034f0000000000003500"), 0x0467c1), - (hex!("0122222222333333334444444455000003500000000000003510"), 0x046861), - (hex!("0122222222333333334444444455000003510000000000003520"), 0x046901), - (hex!("0122222222333333334444444455000003520000000000003530"), 0x0469a1), - (hex!("01222222223333333344444444550000035200000000000056a0"), 0x046a41), - (hex!("0122222222333333334444444455000003530000000000003540"), 0x046ae1), - (hex!("0122222222333333334444444455000003540000000000003550"), 0x046b81), - (hex!("01222222223333333344444444550000035400000000000047b0"), 0x046c21), - (hex!("0122222222333333334444444455000003550000000000003560"), 0x046cc1), - (hex!("0122222222333333334444444455000003550000000000004500"), 0x046d61), - (hex!("0122222222333333334444444455000003560000000000003570"), 0x046e01), - (hex!("0122222222333333334444444455000003560000000000004fc0"), 0x046ea1), - (hex!("0122222222333333334444444455000003560000000000007160"), 0x046f41), - (hex!("0122222222333333334444444455000003560000000000007400"), 0x046fe1), - (hex!("0122222222333333334444444455000003570000000000003580"), 0x047081), - (hex!("0122222222333333334444444455000003580000000000003590"), 0x047121), - (hex!("0122222222333333334444444455000003580000000000005a80"), 0x0471c1), - (hex!("01222222223333333344444444550000035900000000000035a0"), 0x047261), - (hex!("01222222223333333344444444550000035900000000000073b0"), 0x047301), - (hex!("01222222223333333344444444550000035a00000000000035b0"), 0x0473a1), - (hex!("01222222223333333344444444550000035a0000000000004c20"), 0x047441), - (hex!("01222222223333333344444444550000035b00000000000035c0"), 0x0474e1), - (hex!("01222222223333333344444444550000035b0000000000005120"), 0x047581), - (hex!("01222222223333333344444444550000035c00000000000035d0"), 0x047621), - (hex!("01222222223333333344444444550000035c0000000000004300"), 0x0476c1), - (hex!("01222222223333333344444444550000035c0000000000005a40"), 0x047761), - (hex!("01222222223333333344444444550000035c0000000000006620"), 0x047801), - (hex!("01222222223333333344444444550000035c0000000000006ed0"), 0x0478a1), - (hex!("01222222223333333344444444550000035d00000000000035e0"), 0x047941), - (hex!("01222222223333333344444444550000035d0000000000005df0"), 0x0479e1), - (hex!("01222222223333333344444444550000035e00000000000035f0"), 0x047a81), - (hex!("01222222223333333344444444550000035f0000000000003600"), 0x047b21), - (hex!("01222222223333333344444444550000035f00000000000058d0"), 0x047bc1), - (hex!("0122222222333333334444444455000003600000000000003610"), 0x047c61), - (hex!("0122222222333333334444444455000003600000000000007b90"), 0x047d01), - (hex!("0122222222333333334444444455000003610000000000003620"), 0x047da1), - (hex!("0122222222333333334444444455000003610000000000006ad0"), 0x047e41), - (hex!("0122222222333333334444444455000003620000000000003630"), 0x047ee1), - (hex!("01222222223333333344444444550000036200000000000063a0"), 0x047f81), - (hex!("0122222222333333334444444455000003630000000000003640"), 0x048021), - (hex!("0122222222333333334444444455000003630000000000007250"), 0x0480c1), - (hex!("0122222222333333334444444455000003640000000000003650"), 0x048161), - (hex!("0122222222333333334444444455000003640000000000005510"), 0x048201), - (hex!("0122222222333333334444444455000003640000000000007850"), 0x0482a1), - (hex!("0122222222333333334444444455000003650000000000003660"), 0x048341), - (hex!("0122222222333333334444444455000003660000000000003670"), 0x0483e1), - (hex!("0122222222333333334444444455000003660000000000004650"), 0x048481), - (hex!("01222222223333333344444444550000036600000000000050d0"), 0x048521), - (hex!("0122222222333333334444444455000003660000000000006eb0"), 0x0485c1), - (hex!("0122222222333333334444444455000003670000000000003680"), 0x048661), - (hex!("01222222223333333344444444550000036700000000000071f0"), 0x048701), - (hex!("0122222222333333334444444455000003680000000000003690"), 0x0487a1), - (hex!("01222222223333333344444444550000036900000000000036a0"), 0x048841), - (hex!("0122222222333333334444444455000003690000000000005c70"), 0x0488e1), - (hex!("01222222223333333344444444550000036a00000000000036b0"), 0x048981), - (hex!("01222222223333333344444444550000036a00000000000071b0"), 0x048a21), - (hex!("01222222223333333344444444550000036b00000000000036c0"), 0x048ac1), - (hex!("01222222223333333344444444550000036b0000000000004670"), 0x048b61), - (hex!("01222222223333333344444444550000036c00000000000036d0"), 0x048c01), - (hex!("01222222223333333344444444550000036c0000000000004750"), 0x048ca1), - (hex!("01222222223333333344444444550000036c0000000000006fa0"), 0x048d41), - (hex!("01222222223333333344444444550000036d00000000000036e0"), 0x048de1), - (hex!("01222222223333333344444444550000036d0000000000003f70"), 0x048e81), - (hex!("01222222223333333344444444550000036d0000000000004b90"), 0x048f21), - (hex!("01222222223333333344444444550000036d00000000000057a0"), 0x048fc1), - (hex!("01222222223333333344444444550000036e00000000000036f0"), 0x049061), - (hex!("01222222223333333344444444550000036e00000000000075d0"), 0x049101), - (hex!("01222222223333333344444444550000036f0000000000003700"), 0x0491a1), - (hex!("0122222222333333334444444455000003700000000000003710"), 0x049241), - (hex!("0122222222333333334444444455000003700000000000005aa0"), 0x0492e1), - (hex!("0122222222333333334444444455000003710000000000003720"), 0x049381), - (hex!("0122222222333333334444444455000003710000000000005130"), 0x049421), - (hex!("0122222222333333334444444455000003710000000000006fc0"), 0x0494c1), - (hex!("0122222222333333334444444455000003710000000000007b00"), 0x049561), - (hex!("0122222222333333334444444455000003720000000000003730"), 0x049601), - (hex!("01222222223333333344444444550000037200000000000054d0"), 0x0496a1), - (hex!("0122222222333333334444444455000003730000000000003740"), 0x049741), - (hex!("0122222222333333334444444455000003730000000000004220"), 0x0497e1), - (hex!("0122222222333333334444444455000003740000000000003750"), 0x049881), - (hex!("0122222222333333334444444455000003740000000000004720"), 0x049921), - (hex!("0122222222333333334444444455000003750000000000003760"), 0x0499c1), - (hex!("0122222222333333334444444455000003750000000000004110"), 0x049a61), - (hex!("0122222222333333334444444455000003760000000000003770"), 0x049b01), - (hex!("0122222222333333334444444455000003770000000000003780"), 0x049ba1), - (hex!("0122222222333333334444444455000003780000000000003790"), 0x049c41), - (hex!("0122222222333333334444444455000003780000000000004b40"), 0x049ce1), - (hex!("0122222222333333334444444455000003780000000000005660"), 0x049d81), - (hex!("0122222222333333334444444455000003780000000000005ea0"), 0x049e21), - (hex!("01222222223333333344444444550000037900000000000037a0"), 0x049ec1), - (hex!("01222222223333333344444444550000037a00000000000037b0"), 0x049f61), - (hex!("01222222223333333344444444550000037b00000000000037c0"), 0x04a001), - (hex!("01222222223333333344444444550000037c00000000000037d0"), 0x04a0a1), - (hex!("01222222223333333344444444550000037c0000000000004340"), 0x04a141), - (hex!("01222222223333333344444444550000037c0000000000005230"), 0x04a1e1), - (hex!("01222222223333333344444444550000037d00000000000037e0"), 0x04a281), - (hex!("01222222223333333344444444550000037d00000000000051e0"), 0x04a321), - (hex!("01222222223333333344444444550000037e00000000000037f0"), 0x04a3c1), - (hex!("01222222223333333344444444550000037e0000000000004090"), 0x04a461), - (hex!("01222222223333333344444444550000037e0000000000005c20"), 0x04a501), - (hex!("01222222223333333344444444550000037f0000000000003800"), 0x04a5a1), - (hex!("0122222222333333334444444455000003800000000000003810"), 0x04a641), - (hex!("0122222222333333334444444455000003800000000000007630"), 0x04a6e1), - (hex!("0122222222333333334444444455000003810000000000003820"), 0x04a781), - (hex!("0122222222333333334444444455000003820000000000003830"), 0x04a821), - (hex!("0122222222333333334444444455000003820000000000004170"), 0x04a8c1), - (hex!("0122222222333333334444444455000003830000000000003840"), 0x04a961), - (hex!("0122222222333333334444444455000003840000000000003850"), 0x04aa01), - (hex!("0122222222333333334444444455000003850000000000003860"), 0x04aaa1), - (hex!("0122222222333333334444444455000003850000000000004180"), 0x04ab41), - (hex!("0122222222333333334444444455000003850000000000005c90"), 0x04abe1), - (hex!("0122222222333333334444444455000003850000000000005da0"), 0x04ac81), - (hex!("0122222222333333334444444455000003850000000000006ff0"), 0x04ad21), - (hex!("0122222222333333334444444455000003860000000000003870"), 0x04adc1), - (hex!("01222222223333333344444444550000038600000000000065c0"), 0x04ae61), - (hex!("0122222222333333334444444455000003870000000000003880"), 0x04af01), - (hex!("0122222222333333334444444455000003870000000000007cc0"), 0x04afa1), - (hex!("0122222222333333334444444455000003880000000000003890"), 0x04b041), - (hex!("01222222223333333344444444550000038900000000000038a0"), 0x04b0e1), - (hex!("01222222223333333344444444550000038a00000000000038b0"), 0x04b181), - (hex!("01222222223333333344444444550000038a00000000000073e0"), 0x04b221), - (hex!("01222222223333333344444444550000038b00000000000038c0"), 0x04b2c1), - (hex!("01222222223333333344444444550000038c00000000000038d0"), 0x04b361), - (hex!("01222222223333333344444444550000038d00000000000038e0"), 0x04b401), - (hex!("01222222223333333344444444550000038d00000000000069f0"), 0x04b4a1), - (hex!("01222222223333333344444444550000038d0000000000007680"), 0x04b541), - (hex!("01222222223333333344444444550000038e00000000000038f0"), 0x04b5e1), - (hex!("01222222223333333344444444550000038f0000000000003900"), 0x04b681), - (hex!("01222222223333333344444444550000038f00000000000045b0"), 0x04b721), - (hex!("01222222223333333344444444550000038f0000000000007180"), 0x04b7c1), - (hex!("0122222222333333334444444455000003900000000000003910"), 0x04b861), - (hex!("0122222222333333334444444455000003910000000000003920"), 0x04b901), - (hex!("0122222222333333334444444455000003910000000000004a20"), 0x04b9a1), - (hex!("0122222222333333334444444455000003920000000000003930"), 0x04ba41), - (hex!("01222222223333333344444444550000039200000000000059b0"), 0x04bae1), - (hex!("0122222222333333334444444455000003930000000000003940"), 0x04bb81), - (hex!("0122222222333333334444444455000003930000000000006cc0"), 0x04bc21), - (hex!("0122222222333333334444444455000003940000000000003950"), 0x04bcc1), - (hex!("01222222223333333344444444550000039400000000000056c0"), 0x04bd61), - (hex!("0122222222333333334444444455000003950000000000003960"), 0x04be01), - (hex!("0122222222333333334444444455000003950000000000004cc0"), 0x04bea1), - (hex!("0122222222333333334444444455000003950000000000007720"), 0x04bf41), - (hex!("0122222222333333334444444455000003960000000000003970"), 0x04bfe1), - (hex!("0122222222333333334444444455000003960000000000004da0"), 0x04c081), - (hex!("0122222222333333334444444455000003960000000000004df0"), 0x04c121), - (hex!("0122222222333333334444444455000003960000000000004f30"), 0x04c1c1), - (hex!("01222222223333333344444444550000039600000000000050f0"), 0x04c261), - (hex!("0122222222333333334444444455000003960000000000007940"), 0x04c301), - (hex!("0122222222333333334444444455000003970000000000003980"), 0x04c3a1), - (hex!("0122222222333333334444444455000003970000000000005850"), 0x04c441), - (hex!("0122222222333333334444444455000003970000000000007bd0"), 0x04c4e1), - (hex!("0122222222333333334444444455000003980000000000003990"), 0x04c581), - (hex!("0122222222333333334444444455000003980000000000004c00"), 0x04c621), - (hex!("0122222222333333334444444455000003980000000000005580"), 0x04c6c1), - (hex!("01222222223333333344444444550000039900000000000039a0"), 0x04c761), - (hex!("0122222222333333334444444455000003990000000000005820"), 0x04c801), - (hex!("01222222223333333344444444550000039a00000000000039b0"), 0x04c8a1), - (hex!("01222222223333333344444444550000039b00000000000039c0"), 0x04c941), - (hex!("01222222223333333344444444550000039b0000000000004c10"), 0x04c9e1), - (hex!("01222222223333333344444444550000039b0000000000006460"), 0x04ca81), - (hex!("01222222223333333344444444550000039c00000000000039d0"), 0x04cb21), - (hex!("01222222223333333344444444550000039d00000000000039e0"), 0x04cbc1), - (hex!("01222222223333333344444444550000039d00000000000044c0"), 0x04cc61), - (hex!("01222222223333333344444444550000039d00000000000049e0"), 0x04cd01), - (hex!("01222222223333333344444444550000039e00000000000039f0"), 0x04cda1), - (hex!("01222222223333333344444444550000039f0000000000003a00"), 0x04ce41), - (hex!("0122222222333333334444444455000003a00000000000003a10"), 0x04cee1), - (hex!("0122222222333333334444444455000003a10000000000003a20"), 0x04cf81), - (hex!("0122222222333333334444444455000003a10000000000006a80"), 0x04d021), - (hex!("0122222222333333334444444455000003a20000000000003a30"), 0x04d0c1), - (hex!("0122222222333333334444444455000003a200000000000062b0"), 0x04d161), - (hex!("0122222222333333334444444455000003a30000000000003a40"), 0x04d201), - (hex!("0122222222333333334444444455000003a30000000000006ce0"), 0x04d2a1), - (hex!("0122222222333333334444444455000003a40000000000003a50"), 0x04d341), - (hex!("0122222222333333334444444455000003a50000000000003a60"), 0x04d3e1), - (hex!("0122222222333333334444444455000003a60000000000003a70"), 0x04d481), - (hex!("0122222222333333334444444455000003a60000000000007750"), 0x04d521), - (hex!("0122222222333333334444444455000003a70000000000003a80"), 0x04d5c1), - (hex!("0122222222333333334444444455000003a70000000000005b10"), 0x04d661), - (hex!("0122222222333333334444444455000003a80000000000003a90"), 0x04d701), - (hex!("0122222222333333334444444455000003a80000000000006c20"), 0x04d7a1), - (hex!("0122222222333333334444444455000003a90000000000003aa0"), 0x04d841), - (hex!("0122222222333333334444444455000003a90000000000005b70"), 0x04d8e1), - (hex!("0122222222333333334444444455000003a900000000000070e0"), 0x04d981), - (hex!("0122222222333333334444444455000003aa0000000000003ab0"), 0x04da21), - (hex!("0122222222333333334444444455000003aa00000000000049f0"), 0x04dac1), - (hex!("0122222222333333334444444455000003aa0000000000004d60"), 0x04db61), - (hex!("0122222222333333334444444455000003ab0000000000003ac0"), 0x04dc01), - (hex!("0122222222333333334444444455000003ac0000000000003ad0"), 0x04dca1), - (hex!("0122222222333333334444444455000003ac0000000000004580"), 0x04dd41), - (hex!("0122222222333333334444444455000003ad0000000000003ae0"), 0x04dde1), - (hex!("0122222222333333334444444455000003ae0000000000003af0"), 0x04de81), - (hex!("0122222222333333334444444455000003af0000000000003b00"), 0x04df21), - (hex!("0122222222333333334444444455000003b00000000000003b10"), 0x04dfc1), - (hex!("0122222222333333334444444455000003b10000000000003b20"), 0x04e061), - (hex!("0122222222333333334444444455000003b10000000000003fd0"), 0x04e101), - (hex!("0122222222333333334444444455000003b20000000000003b30"), 0x04e1a1), - (hex!("0122222222333333334444444455000003b30000000000003b40"), 0x04e241), - (hex!("0122222222333333334444444455000003b40000000000003b50"), 0x04e2e1), - (hex!("0122222222333333334444444455000003b40000000000007450"), 0x04e381), - (hex!("0122222222333333334444444455000003b50000000000003b60"), 0x04e421), - (hex!("0122222222333333334444444455000003b60000000000003b70"), 0x04e4c1), - (hex!("0122222222333333334444444455000003b70000000000003b80"), 0x04e561), - (hex!("0122222222333333334444444455000003b70000000000006d50"), 0x04e601), - (hex!("0122222222333333334444444455000003b80000000000003b90"), 0x04e6a1), - (hex!("0122222222333333334444444455000003b800000000000057c0"), 0x04e741), - (hex!("0122222222333333334444444455000003b800000000000078a0"), 0x04e7e1), - (hex!("0122222222333333334444444455000003b90000000000003ba0"), 0x04e881), - (hex!("0122222222333333334444444455000003b90000000000006750"), 0x04e921), - (hex!("0122222222333333334444444455000003ba0000000000003bb0"), 0x04e9c1), - (hex!("0122222222333333334444444455000003ba0000000000007a10"), 0x04ea61), - (hex!("0122222222333333334444444455000003ba0000000000007a20"), 0x04eb01), - (hex!("0122222222333333334444444455000003bb0000000000003bc0"), 0x04eba1), - (hex!("0122222222333333334444444455000003bb0000000000005bc0"), 0x04ec41), - (hex!("0122222222333333334444444455000003bc0000000000003bd0"), 0x04ece1), - (hex!("0122222222333333334444444455000003bc0000000000005e80"), 0x04ed81), - (hex!("0122222222333333334444444455000003bc0000000000007ab0"), 0x04ee21), - (hex!("0122222222333333334444444455000003bd0000000000003be0"), 0x04eec1), - (hex!("0122222222333333334444444455000003bd00000000000049b0"), 0x04ef61), - (hex!("0122222222333333334444444455000003be0000000000003bf0"), 0x04f001), - (hex!("0122222222333333334444444455000003be0000000000005780"), 0x04f0a1), - (hex!("0122222222333333334444444455000003be0000000000007930"), 0x04f141), - (hex!("0122222222333333334444444455000003bf0000000000003c00"), 0x04f1e1), - (hex!("0122222222333333334444444455000003bf0000000000005de0"), 0x04f281), - (hex!("0122222222333333334444444455000003bf00000000000060b0"), 0x04f321), - (hex!("0122222222333333334444444455000003bf00000000000060c0"), 0x04f3c1), - (hex!("0122222222333333334444444455000003bf0000000000006a50"), 0x04f461), - (hex!("0122222222333333334444444455000003c00000000000003c10"), 0x04f501), - (hex!("0122222222333333334444444455000003c00000000000004030"), 0x04f5a1), - (hex!("0122222222333333334444444455000003c10000000000003c20"), 0x04f641), - (hex!("0122222222333333334444444455000003c20000000000003c30"), 0x04f6e1), - (hex!("0122222222333333334444444455000003c200000000000040b0"), 0x04f781), - (hex!("0122222222333333334444444455000003c30000000000003c40"), 0x04f821), - (hex!("0122222222333333334444444455000003c40000000000003c50"), 0x04f8c1), - (hex!("0122222222333333334444444455000003c40000000000005ba0"), 0x04f961), - (hex!("0122222222333333334444444455000003c50000000000003c60"), 0x04fa01), - (hex!("0122222222333333334444444455000003c60000000000003c70"), 0x04faa1), - (hex!("0122222222333333334444444455000003c70000000000003c80"), 0x04fb41), - (hex!("0122222222333333334444444455000003c70000000000004270"), 0x04fbe1), - (hex!("0122222222333333334444444455000003c80000000000003c90"), 0x04fc81), - (hex!("0122222222333333334444444455000003c80000000000006e70"), 0x04fd21), - (hex!("0122222222333333334444444455000003c90000000000003ca0"), 0x04fdc1), - (hex!("0122222222333333334444444455000003ca0000000000003cb0"), 0x04fe61), - (hex!("0122222222333333334444444455000003ca0000000000006e20"), 0x04ff01), - (hex!("0122222222333333334444444455000003ca0000000000007c20"), 0x04ffa1), - (hex!("0122222222333333334444444455000003cb0000000000003cc0"), 0x050041), - (hex!("0122222222333333334444444455000003cc0000000000003cd0"), 0x0500e1), - (hex!("0122222222333333334444444455000003cc0000000000006120"), 0x050181), - (hex!("0122222222333333334444444455000003cc0000000000007950"), 0x050221), - (hex!("0122222222333333334444444455000003cd0000000000003ce0"), 0x0502c1), - (hex!("0122222222333333334444444455000003ce0000000000003cf0"), 0x050361), - (hex!("0122222222333333334444444455000003cf0000000000003d00"), 0x050401), - (hex!("0122222222333333334444444455000003d00000000000003d10"), 0x0504a1), - (hex!("0122222222333333334444444455000003d10000000000003d20"), 0x050541), - (hex!("0122222222333333334444444455000003d10000000000005e50"), 0x0505e1), - (hex!("0122222222333333334444444455000003d10000000000007880"), 0x050681), - (hex!("0122222222333333334444444455000003d20000000000003d30"), 0x050721), - (hex!("0122222222333333334444444455000003d20000000000005d00"), 0x0507c1), - (hex!("0122222222333333334444444455000003d30000000000003d40"), 0x050861), - (hex!("0122222222333333334444444455000003d30000000000005d40"), 0x050901), - (hex!("0122222222333333334444444455000003d300000000000063f0"), 0x0509a1), - (hex!("0122222222333333334444444455000003d40000000000003d50"), 0x050a41), - (hex!("0122222222333333334444444455000003d40000000000005700"), 0x050ae1), - (hex!("0122222222333333334444444455000003d400000000000078f0"), 0x050b81), - (hex!("0122222222333333334444444455000003d50000000000003d60"), 0x050c21), - (hex!("0122222222333333334444444455000003d60000000000003d70"), 0x050cc1), - (hex!("0122222222333333334444444455000003d70000000000003d80"), 0x050d61), - (hex!("0122222222333333334444444455000003d80000000000003d90"), 0x050e01), - (hex!("0122222222333333334444444455000003d80000000000006690"), 0x050ea1), - (hex!("0122222222333333334444444455000003d90000000000003da0"), 0x050f41), - (hex!("0122222222333333334444444455000003d900000000000076d0"), 0x050fe1), - (hex!("0122222222333333334444444455000003da0000000000003db0"), 0x051081), - (hex!("0122222222333333334444444455000003db0000000000003dc0"), 0x051121), - (hex!("0122222222333333334444444455000003db0000000000004a30"), 0x0511c1), - (hex!("0122222222333333334444444455000003db0000000000005390"), 0x051261), - (hex!("0122222222333333334444444455000003dc0000000000003dd0"), 0x051301), - (hex!("0122222222333333334444444455000003dc0000000000006d60"), 0x0513a1), - (hex!("0122222222333333334444444455000003dd0000000000003de0"), 0x051441), - (hex!("0122222222333333334444444455000003de0000000000003df0"), 0x0514e1), - (hex!("0122222222333333334444444455000003df0000000000003e00"), 0x051581), - (hex!("0122222222333333334444444455000003df0000000000005240"), 0x051621), - (hex!("0122222222333333334444444455000003df0000000000005610"), 0x0516c1), - (hex!("0122222222333333334444444455000003e00000000000003e10"), 0x051761), - (hex!("0122222222333333334444444455000003e00000000000006500"), 0x051801), - (hex!("0122222222333333334444444455000003e10000000000003e20"), 0x0518a1), - (hex!("0122222222333333334444444455000003e10000000000006a10"), 0x051941), - (hex!("0122222222333333334444444455000003e10000000000007c10"), 0x0519e1), - (hex!("0122222222333333334444444455000003e20000000000003e30"), 0x051a81), - (hex!("0122222222333333334444444455000003e20000000000006310"), 0x051b21), - (hex!("0122222222333333334444444455000003e30000000000003e40"), 0x051bc1), - (hex!("0122222222333333334444444455000003e40000000000003e50"), 0x051c61), - (hex!("0122222222333333334444444455000003e40000000000006780"), 0x051d01), - (hex!("0122222222333333334444444455000003e40000000000007ce0"), 0x051da1), - (hex!("0122222222333333334444444455000003e50000000000003e60"), 0x051e41), - (hex!("0122222222333333334444444455000003e60000000000003e70"), 0x051ee1), - (hex!("0122222222333333334444444455000003e60000000000005040"), 0x051f81), - (hex!("0122222222333333334444444455000003e60000000000005bf0"), 0x052021), - (hex!("0122222222333333334444444455000003e70000000000003e80"), 0x0520c1), - (hex!("0122222222333333334444444455000003e70000000000003f50"), 0x052161), + (hex!("0100000000333333334444444455000000000000000000000010"), 0x004001), + (hex!("0100000000333333334444444455000000000000000000007cb0"), 0x0040a1), + (hex!("0100000000333333334444444455000000010000000000000020"), 0x004141), + (hex!("0100000000333333334444444455000000020000000000000030"), 0x0041e1), + (hex!("01000000003333333344444444550000000200000000000051a0"), 0x004281), + (hex!("0100000000333333334444444455000000030000000000000040"), 0x004321), + (hex!("0100000000333333334444444455000000030000000000006cf0"), 0x0043c1), + (hex!("0100000000333333334444444455000000030000000000007140"), 0x004461), + (hex!("0100000000333333334444444455000000040000000000000050"), 0x004501), + (hex!("01000000003333333344444444550000000400000000000047f0"), 0x0045a1), + (hex!("01000000003333333344444444550000000400000000000072b0"), 0x004641), + (hex!("0100000000333333334444444455000000050000000000000060"), 0x0046e1), + (hex!("0100000000333333334444444455000000050000000000005550"), 0x004781), + (hex!("0100000000333333334444444455000000060000000000000070"), 0x004821), + (hex!("01000000003333333344444444550000000600000000000044a0"), 0x0048c1), + (hex!("0100000000333333334444444455000000060000000000006870"), 0x004961), + (hex!("0100000000333333334444444455000000070000000000000080"), 0x004a01), + (hex!("0100000000333333334444444455000000080000000000000090"), 0x004aa1), + (hex!("0100000000333333334444444455000000080000000000004150"), 0x004b41), + (hex!("01000000003333333344444444550000000900000000000000a0"), 0x004be1), + (hex!("01000000003333333344444444550000000a00000000000000b0"), 0x004c81), + (hex!("01000000003333333344444444550000000a0000000000006680"), 0x004d21), + (hex!("01000000003333333344444444550000000b00000000000000c0"), 0x004dc1), + (hex!("01000000003333333344444444550000000b0000000000006230"), 0x004e61), + (hex!("01000000003333333344444444550000000c00000000000000d0"), 0x004f01), + (hex!("01000000003333333344444444550000000d00000000000000e0"), 0x004fa1), + (hex!("01000000003333333344444444550000000e00000000000000f0"), 0x005041), + (hex!("01000000003333333344444444550000000e0000000000006000"), 0x0050e1), + (hex!("01000000003333333344444444550000000f0000000000000100"), 0x005181), + (hex!("01000000003333333344444444550000000f00000000000053c0"), 0x005221), + (hex!("01000000003333333344444444550000000f0000000000006580"), 0x0052c1), + (hex!("0100000000333333334444444455000000100000000000000110"), 0x005361), + (hex!("01000000003333333344444444550000001000000000000046c0"), 0x005401), + (hex!("0100000000333333334444444455000000100000000000004e40"), 0x0054a1), + (hex!("0100000000333333334444444455000000110000000000000120"), 0x005541), + (hex!("0100000000333333334444444455000000120000000000000130"), 0x0055e1), + (hex!("01000000003333333344444444550000001200000000000066d0"), 0x005681), + (hex!("0100000000333333334444444455000000130000000000000140"), 0x005721), + (hex!("0100000000333333334444444455000000130000000000007710"), 0x0057c1), + (hex!("0100000000333333334444444455000000140000000000000150"), 0x005861), + (hex!("0100000000333333334444444455000000140000000000006c40"), 0x005901), + (hex!("0100000000333333334444444455000000150000000000000160"), 0x0059a1), + (hex!("0100000000333333334444444455000000150000000000005990"), 0x005a41), + (hex!("0100000000333333334444444455000000160000000000000170"), 0x005ae1), + (hex!("0100000000333333334444444455000000160000000000005530"), 0x005b81), + (hex!("0100000000333333334444444455000000170000000000000180"), 0x005c21), + (hex!("0100000000333333334444444455000000170000000000004290"), 0x005cc1), + (hex!("0100000000333333334444444455000000180000000000000190"), 0x005d61), + (hex!("01000000003333333344444444550000001800000000000051c0"), 0x005e01), + (hex!("01000000003333333344444444550000001900000000000001a0"), 0x005ea1), + (hex!("0100000000333333334444444455000000190000000000005420"), 0x005f41), + (hex!("0100000000333333334444444455000000190000000000005770"), 0x005fe1), + (hex!("01000000003333333344444444550000001900000000000079d0"), 0x006081), + (hex!("01000000003333333344444444550000001a00000000000001b0"), 0x006121), + (hex!("01000000003333333344444444550000001a0000000000006f70"), 0x0061c1), + (hex!("01000000003333333344444444550000001a0000000000007150"), 0x006261), + (hex!("01000000003333333344444444550000001b00000000000001c0"), 0x006301), + (hex!("01000000003333333344444444550000001b0000000000005070"), 0x0063a1), + (hex!("01000000003333333344444444550000001c00000000000001d0"), 0x006441), + (hex!("01000000003333333344444444550000001d00000000000001e0"), 0x0064e1), + (hex!("01000000003333333344444444550000001e00000000000001f0"), 0x006581), + (hex!("01000000003333333344444444550000001e0000000000005650"), 0x006621), + (hex!("01000000003333333344444444550000001f0000000000000200"), 0x0066c1), + (hex!("01000000003333333344444444550000001f0000000000006ca0"), 0x006761), + (hex!("0100000000333333334444444455000000200000000000000210"), 0x006801), + (hex!("0100000000333333334444444455000000200000000000005fc0"), 0x0068a1), + (hex!("0100000000333333334444444455000000210000000000000220"), 0x006941), + (hex!("0100000000333333334444444455000000210000000000006430"), 0x0069e1), + (hex!("0100000000333333334444444455000000220000000000000230"), 0x006a81), + (hex!("01000000003333333344444444550000002200000000000040e0"), 0x006b21), + (hex!("0100000000333333334444444455000000230000000000000240"), 0x006bc1), + (hex!("01000000003333333344444444550000002300000000000042d0"), 0x006c61), + (hex!("0100000000333333334444444455000000240000000000000250"), 0x006d01), + (hex!("0100000000333333334444444455000000250000000000000260"), 0x006da1), + (hex!("01000000003333333344444444550000002500000000000058c0"), 0x006e41), + (hex!("0100000000333333334444444455000000260000000000000270"), 0x006ee1), + (hex!("0100000000333333334444444455000000260000000000004020"), 0x006f81), + (hex!("0100000000333333334444444455000000270000000000000280"), 0x007021), + (hex!("0100000000333333334444444455000000280000000000000290"), 0x0070c1), + (hex!("0100000000333333334444444455000000280000000000007c00"), 0x007161), + (hex!("01000000003333333344444444550000002900000000000002a0"), 0x007201), + (hex!("01000000003333333344444444550000002a00000000000002b0"), 0x0072a1), + (hex!("01000000003333333344444444550000002b00000000000002c0"), 0x007341), + (hex!("01000000003333333344444444550000002c00000000000002d0"), 0x0073e1), + (hex!("01000000003333333344444444550000002c00000000000041b0"), 0x007481), + (hex!("01000000003333333344444444550000002c0000000000004c30"), 0x007521), + (hex!("01000000003333333344444444550000002d00000000000002e0"), 0x0075c1), + (hex!("01000000003333333344444444550000002d0000000000005e40"), 0x007661), + (hex!("01000000003333333344444444550000002d0000000000006990"), 0x007701), + (hex!("01000000003333333344444444550000002e00000000000002f0"), 0x0077a1), + (hex!("01000000003333333344444444550000002f0000000000000300"), 0x007841), + (hex!("01000000003333333344444444550000002f0000000000004a70"), 0x0078e1), + (hex!("01000000003333333344444444550000002f0000000000006b40"), 0x007981), + (hex!("0100000000333333334444444455000000300000000000000310"), 0x007a21), + (hex!("0100000000333333334444444455000000310000000000000320"), 0x007ac1), + (hex!("0100000000333333334444444455000000320000000000000330"), 0x007b61), + (hex!("01000000003333333344444444550000003200000000000041a0"), 0x007c01), + (hex!("0100000000333333334444444455000000320000000000007340"), 0x007ca1), + (hex!("0100000000333333334444444455000000320000000000007730"), 0x007d41), + (hex!("0100000000333333334444444455000000330000000000000340"), 0x007de1), + (hex!("01000000003333333344444444550000003300000000000055a0"), 0x007e81), + (hex!("0100000000333333334444444455000000340000000000000350"), 0x007f21), + (hex!("0100000000333333334444444455000000350000000000000360"), 0x007fc1), + (hex!("01000000003333333344444444550000003500000000000077a0"), 0x008061), + (hex!("0100000000333333334444444455000000360000000000000370"), 0x008101), + (hex!("0100000000333333334444444455000000370000000000000380"), 0x0081a1), + (hex!("0100000000333333334444444455000000380000000000000390"), 0x008241), + (hex!("01000000003333333344444444550000003900000000000003a0"), 0x0082e1), + (hex!("01000000003333333344444444550000003a00000000000003b0"), 0x008381), + (hex!("01000000003333333344444444550000003a00000000000071c0"), 0x008421), + (hex!("01000000003333333344444444550000003b00000000000003c0"), 0x0084c1), + (hex!("01000000003333333344444444550000003c00000000000003d0"), 0x008561), + (hex!("01000000003333333344444444550000003d00000000000003e0"), 0x008601), + (hex!("01000000003333333344444444550000003e00000000000003f0"), 0x0086a1), + (hex!("01000000003333333344444444550000003e00000000000062e0"), 0x008741), + (hex!("01000000003333333344444444550000003f0000000000000400"), 0x0087e1), + (hex!("0100000000333333334444444455000000400000000000000410"), 0x008881), + (hex!("0100000000333333334444444455000000400000000000004460"), 0x008921), + (hex!("0100000000333333334444444455000000400000000000005b90"), 0x0089c1), + (hex!("01000000003333333344444444550000004000000000000079b0"), 0x008a61), + (hex!("0100000000333333334444444455000000410000000000000420"), 0x008b01), + (hex!("0100000000333333334444444455000000420000000000000430"), 0x008ba1), + (hex!("0100000000333333334444444455000000420000000000005640"), 0x008c41), + (hex!("0100000000333333334444444455000000430000000000000440"), 0x008ce1), + (hex!("01000000003333333344444444550000004300000000000072a0"), 0x008d81), + (hex!("0100000000333333334444444455000000440000000000000450"), 0x008e21), + (hex!("0100000000333333334444444455000000450000000000000460"), 0x008ec1), + (hex!("0100000000333333334444444455000000450000000000005750"), 0x008f61), + (hex!("01000000003333333344444444550000004500000000000077b0"), 0x009001), + (hex!("0100000000333333334444444455000000460000000000000470"), 0x0090a1), + (hex!("0100000000333333334444444455000000470000000000000480"), 0x009141), + (hex!("0100000000333333334444444455000000480000000000000490"), 0x0091e1), + (hex!("01000000003333333344444444550000004800000000000069e0"), 0x009281), + (hex!("01000000003333333344444444550000004900000000000004a0"), 0x009321), + (hex!("0100000000333333334444444455000000490000000000007370"), 0x0093c1), + (hex!("01000000003333333344444444550000004a00000000000004b0"), 0x009461), + (hex!("01000000003333333344444444550000004a0000000000005cb0"), 0x009501), + (hex!("01000000003333333344444444550000004b00000000000004c0"), 0x0095a1), + (hex!("01000000003333333344444444550000004c00000000000004d0"), 0x009641), + (hex!("01000000003333333344444444550000004c0000000000004880"), 0x0096e1), + (hex!("01000000003333333344444444550000004c0000000000007a40"), 0x009781), + (hex!("01000000003333333344444444550000004d00000000000004e0"), 0x009821), + (hex!("01000000003333333344444444550000004d0000000000006390"), 0x0098c1), + (hex!("01000000003333333344444444550000004e00000000000004f0"), 0x009961), + (hex!("01000000003333333344444444550000004e0000000000004db0"), 0x009a01), + (hex!("01000000003333333344444444550000004f0000000000000500"), 0x009aa1), + (hex!("0100000000333333334444444455000000500000000000000510"), 0x009b41), + (hex!("0100000000333333334444444455000000510000000000000520"), 0x009be1), + (hex!("01000000003333333344444444550000005100000000000069c0"), 0x009c81), + (hex!("0100000000333333334444444455000000520000000000000530"), 0x009d21), + (hex!("0100000000333333334444444455000000520000000000006e60"), 0x009dc1), + (hex!("01000000003333333344444444550000005200000000000070c0"), 0x009e61), + (hex!("0100000000333333334444444455000000530000000000000540"), 0x009f01), + (hex!("0100000000333333334444444455000000530000000000005840"), 0x009fa1), + (hex!("0100000000333333334444444455000000540000000000000550"), 0x00a041), + (hex!("01000000003333333344444444550000005400000000000043e0"), 0x00a0e1), + (hex!("01000000003333333344444444550000005400000000000074e0"), 0x00a181), + (hex!("0100000000333333334444444455000000550000000000000560"), 0x00a221), + (hex!("0100000000333333334444444455000000550000000000003ee0"), 0x00a2c1), + (hex!("0100000000333333334444444455000000560000000000000570"), 0x00a361), + (hex!("0100000000333333334444444455000000570000000000000580"), 0x00a401), + (hex!("0100000000333333334444444455000000570000000000007030"), 0x00a4a1), + (hex!("0100000000333333334444444455000000580000000000000590"), 0x00a541), + (hex!("0100000000333333334444444455000000580000000000005340"), 0x00a5e1), + (hex!("01000000003333333344444444550000005800000000000059f0"), 0x00a681), + (hex!("0100000000333333334444444455000000580000000000006930"), 0x00a721), + (hex!("01000000003333333344444444550000005900000000000005a0"), 0x00a7c1), + (hex!("0100000000333333334444444455000000590000000000003f90"), 0x00a861), + (hex!("01000000003333333344444444550000005a00000000000005b0"), 0x00a901), + (hex!("01000000003333333344444444550000005b00000000000005c0"), 0x00a9a1), + (hex!("01000000003333333344444444550000005b00000000000062c0"), 0x00aa41), + (hex!("01000000003333333344444444550000005c00000000000005d0"), 0x00aae1), + (hex!("01000000003333333344444444550000005c0000000000005a70"), 0x00ab81), + (hex!("01000000003333333344444444550000005c0000000000005dd0"), 0x00ac21), + (hex!("01000000003333333344444444550000005d00000000000005e0"), 0x00acc1), + (hex!("01000000003333333344444444550000005d0000000000005730"), 0x00ad61), + (hex!("01000000003333333344444444550000005e00000000000005f0"), 0x00ae01), + (hex!("01000000003333333344444444550000005e0000000000004f40"), 0x00aea1), + (hex!("01000000003333333344444444550000005f0000000000000600"), 0x00af41), + (hex!("0100000000333333334444444455000000600000000000000610"), 0x00afe1), + (hex!("0100000000333333334444444455000000600000000000007c40"), 0x00b081), + (hex!("0100000000333333334444444455000000610000000000000620"), 0x00b121), + (hex!("0100000000333333334444444455000000610000000000007860"), 0x00b1c1), + (hex!("0100000000333333334444444455000000620000000000000630"), 0x00b261), + (hex!("0100000000333333334444444455000000620000000000005050"), 0x00b301), + (hex!("0100000000333333334444444455000000630000000000000640"), 0x00b3a1), + (hex!("0100000000333333334444444455000000640000000000000650"), 0x00b441), + (hex!("0100000000333333334444444455000000650000000000000660"), 0x00b4e1), + (hex!("0100000000333333334444444455000000650000000000005330"), 0x00b581), + (hex!("0100000000333333334444444455000000660000000000000670"), 0x00b621), + (hex!("0100000000333333334444444455000000660000000000004e20"), 0x00b6c1), + (hex!("0100000000333333334444444455000000660000000000005ee0"), 0x00b761), + (hex!("0100000000333333334444444455000000660000000000006360"), 0x00b801), + (hex!("0100000000333333334444444455000000670000000000000680"), 0x00b8a1), + (hex!("0100000000333333334444444455000000670000000000004040"), 0x00b941), + (hex!("0100000000333333334444444455000000680000000000000690"), 0x00b9e1), + (hex!("0100000000333333334444444455000000680000000000003f80"), 0x00ba81), + (hex!("01000000003333333344444444550000006800000000000041e0"), 0x00bb21), + (hex!("01000000003333333344444444550000006900000000000006a0"), 0x00bbc1), + (hex!("0100000000333333334444444455000000690000000000006080"), 0x00bc61), + (hex!("01000000003333333344444444550000006a00000000000006b0"), 0x00bd01), + (hex!("01000000003333333344444444550000006a00000000000042f0"), 0x00bda1), + (hex!("01000000003333333344444444550000006b00000000000006c0"), 0x00be41), + (hex!("01000000003333333344444444550000006b00000000000052f0"), 0x00bee1), + (hex!("01000000003333333344444444550000006b0000000000005980"), 0x00bf81), + (hex!("01000000003333333344444444550000006b0000000000006170"), 0x00c021), + (hex!("01000000003333333344444444550000006c00000000000006d0"), 0x00c0c1), + (hex!("01000000003333333344444444550000006d00000000000006e0"), 0x00c161), + (hex!("01000000003333333344444444550000006d0000000000006fb0"), 0x00c201), + (hex!("01000000003333333344444444550000006e00000000000006f0"), 0x00c2a1), + (hex!("01000000003333333344444444550000006e00000000000065b0"), 0x00c341), + (hex!("01000000003333333344444444550000006e0000000000007970"), 0x00c3e1), + (hex!("01000000003333333344444444550000006f0000000000000700"), 0x00c481), + (hex!("01000000003333333344444444550000006f0000000000005900"), 0x00c521), + (hex!("01000000003333333344444444550000006f0000000000006d90"), 0x00c5c1), + (hex!("0100000000333333334444444455000000700000000000000710"), 0x00c661), + (hex!("01000000003333333344444444550000007000000000000045c0"), 0x00c701), + (hex!("0100000000333333334444444455000000700000000000004d40"), 0x00c7a1), + (hex!("0100000000333333334444444455000000710000000000000720"), 0x00c841), + (hex!("0100000000333333334444444455000000710000000000004dc0"), 0x00c8e1), + (hex!("0100000000333333334444444455000000710000000000007550"), 0x00c981), + (hex!("0100000000333333334444444455000000720000000000000730"), 0x00ca21), + (hex!("0100000000333333334444444455000000720000000000003ec0"), 0x00cac1), + (hex!("01000000003333333344444444550000007200000000000045a0"), 0x00cb61), + (hex!("0100000000333333334444444455000000720000000000006770"), 0x00cc01), + (hex!("0100000000333333334444444455000000720000000000006bc0"), 0x00cca1), + (hex!("0100000000333333334444444455000000730000000000000740"), 0x00cd41), + (hex!("0100000000333333334444444455000000730000000000005250"), 0x00cde1), + (hex!("01000000003333333344444444550000007300000000000075f0"), 0x00ce81), + (hex!("0100000000333333334444444455000000740000000000000750"), 0x00cf21), + (hex!("0100000000333333334444444455000000740000000000003ff0"), 0x00cfc1), + (hex!("01000000003333333344444444550000007400000000000079e0"), 0x00d061), + (hex!("0100000000333333334444444455000000750000000000000760"), 0x00d101), + (hex!("0100000000333333334444444455000000750000000000004310"), 0x00d1a1), + (hex!("0100000000333333334444444455000000760000000000000770"), 0x00d241), + (hex!("0100000000333333334444444455000000770000000000000780"), 0x00d2e1), + (hex!("01000000003333333344444444550000007700000000000062f0"), 0x00d381), + (hex!("0100000000333333334444444455000000770000000000006940"), 0x00d421), + (hex!("0100000000333333334444444455000000780000000000000790"), 0x00d4c1), + (hex!("01000000003333333344444444550000007900000000000007a0"), 0x00d561), + (hex!("0100000000333333334444444455000000790000000000007af0"), 0x00d601), + (hex!("01000000003333333344444444550000007a00000000000007b0"), 0x00d6a1), + (hex!("01000000003333333344444444550000007b00000000000007c0"), 0x00d741), + (hex!("01000000003333333344444444550000007b00000000000067e0"), 0x00d7e1), + (hex!("01000000003333333344444444550000007b0000000000007890"), 0x00d881), + (hex!("01000000003333333344444444550000007c00000000000007d0"), 0x00d921), + (hex!("01000000003333333344444444550000007d00000000000007e0"), 0x00d9c1), + (hex!("01000000003333333344444444550000007e00000000000007f0"), 0x00da61), + (hex!("01000000003333333344444444550000007f0000000000000800"), 0x00db01), + (hex!("01000000003333333344444444550000007f0000000000005be0"), 0x00dba1), + (hex!("0100000000333333334444444455000000800000000000000810"), 0x00dc41), + (hex!("0100000000333333334444444455000000810000000000000820"), 0x00dce1), + (hex!("0100000000333333334444444455000000810000000000007190"), 0x00dd81), + (hex!("0100000000333333334444444455000000820000000000000830"), 0x00de21), + (hex!("0100000000333333334444444455000000820000000000004ab0"), 0x00dec1), + (hex!("0100000000333333334444444455000000830000000000000840"), 0x00df61), + (hex!("0100000000333333334444444455000000830000000000006720"), 0x00e001), + (hex!("0100000000333333334444444455000000840000000000000850"), 0x00e0a1), + (hex!("0100000000333333334444444455000000850000000000000860"), 0x00e141), + (hex!("01000000003333333344444444550000008500000000000054f0"), 0x00e1e1), + (hex!("0100000000333333334444444455000000850000000000007920"), 0x00e281), + (hex!("0100000000333333334444444455000000860000000000000870"), 0x00e321), + (hex!("01000000003333333344444444550000008600000000000060e0"), 0x00e3c1), + (hex!("0100000000333333334444444455000000860000000000006be0"), 0x00e461), + (hex!("0100000000333333334444444455000000870000000000000880"), 0x00e501), + (hex!("0100000000333333334444444455000000870000000000006820"), 0x00e5a1), + (hex!("0100000000333333334444444455000000880000000000000890"), 0x00e641), + (hex!("01000000003333333344444444550000008900000000000008a0"), 0x00e6e1), + (hex!("0100000000333333334444444455000000890000000000007c30"), 0x00e781), + (hex!("01000000003333333344444444550000008a00000000000008b0"), 0x00e821), + (hex!("01000000003333333344444444550000008b00000000000008c0"), 0x00e8c1), + (hex!("01000000003333333344444444550000008b0000000000005910"), 0x00e961), + (hex!("01000000003333333344444444550000008b0000000000006fe0"), 0x00ea01), + (hex!("01000000003333333344444444550000008c00000000000008d0"), 0x00eaa1), + (hex!("01000000003333333344444444550000008c0000000000006800"), 0x00eb41), + (hex!("01000000003333333344444444550000008d00000000000008e0"), 0x00ebe1), + (hex!("01000000003333333344444444550000008d0000000000005810"), 0x00ec81), + (hex!("01000000003333333344444444550000008d0000000000007c90"), 0x00ed21), + (hex!("01000000003333333344444444550000008e00000000000008f0"), 0x00edc1), + (hex!("01000000003333333344444444550000008e00000000000058f0"), 0x00ee61), + (hex!("01000000003333333344444444550000008f0000000000000900"), 0x00ef01), + (hex!("01000000003333333344444444550000008f0000000000005a30"), 0x00efa1), + (hex!("0100000000333333334444444455000000900000000000000910"), 0x00f041), + (hex!("0100000000333333334444444455000000900000000000006130"), 0x00f0e1), + (hex!("0100000000333333334444444455000000900000000000006550"), 0x00f181), + (hex!("0100000000333333334444444455000000910000000000000920"), 0x00f221), + (hex!("01000000003333333344444444550000009100000000000079f0"), 0x00f2c1), + (hex!("0100000000333333334444444455000000920000000000000930"), 0x00f361), + (hex!("0100000000333333334444444455000000920000000000005620"), 0x00f401), + (hex!("0100000000333333334444444455000000920000000000005e90"), 0x00f4a1), + (hex!("01000000003333333344444444550000009200000000000063d0"), 0x00f541), + (hex!("01000000003333333344444444550000009200000000000076c0"), 0x00f5e1), + (hex!("0100000000333333334444444455000000930000000000000940"), 0x00f681), + (hex!("01000000003333333344444444550000009300000000000044e0"), 0x00f721), + (hex!("0100000000333333334444444455000000940000000000000950"), 0x00f7c1), + (hex!("0100000000333333334444444455000000940000000000007a30"), 0x00f861), + (hex!("0100000000333333334444444455000000950000000000000960"), 0x00f901), + (hex!("0100000000333333334444444455000000950000000000007a70"), 0x00f9a1), + (hex!("0100000000333333334444444455000000960000000000000970"), 0x00fa41), + (hex!("0100000000333333334444444455000000970000000000000980"), 0x00fae1), + (hex!("0100000000333333334444444455000000970000000000007330"), 0x00fb81), + (hex!("0100000000333333334444444455000000980000000000000990"), 0x00fc21), + (hex!("0100000000333333334444444455000000980000000000005af0"), 0x00fcc1), + (hex!("0100000000333333334444444455000000980000000000007ae0"), 0x00fd61), + (hex!("01000000003333333344444444550000009900000000000009a0"), 0x00fe01), + (hex!("0100000000333333334444444455000000990000000000005160"), 0x00fea1), + (hex!("0100000000333333334444444455000000990000000000006850"), 0x00ff41), + (hex!("01000000003333333344444444550000009a00000000000009b0"), 0x00ffe1), + (hex!("01000000003333333344444444550000009b00000000000009c0"), 0x010081), + (hex!("01000000003333333344444444550000009b0000000000005010"), 0x010121), + (hex!("01000000003333333344444444550000009c00000000000009d0"), 0x0101c1), + (hex!("01000000003333333344444444550000009c00000000000042e0"), 0x010261), + (hex!("01000000003333333344444444550000009d00000000000009e0"), 0x010301), + (hex!("01000000003333333344444444550000009d00000000000057f0"), 0x0103a1), + (hex!("01000000003333333344444444550000009e00000000000009f0"), 0x010441), + (hex!("01000000003333333344444444550000009e0000000000004ef0"), 0x0104e1), + (hex!("01000000003333333344444444550000009f0000000000000a00"), 0x010581), + (hex!("01000000003333333344444444550000009f0000000000006110"), 0x010621), + (hex!("0100000000333333334444444455000000a00000000000000a10"), 0x0106c1), + (hex!("0100000000333333334444444455000000a10000000000000a20"), 0x010761), + (hex!("0100000000333333334444444455000000a100000000000040d0"), 0x010801), + (hex!("0100000000333333334444444455000000a10000000000007670"), 0x0108a1), + (hex!("0100000000333333334444444455000000a20000000000000a30"), 0x010941), + (hex!("0100000000333333334444444455000000a200000000000074d0"), 0x0109e1), + (hex!("0100000000333333334444444455000000a30000000000000a40"), 0x010a81), + (hex!("0100000000333333334444444455000000a30000000000004c90"), 0x010b21), + (hex!("0100000000333333334444444455000000a40000000000000a50"), 0x010bc1), + (hex!("0100000000333333334444444455000000a50000000000000a60"), 0x010c61), + (hex!("0100000000333333334444444455000000a60000000000000a70"), 0x010d01), + (hex!("0100000000333333334444444455000000a60000000000006d80"), 0x010da1), + (hex!("0100000000333333334444444455000000a60000000000007830"), 0x010e41), + (hex!("0100000000333333334444444455000000a70000000000000a80"), 0x010ee1), + (hex!("0100000000333333334444444455000000a700000000000064f0"), 0x010f81), + (hex!("0100000000333333334444444455000000a80000000000000a90"), 0x011021), + (hex!("0100000000333333334444444455000000a90000000000000aa0"), 0x0110c1), + (hex!("0100000000333333334444444455000000a90000000000005e30"), 0x011161), + (hex!("0100000000333333334444444455000000aa0000000000000ab0"), 0x011201), + (hex!("0100000000333333334444444455000000ab0000000000000ac0"), 0x0112a1), + (hex!("0100000000333333334444444455000000ac0000000000000ad0"), 0x011341), + (hex!("0100000000333333334444444455000000ac0000000000006d20"), 0x0113e1), + (hex!("0100000000333333334444444455000000ac0000000000007000"), 0x011481), + (hex!("0100000000333333334444444455000000ad0000000000000ae0"), 0x011521), + (hex!("0100000000333333334444444455000000ae0000000000000af0"), 0x0115c1), + (hex!("0100000000333333334444444455000000ae0000000000004a10"), 0x011661), + (hex!("0100000000333333334444444455000000af0000000000000b00"), 0x011701), + (hex!("0100000000333333334444444455000000af0000000000004e10"), 0x0117a1), + (hex!("0100000000333333334444444455000000b00000000000000b10"), 0x011841), + (hex!("0100000000333333334444444455000000b00000000000004280"), 0x0118e1), + (hex!("0100000000333333334444444455000000b000000000000077e0"), 0x011981), + (hex!("0100000000333333334444444455000000b10000000000000b20"), 0x011a21), + (hex!("0100000000333333334444444455000000b20000000000000b30"), 0x011ac1), + (hex!("0100000000333333334444444455000000b30000000000000b40"), 0x011b61), + (hex!("0100000000333333334444444455000000b30000000000004bc0"), 0x011c01), + (hex!("0100000000333333334444444455000000b40000000000000b50"), 0x011ca1), + (hex!("0100000000333333334444444455000000b50000000000000b60"), 0x011d41), + (hex!("0100000000333333334444444455000000b50000000000004fa0"), 0x011de1), + (hex!("0100000000333333334444444455000000b50000000000006a60"), 0x011e81), + (hex!("0100000000333333334444444455000000b60000000000000b70"), 0x011f21), + (hex!("0100000000333333334444444455000000b60000000000005630"), 0x011fc1), + (hex!("0100000000333333334444444455000000b70000000000000b80"), 0x012061), + (hex!("0100000000333333334444444455000000b80000000000000b90"), 0x012101), + (hex!("0100000000333333334444444455000000b80000000000006f80"), 0x0121a1), + (hex!("0100000000333333334444444455000000b90000000000000ba0"), 0x012241), + (hex!("0100000000333333334444444455000000ba0000000000000bb0"), 0x0122e1), + (hex!("0100000000333333334444444455000000bb0000000000000bc0"), 0x012381), + (hex!("0100000000333333334444444455000000bb00000000000047c0"), 0x012421), + (hex!("0100000000333333334444444455000000bb0000000000006060"), 0x0124c1), + (hex!("0100000000333333334444444455000000bc0000000000000bd0"), 0x012561), + (hex!("0100000000333333334444444455000000bd0000000000000be0"), 0x012601), + (hex!("0100000000333333334444444455000000bd0000000000004e80"), 0x0126a1), + (hex!("0100000000333333334444444455000000be0000000000000bf0"), 0x012741), + (hex!("0100000000333333334444444455000000bf0000000000000c00"), 0x0127e1), + (hex!("0100000000333333334444444455000000bf00000000000047a0"), 0x012881), + (hex!("0100000000333333334444444455000000bf0000000000006da0"), 0x012921), + (hex!("0100000000333333334444444455000000c00000000000000c10"), 0x0129c1), + (hex!("0100000000333333334444444455000000c10000000000000c20"), 0x012a61), + (hex!("0100000000333333334444444455000000c20000000000000c30"), 0x012b01), + (hex!("0100000000333333334444444455000000c20000000000004bd0"), 0x012ba1), + (hex!("0100000000333333334444444455000000c20000000000006ac0"), 0x012c41), + (hex!("0100000000333333334444444455000000c30000000000000c40"), 0x012ce1), + (hex!("0100000000333333334444444455000000c30000000000004660"), 0x012d81), + (hex!("0100000000333333334444444455000000c40000000000000c50"), 0x012e21), + (hex!("0100000000333333334444444455000000c50000000000000c60"), 0x012ec1), + (hex!("0100000000333333334444444455000000c60000000000000c70"), 0x012f61), + (hex!("0100000000333333334444444455000000c60000000000005880"), 0x013001), + (hex!("0100000000333333334444444455000000c60000000000006b70"), 0x0130a1), + (hex!("0100000000333333334444444455000000c70000000000000c80"), 0x013141), + (hex!("0100000000333333334444444455000000c80000000000000c90"), 0x0131e1), + (hex!("0100000000333333334444444455000000c80000000000005310"), 0x013281), + (hex!("0100000000333333334444444455000000c80000000000005db0"), 0x013321), + (hex!("0100000000333333334444444455000000c80000000000007040"), 0x0133c1), + (hex!("0100000000333333334444444455000000c80000000000007290"), 0x013461), + (hex!("0100000000333333334444444455000000c90000000000000ca0"), 0x013501), + (hex!("0100000000333333334444444455000000c90000000000004fe0"), 0x0135a1), + (hex!("0100000000333333334444444455000000ca0000000000000cb0"), 0x013641), + (hex!("0100000000333333334444444455000000ca0000000000006140"), 0x0136e1), + (hex!("0100000000333333334444444455000000ca0000000000007700"), 0x013781), + (hex!("0100000000333333334444444455000000cb0000000000000cc0"), 0x013821), + (hex!("0100000000333333334444444455000000cc0000000000000cd0"), 0x0138c1), + (hex!("0100000000333333334444444455000000cd0000000000000ce0"), 0x013961), + (hex!("0100000000333333334444444455000000cd0000000000003f20"), 0x013a01), + (hex!("0100000000333333334444444455000000cd00000000000040f0"), 0x013aa1), + (hex!("0100000000333333334444444455000000cd0000000000004ec0"), 0x013b41), + (hex!("0100000000333333334444444455000000ce0000000000000cf0"), 0x013be1), + (hex!("0100000000333333334444444455000000ce0000000000007200"), 0x013c81), + (hex!("0100000000333333334444444455000000cf0000000000000d00"), 0x013d21), + (hex!("0100000000333333334444444455000000cf00000000000046a0"), 0x013dc1), + (hex!("0100000000333333334444444455000000cf0000000000005960"), 0x013e61), + (hex!("0100000000333333334444444455000000d00000000000000d10"), 0x013f01), + (hex!("0100000000333333334444444455000000d00000000000005f30"), 0x013fa1), + (hex!("0100000000333333334444444455000000d10000000000000d20"), 0x014041), + (hex!("0100000000333333334444444455000000d10000000000007a00"), 0x0140e1), + (hex!("0100000000333333334444444455000000d20000000000000d30"), 0x014181), + (hex!("0100000000333333334444444455000000d30000000000000d40"), 0x014221), + (hex!("0100000000333333334444444455000000d40000000000000d50"), 0x0142c1), + (hex!("0100000000333333334444444455000000d50000000000000d60"), 0x014361), + (hex!("0100000000333333334444444455000000d50000000000004960"), 0x014401), + (hex!("0100000000333333334444444455000000d500000000000055d0"), 0x0144a1), + (hex!("0100000000333333334444444455000000d500000000000067d0"), 0x014541), + (hex!("0100000000333333334444444455000000d60000000000000d70"), 0x0145e1), + (hex!("0100000000333333334444444455000000d70000000000000d80"), 0x014681), + (hex!("0100000000333333334444444455000000d80000000000000d90"), 0x014721), + (hex!("0100000000333333334444444455000000d800000000000065f0"), 0x0147c1), + (hex!("0100000000333333334444444455000000d90000000000000da0"), 0x014861), + (hex!("0100000000333333334444444455000000d90000000000004980"), 0x014901), + (hex!("0100000000333333334444444455000000da0000000000000db0"), 0x0149a1), + (hex!("0100000000333333334444444455000000da00000000000048c0"), 0x014a41), + (hex!("0100000000333333334444444455000000da00000000000072c0"), 0x014ae1), + (hex!("0100000000333333334444444455000000da00000000000076b0"), 0x014b81), + (hex!("0100000000333333334444444455000000db0000000000000dc0"), 0x014c21), + (hex!("0100000000333333334444444455000000dc0000000000000dd0"), 0x014cc1), + (hex!("0100000000333333334444444455000000dc00000000000040a0"), 0x014d61), + (hex!("0100000000333333334444444455000000dc00000000000074c0"), 0x014e01), + (hex!("0100000000333333334444444455000000dd0000000000000de0"), 0x014ea1), + (hex!("0100000000333333334444444455000000dd0000000000004e50"), 0x014f41), + (hex!("0100000000333333334444444455000000dd0000000000007270"), 0x014fe1), + (hex!("0100000000333333334444444455000000de0000000000000df0"), 0x015081), + (hex!("0100000000333333334444444455000000de00000000000078d0"), 0x015121), + (hex!("0100000000333333334444444455000000df0000000000000e00"), 0x0151c1), + (hex!("0100000000333333334444444455000000df0000000000004d30"), 0x015261), + (hex!("0100000000333333334444444455000000df0000000000006c30"), 0x015301), + (hex!("0100000000333333334444444455000000e00000000000000e10"), 0x0153a1), + (hex!("0100000000333333334444444455000000e00000000000005d30"), 0x015441), + (hex!("0100000000333333334444444455000000e10000000000000e20"), 0x0154e1), + (hex!("0100000000333333334444444455000000e10000000000004610"), 0x015581), + (hex!("0100000000333333334444444455000000e100000000000051d0"), 0x015621), + (hex!("0100000000333333334444444455000000e10000000000005f10"), 0x0156c1), + (hex!("0100000000333333334444444455000000e20000000000000e30"), 0x015761), + (hex!("0100000000333333334444444455000000e20000000000007a90"), 0x015801), + (hex!("0100000000333333334444444455000000e30000000000000e40"), 0x0158a1), + (hex!("0100000000333333334444444455000000e30000000000005ae0"), 0x015941), + (hex!("0100000000333333334444444455000000e40000000000000e50"), 0x0159e1), + (hex!("0100000000333333334444444455000000e50000000000000e60"), 0x015a81), + (hex!("0100000000333333334444444455000000e50000000000004700"), 0x015b21), + (hex!("0100000000333333334444444455000000e500000000000065d0"), 0x015bc1), + (hex!("0100000000333333334444444455000000e60000000000000e70"), 0x015c61), + (hex!("0100000000333333334444444455000000e60000000000004fd0"), 0x015d01), + (hex!("0100000000333333334444444455000000e70000000000000e80"), 0x015da1), + (hex!("0100000000333333334444444455000000e70000000000005150"), 0x015e41), + (hex!("0100000000333333334444444455000000e70000000000005920"), 0x015ee1), + (hex!("0100000000333333334444444455000000e80000000000000e90"), 0x015f81), + (hex!("0100000000333333334444444455000000e80000000000004320"), 0x016021), + (hex!("0100000000333333334444444455000000e80000000000005ec0"), 0x0160c1), + (hex!("0100000000333333334444444455000000e90000000000000ea0"), 0x016161), + (hex!("0100000000333333334444444455000000e900000000000043b0"), 0x016201), + (hex!("0100000000333333334444444455000000ea0000000000000eb0"), 0x0162a1), + (hex!("0100000000333333334444444455000000ea0000000000003ea0"), 0x016341), + (hex!("0100000000333333334444444455000000ea0000000000004f50"), 0x0163e1), + (hex!("0100000000333333334444444455000000ea0000000000007520"), 0x016481), + (hex!("0100000000333333334444444455000000eb0000000000000ec0"), 0x016521), + (hex!("0100000000333333334444444455000000ec0000000000000ed0"), 0x0165c1), + (hex!("0100000000333333334444444455000000ec0000000000006670"), 0x016661), + (hex!("0100000000333333334444444455000000ed0000000000000ee0"), 0x016701), + (hex!("0100000000333333334444444455000000ee0000000000000ef0"), 0x0167a1), + (hex!("0100000000333333334444444455000000ee0000000000004d10"), 0x016841), + (hex!("0100000000333333334444444455000000ef0000000000000f00"), 0x0168e1), + (hex!("0100000000333333334444444455000000f00000000000000f10"), 0x016981), + (hex!("0100000000333333334444444455000000f00000000000007220"), 0x016a21), + (hex!("0100000000333333334444444455000000f00000000000007540"), 0x016ac1), + (hex!("0100000000333333334444444455000000f10000000000000f20"), 0x016b61), + (hex!("0100000000333333334444444455000000f100000000000066f0"), 0x016c01), + (hex!("0100000000333333334444444455000000f20000000000000f30"), 0x016ca1), + (hex!("0100000000333333334444444455000000f20000000000007810"), 0x016d41), + (hex!("0100000000333333334444444455000000f30000000000000f40"), 0x016de1), + (hex!("0100000000333333334444444455000000f30000000000007b70"), 0x016e81), + (hex!("0100000000333333334444444455000000f40000000000000f50"), 0x016f21), + (hex!("0100000000333333334444444455000000f400000000000059c0"), 0x016fc1), + (hex!("0100000000333333334444444455000000f50000000000000f60"), 0x017061), + (hex!("0100000000333333334444444455000000f50000000000003fb0"), 0x017101), + (hex!("0100000000333333334444444455000000f50000000000005740"), 0x0171a1), + (hex!("0100000000333333334444444455000000f500000000000064d0"), 0x017241), + (hex!("0100000000333333334444444455000000f50000000000006960"), 0x0172e1), + (hex!("0100000000333333334444444455000000f60000000000000f70"), 0x017381), + (hex!("0100000000333333334444444455000000f60000000000006d00"), 0x017421), + (hex!("0100000000333333334444444455000000f70000000000000f80"), 0x0174c1), + (hex!("0100000000333333334444444455000000f80000000000000f90"), 0x017561), + (hex!("0100000000333333334444444455000000f90000000000000fa0"), 0x017601), + (hex!("0100000000333333334444444455000000fa0000000000000fb0"), 0x0176a1), + (hex!("0100000000333333334444444455000000fa00000000000067b0"), 0x017741), + (hex!("0100000000333333334444444455000000fb0000000000000fc0"), 0x0177e1), + (hex!("0100000000333333334444444455000000fb0000000000004eb0"), 0x017881), + (hex!("0100000000333333334444444455000000fb0000000000006ef0"), 0x017921), + (hex!("0100000000333333334444444455000000fc0000000000000fd0"), 0x0179c1), + (hex!("0100000000333333334444444455000000fc0000000000004470"), 0x017a61), + (hex!("0100000000333333334444444455000000fc0000000000005940"), 0x017b01), + (hex!("0100000000333333334444444455000000fd0000000000000fe0"), 0x017ba1), + (hex!("0100000000333333334444444455000000fe0000000000000ff0"), 0x017c41), + (hex!("0100000000333333334444444455000000ff0000000000001000"), 0x017ce1), + (hex!("0100000000333333334444444455000000ff0000000000005690"), 0x017d81), + (hex!("0100000000333333334444444455000001000000000000001010"), 0x017e21), + (hex!("0100000000333333334444444455000001000000000000005210"), 0x017ec1), + (hex!("01000000003333333344444444550000010000000000000070a0"), 0x017f61), + (hex!("0100000000333333334444444455000001010000000000001020"), 0x018001), + (hex!("0100000000333333334444444455000001010000000000006b80"), 0x0180a1), + (hex!("0100000000333333334444444455000001020000000000001030"), 0x018141), + (hex!("0100000000333333334444444455000001030000000000001040"), 0x0181e1), + (hex!("0100000000333333334444444455000001030000000000004c80"), 0x018281), + (hex!("0100000000333333334444444455000001040000000000001050"), 0x018321), + (hex!("0100000000333333334444444455000001040000000000004850"), 0x0183c1), + (hex!("01000000003333333344444444550000010400000000000057b0"), 0x018461), + (hex!("0100000000333333334444444455000001050000000000001060"), 0x018501), + (hex!("01000000003333333344444444550000010500000000000048d0"), 0x0185a1), + (hex!("0100000000333333334444444455000001050000000000007870"), 0x018641), + (hex!("0100000000333333334444444455000001060000000000001070"), 0x0186e1), + (hex!("0100000000333333334444444455000001060000000000004f90"), 0x018781), + (hex!("0100000000333333334444444455000001060000000000006270"), 0x018821), + (hex!("0100000000333333334444444455000001070000000000001080"), 0x0188c1), + (hex!("01000000003333333344444444550000010700000000000063b0"), 0x018961), + (hex!("0100000000333333334444444455000001080000000000001090"), 0x018a01), + (hex!("01000000003333333344444444550000010900000000000010a0"), 0x018aa1), + (hex!("0100000000333333334444444455000001090000000000006f40"), 0x018b41), + (hex!("01000000003333333344444444550000010a00000000000010b0"), 0x018be1), + (hex!("01000000003333333344444444550000010a0000000000006640"), 0x018c81), + (hex!("01000000003333333344444444550000010b00000000000010c0"), 0x018d21), + (hex!("01000000003333333344444444550000010c00000000000010d0"), 0x018dc1), + (hex!("01000000003333333344444444550000010d00000000000010e0"), 0x018e61), + (hex!("01000000003333333344444444550000010e00000000000010f0"), 0x018f01), + (hex!("01000000003333333344444444550000010e0000000000005c40"), 0x018fa1), + (hex!("01000000003333333344444444550000010e0000000000007ba0"), 0x019041), + (hex!("01000000003333333344444444550000010f0000000000001100"), 0x0190e1), + (hex!("01000000003333333344444444550000010f0000000000005c30"), 0x019181), + (hex!("0100000000333333334444444455000001100000000000001110"), 0x019221), + (hex!("0100000000333333334444444455000001100000000000007640"), 0x0192c1), + (hex!("0100000000333333334444444455000001110000000000001120"), 0x019361), + (hex!("01000000003333333344444444550000011100000000000052c0"), 0x019401), + (hex!("0100000000333333334444444455000001110000000000005710"), 0x0194a1), + (hex!("0100000000333333334444444455000001110000000000006a00"), 0x019541), + (hex!("0100000000333333334444444455000001120000000000001130"), 0x0195e1), + (hex!("0100000000333333334444444455000001130000000000001140"), 0x019681), + (hex!("0100000000333333334444444455000001140000000000001150"), 0x019721), + (hex!("0100000000333333334444444455000001140000000000003fa0"), 0x0197c1), + (hex!("01000000003333333344444444550000011400000000000054b0"), 0x019861), + (hex!("0100000000333333334444444455000001140000000000006070"), 0x019901), + (hex!("0100000000333333334444444455000001150000000000001160"), 0x0199a1), + (hex!("0100000000333333334444444455000001150000000000005320"), 0x019a41), + (hex!("0100000000333333334444444455000001150000000000006600"), 0x019ae1), + (hex!("0100000000333333334444444455000001150000000000006df0"), 0x019b81), + (hex!("01000000003333333344444444550000011500000000000079c0"), 0x019c21), + (hex!("0100000000333333334444444455000001160000000000001170"), 0x019cc1), + (hex!("0100000000333333334444444455000001170000000000001180"), 0x019d61), + (hex!("0100000000333333334444444455000001170000000000004a60"), 0x019e01), + (hex!("01000000003333333344444444550000011700000000000063c0"), 0x019ea1), + (hex!("0100000000333333334444444455000001180000000000001190"), 0x019f41), + (hex!("0100000000333333334444444455000001180000000000004530"), 0x019fe1), + (hex!("01000000003333333344444444550000011800000000000077c0"), 0x01a081), + (hex!("01000000003333333344444444550000011900000000000011a0"), 0x01a121), + (hex!("01000000003333333344444444550000011a00000000000011b0"), 0x01a1c1), + (hex!("01000000003333333344444444550000011a00000000000041c0"), 0x01a261), + (hex!("01000000003333333344444444550000011a00000000000061e0"), 0x01a301), + (hex!("01000000003333333344444444550000011b00000000000011c0"), 0x01a3a1), + (hex!("01000000003333333344444444550000011c00000000000011d0"), 0x01a441), + (hex!("01000000003333333344444444550000011c0000000000005f90"), 0x01a4e1), + (hex!("01000000003333333344444444550000011d00000000000011e0"), 0x01a581), + (hex!("01000000003333333344444444550000011d0000000000004160"), 0x01a621), + (hex!("01000000003333333344444444550000011e00000000000011f0"), 0x01a6c1), + (hex!("01000000003333333344444444550000011e00000000000056d0"), 0x01a761), + (hex!("01000000003333333344444444550000011f0000000000001200"), 0x01a801), + (hex!("01000000003333333344444444550000011f0000000000004510"), 0x01a8a1), + (hex!("0100000000333333334444444455000001200000000000001210"), 0x01a941), + (hex!("0100000000333333334444444455000001210000000000001220"), 0x01a9e1), + (hex!("0100000000333333334444444455000001210000000000005140"), 0x01aa81), + (hex!("0100000000333333334444444455000001210000000000006710"), 0x01ab21), + (hex!("0100000000333333334444444455000001210000000000006f50"), 0x01abc1), + (hex!("0100000000333333334444444455000001220000000000001230"), 0x01ac61), + (hex!("0100000000333333334444444455000001220000000000005570"), 0x01ad01), + (hex!("0100000000333333334444444455000001220000000000007ac0"), 0x01ada1), + (hex!("0100000000333333334444444455000001230000000000001240"), 0x01ae41), + (hex!("0100000000333333334444444455000001240000000000001250"), 0x01aee1), + (hex!("0100000000333333334444444455000001240000000000006cd0"), 0x01af81), + (hex!("0100000000333333334444444455000001250000000000001260"), 0x01b021), + (hex!("01000000003333333344444444550000012500000000000046b0"), 0x01b0c1), + (hex!("0100000000333333334444444455000001250000000000005eb0"), 0x01b161), + (hex!("0100000000333333334444444455000001260000000000001270"), 0x01b201), + (hex!("0100000000333333334444444455000001260000000000004630"), 0x01b2a1), + (hex!("0100000000333333334444444455000001270000000000001280"), 0x01b341), + (hex!("0100000000333333334444444455000001270000000000004ff0"), 0x01b3e1), + (hex!("0100000000333333334444444455000001270000000000006ec0"), 0x01b481), + (hex!("0100000000333333334444444455000001280000000000001290"), 0x01b521), + (hex!("01000000003333333344444444550000012900000000000012a0"), 0x01b5c1), + (hex!("0100000000333333334444444455000001290000000000005f60"), 0x01b661), + (hex!("01000000003333333344444444550000012a00000000000012b0"), 0x01b701), + (hex!("01000000003333333344444444550000012a0000000000005480"), 0x01b7a1), + (hex!("01000000003333333344444444550000012b00000000000012c0"), 0x01b841), + (hex!("01000000003333333344444444550000012b00000000000065a0"), 0x01b8e1), + (hex!("01000000003333333344444444550000012b00000000000066c0"), 0x01b981), + (hex!("01000000003333333344444444550000012c00000000000012d0"), 0x01ba21), + (hex!("01000000003333333344444444550000012c00000000000064b0"), 0x01bac1), + (hex!("01000000003333333344444444550000012d00000000000012e0"), 0x01bb61), + (hex!("01000000003333333344444444550000012d00000000000049c0"), 0x01bc01), + (hex!("01000000003333333344444444550000012d0000000000004bf0"), 0x01bca1), + (hex!("01000000003333333344444444550000012e00000000000012f0"), 0x01bd41), + (hex!("01000000003333333344444444550000012e0000000000005ed0"), 0x01bde1), + (hex!("01000000003333333344444444550000012f0000000000001300"), 0x01be81), + (hex!("01000000003333333344444444550000012f00000000000049a0"), 0x01bf21), + (hex!("0100000000333333334444444455000001300000000000001310"), 0x01bfc1), + (hex!("0100000000333333334444444455000001300000000000007840"), 0x01c061), + (hex!("0100000000333333334444444455000001310000000000001320"), 0x01c101), + (hex!("0100000000333333334444444455000001310000000000005f70"), 0x01c1a1), + (hex!("0100000000333333334444444455000001320000000000001330"), 0x01c241), + (hex!("0100000000333333334444444455000001320000000000005a00"), 0x01c2e1), + (hex!("0100000000333333334444444455000001330000000000001340"), 0x01c381), + (hex!("0100000000333333334444444455000001330000000000006c70"), 0x01c421), + (hex!("0100000000333333334444444455000001340000000000001350"), 0x01c4c1), + (hex!("0100000000333333334444444455000001340000000000005c60"), 0x01c561), + (hex!("0100000000333333334444444455000001350000000000001360"), 0x01c601), + (hex!("0100000000333333334444444455000001350000000000004f10"), 0x01c6a1), + (hex!("0100000000333333334444444455000001360000000000001370"), 0x01c741), + (hex!("0100000000333333334444444455000001360000000000004c60"), 0x01c7e1), + (hex!("0100000000333333334444444455000001370000000000001380"), 0x01c881), + (hex!("0100000000333333334444444455000001380000000000001390"), 0x01c921), + (hex!("01000000003333333344444444550000013900000000000013a0"), 0x01c9c1), + (hex!("0100000000333333334444444455000001390000000000004ea0"), 0x01ca61), + (hex!("01000000003333333344444444550000013a00000000000013b0"), 0x01cb01), + (hex!("01000000003333333344444444550000013a0000000000007350"), 0x01cba1), + (hex!("01000000003333333344444444550000013b00000000000013c0"), 0x01cc41), + (hex!("01000000003333333344444444550000013c00000000000013d0"), 0x01cce1), + (hex!("01000000003333333344444444550000013c0000000000007050"), 0x01cd81), + (hex!("01000000003333333344444444550000013d00000000000013e0"), 0x01ce21), + (hex!("01000000003333333344444444550000013d0000000000006bd0"), 0x01cec1), + (hex!("01000000003333333344444444550000013e00000000000013f0"), 0x01cf61), + (hex!("01000000003333333344444444550000013e00000000000058e0"), 0x01d001), + (hex!("01000000003333333344444444550000013f0000000000001400"), 0x01d0a1), + (hex!("01000000003333333344444444550000013f0000000000004740"), 0x01d141), + (hex!("0100000000333333334444444455000001400000000000001410"), 0x01d1e1), + (hex!("0100000000333333334444444455000001400000000000003f10"), 0x01d281), + (hex!("0100000000333333334444444455000001400000000000006d40"), 0x01d321), + (hex!("01000000003333333344444444550000014000000000000072d0"), 0x01d3c1), + (hex!("0100000000333333334444444455000001410000000000001420"), 0x01d461), + (hex!("0100000000333333334444444455000001420000000000001430"), 0x01d501), + (hex!("0100000000333333334444444455000001430000000000001440"), 0x01d5a1), + (hex!("0100000000333333334444444455000001440000000000001450"), 0x01d641), + (hex!("0100000000333333334444444455000001450000000000001460"), 0x01d6e1), + (hex!("0100000000333333334444444455000001460000000000001470"), 0x01d781), + (hex!("01000000003333333344444444550000014600000000000055c0"), 0x01d821), + (hex!("0100000000333333334444444455000001470000000000001480"), 0x01d8c1), + (hex!("0100000000333333334444444455000001470000000000004570"), 0x01d961), + (hex!("0100000000333333334444444455000001470000000000004be0"), 0x01da01), + (hex!("0100000000333333334444444455000001480000000000001490"), 0x01daa1), + (hex!("0100000000333333334444444455000001480000000000005360"), 0x01db41), + (hex!("01000000003333333344444444550000014900000000000014a0"), 0x01dbe1), + (hex!("01000000003333333344444444550000014a00000000000014b0"), 0x01dc81), + (hex!("01000000003333333344444444550000014a00000000000053d0"), 0x01dd21), + (hex!("01000000003333333344444444550000014b00000000000014c0"), 0x01ddc1), + (hex!("01000000003333333344444444550000014b0000000000005950"), 0x01de61), + (hex!("01000000003333333344444444550000014c00000000000014d0"), 0x01df01), + (hex!("01000000003333333344444444550000014c0000000000004f60"), 0x01dfa1), + (hex!("01000000003333333344444444550000014d00000000000014e0"), 0x01e041), + (hex!("01000000003333333344444444550000014d0000000000004520"), 0x01e0e1), + (hex!("01000000003333333344444444550000014d0000000000005200"), 0x01e181), + (hex!("01000000003333333344444444550000014e00000000000014f0"), 0x01e221), + (hex!("01000000003333333344444444550000014e0000000000005bd0"), 0x01e2c1), + (hex!("01000000003333333344444444550000014f0000000000001500"), 0x01e361), + (hex!("01000000003333333344444444550000014f00000000000060d0"), 0x01e401), + (hex!("0100000000333333334444444455000001500000000000001510"), 0x01e4a1), + (hex!("01000000003333333344444444550000015000000000000075e0"), 0x01e541), + (hex!("0100000000333333334444444455000001510000000000001520"), 0x01e5e1), + (hex!("0100000000333333334444444455000001510000000000005c00"), 0x01e681), + (hex!("0100000000333333334444444455000001510000000000006af0"), 0x01e721), + (hex!("0100000000333333334444444455000001510000000000007b80"), 0x01e7c1), + (hex!("0100000000333333334444444455000001520000000000001530"), 0x01e861), + (hex!("0100000000333333334444444455000001520000000000004c70"), 0x01e901), + (hex!("0100000000333333334444444455000001530000000000001540"), 0x01e9a1), + (hex!("0100000000333333334444444455000001540000000000001550"), 0x01ea41), + (hex!("0100000000333333334444444455000001540000000000007cd0"), 0x01eae1), + (hex!("0100000000333333334444444455000001550000000000001560"), 0x01eb81), + (hex!("0100000000333333334444444455000001550000000000004ae0"), 0x01ec21), + (hex!("01000000003333333344444444550000015500000000000068c0"), 0x01ecc1), + (hex!("0100000000333333334444444455000001560000000000001570"), 0x01ed61), + (hex!("01000000003333333344444444550000015600000000000064a0"), 0x01ee01), + (hex!("0100000000333333334444444455000001570000000000001580"), 0x01eea1), + (hex!("0100000000333333334444444455000001580000000000001590"), 0x01ef41), + (hex!("0100000000333333334444444455000001580000000000006d30"), 0x01efe1), + (hex!("01000000003333333344444444550000015800000000000074f0"), 0x01f081), + (hex!("01000000003333333344444444550000015900000000000015a0"), 0x01f121), + (hex!("01000000003333333344444444550000015900000000000053a0"), 0x01f1c1), + (hex!("01000000003333333344444444550000015900000000000055e0"), 0x01f261), + (hex!("0100000000333333334444444455000001590000000000006210"), 0x01f301), + (hex!("01000000003333333344444444550000015900000000000067c0"), 0x01f3a1), + (hex!("01000000003333333344444444550000015a00000000000015b0"), 0x01f441), + (hex!("01000000003333333344444444550000015b00000000000015c0"), 0x01f4e1), + (hex!("01000000003333333344444444550000015c00000000000015d0"), 0x01f581), + (hex!("01000000003333333344444444550000015c0000000000004d80"), 0x01f621), + (hex!("01000000003333333344444444550000015c00000000000073f0"), 0x01f6c1), + (hex!("01000000003333333344444444550000015d00000000000015e0"), 0x01f761), + (hex!("01000000003333333344444444550000015e00000000000015f0"), 0x01f801), + (hex!("01000000003333333344444444550000015e0000000000004120"), 0x01f8a1), + (hex!("01000000003333333344444444550000015e0000000000004350"), 0x01f941), + (hex!("01000000003333333344444444550000015e0000000000007c50"), 0x01f9e1), + (hex!("01000000003333333344444444550000015f0000000000001600"), 0x01fa81), + (hex!("0100000000333333334444444455000001600000000000001610"), 0x01fb21), + (hex!("0100000000333333334444444455000001600000000000004840"), 0x01fbc1), + (hex!("0100000000333333334444444455000001600000000000004b10"), 0x01fc61), + (hex!("0100000000333333334444444455000001600000000000007060"), 0x01fd01), + (hex!("0100000000333333334444444455000001610000000000001620"), 0x01fda1), + (hex!("0100000000333333334444444455000001610000000000005300"), 0x01fe41), + (hex!("0100000000333333334444444455000001620000000000001630"), 0x01fee1), + (hex!("0100000000333333334444444455000001620000000000006530"), 0x01ff81), + (hex!("0100000000333333334444444455000001630000000000001640"), 0x020021), + (hex!("0100000000333333334444444455000001640000000000001650"), 0x0200c1), + (hex!("0100000000333333334444444455000001650000000000001660"), 0x020161), + (hex!("0100000000333333334444444455000001660000000000001670"), 0x020201), + (hex!("0100000000333333334444444455000001670000000000001680"), 0x0202a1), + (hex!("0100000000333333334444444455000001670000000000007310"), 0x020341), + (hex!("0100000000333333334444444455000001680000000000001690"), 0x0203e1), + (hex!("0100000000333333334444444455000001680000000000007b50"), 0x020481), + (hex!("01000000003333333344444444550000016900000000000016a0"), 0x020521), + (hex!("01000000003333333344444444550000016900000000000049d0"), 0x0205c1), + (hex!("01000000003333333344444444550000016a00000000000016b0"), 0x020661), + (hex!("01000000003333333344444444550000016a00000000000078b0"), 0x020701), + (hex!("01000000003333333344444444550000016b00000000000016c0"), 0x0207a1), + (hex!("01000000003333333344444444550000016b0000000000004100"), 0x020841), + (hex!("01000000003333333344444444550000016c00000000000016d0"), 0x0208e1), + (hex!("01000000003333333344444444550000016c0000000000006e00"), 0x020981), + (hex!("01000000003333333344444444550000016d00000000000016e0"), 0x020a21), + (hex!("01000000003333333344444444550000016e00000000000016f0"), 0x020ac1), + (hex!("01000000003333333344444444550000016e0000000000004ac0"), 0x020b61), + (hex!("01000000003333333344444444550000016e0000000000007820"), 0x020c01), + (hex!("01000000003333333344444444550000016f0000000000001700"), 0x020ca1), + (hex!("0100000000333333334444444455000001700000000000001710"), 0x020d41), + (hex!("0100000000333333334444444455000001700000000000005830"), 0x020de1), + (hex!("0100000000333333334444444455000001710000000000001720"), 0x020e81), + (hex!("01000000003333333344444444550000017100000000000072f0"), 0x020f21), + (hex!("0100000000333333334444444455000001720000000000001730"), 0x020fc1), + (hex!("0100000000333333334444444455000001720000000000004870"), 0x021061), + (hex!("01000000003333333344444444550000017200000000000070b0"), 0x021101), + (hex!("0100000000333333334444444455000001730000000000001740"), 0x0211a1), + (hex!("0100000000333333334444444455000001740000000000001750"), 0x021241), + (hex!("0100000000333333334444444455000001750000000000001760"), 0x0212e1), + (hex!("0100000000333333334444444455000001750000000000005670"), 0x021381), + (hex!("0100000000333333334444444455000001750000000000005870"), 0x021421), + (hex!("0100000000333333334444444455000001760000000000001770"), 0x0214c1), + (hex!("0100000000333333334444444455000001770000000000001780"), 0x021561), + (hex!("0100000000333333334444444455000001770000000000005000"), 0x021601), + (hex!("0100000000333333334444444455000001770000000000007090"), 0x0216a1), + (hex!("0100000000333333334444444455000001780000000000001790"), 0x021741), + (hex!("01000000003333333344444444550000017800000000000048a0"), 0x0217e1), + (hex!("0100000000333333334444444455000001780000000000006bf0"), 0x021881), + (hex!("01000000003333333344444444550000017900000000000017a0"), 0x021921), + (hex!("01000000003333333344444444550000017900000000000057d0"), 0x0219c1), + (hex!("0100000000333333334444444455000001790000000000006660"), 0x021a61), + (hex!("01000000003333333344444444550000017a00000000000017b0"), 0x021b01), + (hex!("01000000003333333344444444550000017a0000000000004970"), 0x021ba1), + (hex!("01000000003333333344444444550000017a0000000000005dc0"), 0x021c41), + (hex!("01000000003333333344444444550000017b00000000000017c0"), 0x021ce1), + (hex!("01000000003333333344444444550000017b0000000000004ee0"), 0x021d81), + (hex!("01000000003333333344444444550000017b00000000000054c0"), 0x021e21), + (hex!("01000000003333333344444444550000017c00000000000017d0"), 0x021ec1), + (hex!("01000000003333333344444444550000017c0000000000003fc0"), 0x021f61), + (hex!("01000000003333333344444444550000017c00000000000063e0"), 0x022001), + (hex!("01000000003333333344444444550000017c0000000000006520"), 0x0220a1), + (hex!("01000000003333333344444444550000017d00000000000017e0"), 0x022141), + (hex!("01000000003333333344444444550000017d0000000000006220"), 0x0221e1), + (hex!("01000000003333333344444444550000017d0000000000007120"), 0x022281), + (hex!("01000000003333333344444444550000017e00000000000017f0"), 0x022321), + (hex!("01000000003333333344444444550000017f0000000000001800"), 0x0223c1), + (hex!("0100000000333333334444444455000001800000000000001810"), 0x022461), + (hex!("0100000000333333334444444455000001810000000000001820"), 0x022501), + (hex!("01000000003333333344444444550000018100000000000041f0"), 0x0225a1), + (hex!("0100000000333333334444444455000001810000000000007590"), 0x022641), + (hex!("0100000000333333334444444455000001820000000000001830"), 0x0226e1), + (hex!("0100000000333333334444444455000001820000000000004ce0"), 0x022781), + (hex!("0100000000333333334444444455000001830000000000001840"), 0x022821), + (hex!("01000000003333333344444444550000018300000000000042c0"), 0x0228c1), + (hex!("0100000000333333334444444455000001840000000000001850"), 0x022961), + (hex!("0100000000333333334444444455000001840000000000004f70"), 0x022a01), + (hex!("0100000000333333334444444455000001850000000000001860"), 0x022aa1), + (hex!("0100000000333333334444444455000001850000000000006470"), 0x022b41), + (hex!("0100000000333333334444444455000001850000000000007500"), 0x022be1), + (hex!("0100000000333333334444444455000001860000000000001870"), 0x022c81), + (hex!("0100000000333333334444444455000001860000000000004770"), 0x022d21), + (hex!("0100000000333333334444444455000001870000000000001880"), 0x022dc1), + (hex!("0100000000333333334444444455000001870000000000006a30"), 0x022e61), + (hex!("0100000000333333334444444455000001880000000000001890"), 0x022f01), + (hex!("0100000000333333334444444455000001880000000000007410"), 0x022fa1), + (hex!("01000000003333333344444444550000018900000000000018a0"), 0x023041), + (hex!("01000000003333333344444444550000018900000000000044d0"), 0x0230e1), + (hex!("0100000000333333334444444455000001890000000000005ac0"), 0x023181), + (hex!("01000000003333333344444444550000018a00000000000018b0"), 0x023221), + (hex!("01000000003333333344444444550000018a0000000000006260"), 0x0232c1), + (hex!("01000000003333333344444444550000018a0000000000006d70"), 0x023361), + (hex!("01000000003333333344444444550000018b00000000000018c0"), 0x023401), + (hex!("01000000003333333344444444550000018b0000000000004aa0"), 0x0234a1), + (hex!("01000000003333333344444444550000018b0000000000006fd0"), 0x023541), + (hex!("01000000003333333344444444550000018c00000000000018d0"), 0x0235e1), + (hex!("01000000003333333344444444550000018c00000000000051b0"), 0x023681), + (hex!("01000000003333333344444444550000018c0000000000006650"), 0x023721), + (hex!("01000000003333333344444444550000018d00000000000018e0"), 0x0237c1), + (hex!("01000000003333333344444444550000018e00000000000018f0"), 0x023861), + (hex!("01000000003333333344444444550000018e00000000000041d0"), 0x023901), + (hex!("01000000003333333344444444550000018f0000000000001900"), 0x0239a1), + (hex!("01000000003333333344444444550000018f0000000000007600"), 0x023a41), + (hex!("0100000000333333334444444455000001900000000000001910"), 0x023ae1), + (hex!("0100000000333333334444444455000001900000000000005410"), 0x023b81), + (hex!("0100000000333333334444444455000001900000000000006760"), 0x023c21), + (hex!("0100000000333333334444444455000001910000000000001920"), 0x023cc1), + (hex!("0100000000333333334444444455000001920000000000001930"), 0x023d61), + (hex!("0100000000333333334444444455000001920000000000004ca0"), 0x023e01), + (hex!("0100000000333333334444444455000001920000000000005d80"), 0x023ea1), + (hex!("0100000000333333334444444455000001920000000000005fd0"), 0x023f41), + (hex!("01000000003333333344444444550000019200000000000070d0"), 0x023fe1), + (hex!("0100000000333333334444444455000001930000000000001940"), 0x024081), + (hex!("0100000000333333334444444455000001930000000000004010"), 0x024121), + (hex!("0100000000333333334444444455000001930000000000007ca0"), 0x0241c1), + (hex!("0100000000333333334444444455000001940000000000001950"), 0x024261), + (hex!("0100000000333333334444444455000001950000000000001960"), 0x024301), + (hex!("0100000000333333334444444455000001950000000000005380"), 0x0243a1), + (hex!("0100000000333333334444444455000001960000000000001970"), 0x024441), + (hex!("0100000000333333334444444455000001960000000000006de0"), 0x0244e1), + (hex!("0100000000333333334444444455000001970000000000001980"), 0x024581), + (hex!("01000000003333333344444444550000019700000000000048f0"), 0x024621), + (hex!("0100000000333333334444444455000001980000000000001990"), 0x0246c1), + (hex!("0100000000333333334444444455000001980000000000006510"), 0x024761), + (hex!("01000000003333333344444444550000019900000000000019a0"), 0x024801), + (hex!("0100000000333333334444444455000001990000000000007570"), 0x0248a1), + (hex!("0100000000333333334444444455000001990000000000007580"), 0x024941), + (hex!("01000000003333333344444444550000019a00000000000019b0"), 0x0249e1), + (hex!("01000000003333333344444444550000019a0000000000004050"), 0x024a81), + (hex!("01000000003333333344444444550000019a0000000000004ba0"), 0x024b21), + (hex!("01000000003333333344444444550000019a0000000000005540"), 0x024bc1), + (hex!("01000000003333333344444444550000019a00000000000061c0"), 0x024c61), + (hex!("01000000003333333344444444550000019a0000000000007c60"), 0x024d01), + (hex!("01000000003333333344444444550000019b00000000000019c0"), 0x024da1), + (hex!("01000000003333333344444444550000019b0000000000006240"), 0x024e41), + (hex!("01000000003333333344444444550000019c00000000000019d0"), 0x024ee1), + (hex!("01000000003333333344444444550000019d00000000000019e0"), 0x024f81), + (hex!("01000000003333333344444444550000019d0000000000004640"), 0x025021), + (hex!("01000000003333333344444444550000019d00000000000052a0"), 0x0250c1), + (hex!("01000000003333333344444444550000019d00000000000052b0"), 0x025161), + (hex!("01000000003333333344444444550000019e00000000000019f0"), 0x025201), + (hex!("01000000003333333344444444550000019f0000000000001a00"), 0x0252a1), + (hex!("01000000003333333344444444550000019f0000000000006b20"), 0x025341), + (hex!("0100000000333333334444444455000001a00000000000001a10"), 0x0253e1), + (hex!("0100000000333333334444444455000001a10000000000001a20"), 0x025481), + (hex!("0100000000333333334444444455000001a10000000000005460"), 0x025521), + (hex!("0100000000333333334444444455000001a10000000000005d20"), 0x0255c1), + (hex!("0100000000333333334444444455000001a100000000000068f0"), 0x025661), + (hex!("0100000000333333334444444455000001a20000000000001a30"), 0x025701), + (hex!("0100000000333333334444444455000001a20000000000007170"), 0x0257a1), + (hex!("0100000000333333334444444455000001a30000000000001a40"), 0x025841), + (hex!("0100000000333333334444444455000001a40000000000001a50"), 0x0258e1), + (hex!("0100000000333333334444444455000001a50000000000001a60"), 0x025981), + (hex!("0100000000333333334444444455000001a60000000000001a70"), 0x025a21), + (hex!("0100000000333333334444444455000001a70000000000001a80"), 0x025ac1), + (hex!("0100000000333333334444444455000001a70000000000005a90"), 0x025b61), + (hex!("0100000000333333334444444455000001a70000000000006440"), 0x025c01), + (hex!("0100000000333333334444444455000001a80000000000001a90"), 0x025ca1), + (hex!("0100000000333333334444444455000001a80000000000004800"), 0x025d41), + (hex!("0100000000333333334444444455000001a90000000000001aa0"), 0x025de1), + (hex!("0100000000333333334444444455000001aa0000000000001ab0"), 0x025e81), + (hex!("0100000000333333334444444455000001aa0000000000005b60"), 0x025f21), + (hex!("0100000000333333334444444455000001ab0000000000001ac0"), 0x025fc1), + (hex!("0100000000333333334444444455000001ab0000000000006700"), 0x026061), + (hex!("0100000000333333334444444455000001ab00000000000071d0"), 0x026101), + (hex!("0100000000333333334444444455000001ac0000000000001ad0"), 0x0261a1), + (hex!("0100000000333333334444444455000001ac0000000000007380"), 0x026241), + (hex!("0100000000333333334444444455000001ad0000000000001ae0"), 0x0262e1), + (hex!("0100000000333333334444444455000001ad0000000000006350"), 0x026381), + (hex!("0100000000333333334444444455000001ae0000000000001af0"), 0x026421), + (hex!("0100000000333333334444444455000001af0000000000001b00"), 0x0264c1), + (hex!("0100000000333333334444444455000001af0000000000007390"), 0x026561), + (hex!("0100000000333333334444444455000001b00000000000001b10"), 0x026601), + (hex!("0100000000333333334444444455000001b10000000000001b20"), 0x0266a1), + (hex!("0100000000333333334444444455000001b10000000000005cc0"), 0x026741), + (hex!("0100000000333333334444444455000001b20000000000001b30"), 0x0267e1), + (hex!("0100000000333333334444444455000001b20000000000004fb0"), 0x026881), + (hex!("0100000000333333334444444455000001b30000000000001b40"), 0x026921), + (hex!("0100000000333333334444444455000001b40000000000001b50"), 0x0269c1), + (hex!("0100000000333333334444444455000001b50000000000001b60"), 0x026a61), + (hex!("0100000000333333334444444455000001b60000000000001b70"), 0x026b01), + (hex!("0100000000333333334444444455000001b600000000000048e0"), 0x026ba1), + (hex!("0100000000333333334444444455000001b70000000000001b80"), 0x026c41), + (hex!("0100000000333333334444444455000001b70000000000005ca0"), 0x026ce1), + (hex!("0100000000333333334444444455000001b70000000000007900"), 0x026d81), + (hex!("0100000000333333334444444455000001b80000000000001b90"), 0x026e21), + (hex!("0100000000333333334444444455000001b80000000000004d90"), 0x026ec1), + (hex!("0100000000333333334444444455000001b90000000000001ba0"), 0x026f61), + (hex!("0100000000333333334444444455000001b90000000000003f40"), 0x027001), + (hex!("0100000000333333334444444455000001ba0000000000001bb0"), 0x0270a1), + (hex!("0100000000333333334444444455000001ba00000000000042a0"), 0x027141), + (hex!("0100000000333333334444444455000001ba00000000000067f0"), 0x0271e1), + (hex!("0100000000333333334444444455000001ba00000000000073a0"), 0x027281), + (hex!("0100000000333333334444444455000001bb0000000000001bc0"), 0x027321), + (hex!("0100000000333333334444444455000001bb0000000000004a00"), 0x0273c1), + (hex!("0100000000333333334444444455000001bb0000000000005e00"), 0x027461), + (hex!("0100000000333333334444444455000001bc0000000000001bd0"), 0x027501), + (hex!("0100000000333333334444444455000001bc0000000000004230"), 0x0275a1), + (hex!("0100000000333333334444444455000001bc0000000000005860"), 0x027641), + (hex!("0100000000333333334444444455000001bd0000000000001be0"), 0x0276e1), + (hex!("0100000000333333334444444455000001bd0000000000007c70"), 0x027781), + (hex!("0100000000333333334444444455000001be0000000000001bf0"), 0x027821), + (hex!("0100000000333333334444444455000001be0000000000007770"), 0x0278c1), + (hex!("0100000000333333334444444455000001be0000000000007cf0"), 0x027961), + (hex!("0100000000333333334444444455000001bf0000000000001c00"), 0x027a01), + (hex!("0100000000333333334444444455000001bf0000000000006490"), 0x027aa1), + (hex!("0100000000333333334444444455000001c00000000000001c10"), 0x027b41), + (hex!("0100000000333333334444444455000001c10000000000001c20"), 0x027be1), + (hex!("0100000000333333334444444455000001c10000000000004600"), 0x027c81), + (hex!("0100000000333333334444444455000001c20000000000001c30"), 0x027d21), + (hex!("0100000000333333334444444455000001c20000000000006e30"), 0x027dc1), + (hex!("0100000000333333334444444455000001c30000000000001c40"), 0x027e61), + (hex!("0100000000333333334444444455000001c40000000000001c50"), 0x027f01), + (hex!("0100000000333333334444444455000001c50000000000001c60"), 0x027fa1), + (hex!("0100000000333333334444444455000001c60000000000001c70"), 0x028041), + (hex!("0100000000333333334444444455000001c60000000000004240"), 0x0280e1), + (hex!("0100000000333333334444444455000001c60000000000005bb0"), 0x028181), + (hex!("0100000000333333334444444455000001c70000000000001c80"), 0x028221), + (hex!("0100000000333333334444444455000001c80000000000001c90"), 0x0282c1), + (hex!("0100000000333333334444444455000001c90000000000001ca0"), 0x028361), + (hex!("0100000000333333334444444455000001c90000000000006730"), 0x028401), + (hex!("0100000000333333334444444455000001ca0000000000001cb0"), 0x0284a1), + (hex!("0100000000333333334444444455000001ca00000000000070f0"), 0x028541), + (hex!("0100000000333333334444444455000001cb0000000000001cc0"), 0x0285e1), + (hex!("0100000000333333334444444455000001cb00000000000071a0"), 0x028681), + (hex!("0100000000333333334444444455000001cc0000000000001cd0"), 0x028721), + (hex!("0100000000333333334444444455000001cc0000000000005280"), 0x0287c1), + (hex!("0100000000333333334444444455000001cc0000000000005d90"), 0x028861), + (hex!("0100000000333333334444444455000001cd0000000000001ce0"), 0x028901), + (hex!("0100000000333333334444444455000001cd00000000000069b0"), 0x0289a1), + (hex!("0100000000333333334444444455000001ce0000000000001cf0"), 0x028a41), + (hex!("0100000000333333334444444455000001ce0000000000004540"), 0x028ae1), + (hex!("0100000000333333334444444455000001cf0000000000001d00"), 0x028b81), + (hex!("0100000000333333334444444455000001cf00000000000076a0"), 0x028c21), + (hex!("0100000000333333334444444455000001d00000000000001d10"), 0x028cc1), + (hex!("0100000000333333334444444455000001d000000000000060a0"), 0x028d61), + (hex!("0100000000333333334444444455000001d10000000000001d20"), 0x028e01), + (hex!("0100000000333333334444444455000001d20000000000001d30"), 0x028ea1), + (hex!("0100000000333333334444444455000001d30000000000001d40"), 0x028f41), + (hex!("0100000000333333334444444455000001d30000000000004000"), 0x028fe1), + (hex!("0100000000333333334444444455000001d30000000000004140"), 0x029081), + (hex!("0100000000333333334444444455000001d30000000000006790"), 0x029121), + (hex!("0100000000333333334444444455000001d40000000000001d50"), 0x0291c1), + (hex!("0100000000333333334444444455000001d50000000000001d60"), 0x029261), + (hex!("0100000000333333334444444455000001d60000000000001d70"), 0x029301), + (hex!("0100000000333333334444444455000001d60000000000004b50"), 0x0293a1), + (hex!("0100000000333333334444444455000001d60000000000007430"), 0x029441), + (hex!("0100000000333333334444444455000001d70000000000001d80"), 0x0294e1), + (hex!("0100000000333333334444444455000001d70000000000006920"), 0x029581), + (hex!("0100000000333333334444444455000001d80000000000001d90"), 0x029621), + (hex!("0100000000333333334444444455000001d80000000000005b30"), 0x0296c1), + (hex!("0100000000333333334444444455000001d90000000000001da0"), 0x029761), + (hex!("0100000000333333334444444455000001da0000000000001db0"), 0x029801), + (hex!("0100000000333333334444444455000001da0000000000004af0"), 0x0298a1), + (hex!("0100000000333333334444444455000001da0000000000007240"), 0x029941), + (hex!("0100000000333333334444444455000001da0000000000007470"), 0x0299e1), + (hex!("0100000000333333334444444455000001db0000000000001dc0"), 0x029a81), + (hex!("0100000000333333334444444455000001db00000000000045d0"), 0x029b21), + (hex!("0100000000333333334444444455000001dc0000000000001dd0"), 0x029bc1), + (hex!("0100000000333333334444444455000001dd0000000000001de0"), 0x029c61), + (hex!("0100000000333333334444444455000001dd0000000000004bb0"), 0x029d01), + (hex!("0100000000333333334444444455000001dd0000000000004cd0"), 0x029da1), + (hex!("0100000000333333334444444455000001dd0000000000006100"), 0x029e41), + (hex!("0100000000333333334444444455000001dd0000000000007bb0"), 0x029ee1), + (hex!("0100000000333333334444444455000001de0000000000001df0"), 0x029f81), + (hex!("0100000000333333334444444455000001de0000000000004260"), 0x02a021), + (hex!("0100000000333333334444444455000001de0000000000006040"), 0x02a0c1), + (hex!("0100000000333333334444444455000001df0000000000001e00"), 0x02a161), + (hex!("0100000000333333334444444455000001df0000000000005fa0"), 0x02a201), + (hex!("0100000000333333334444444455000001df0000000000006a70"), 0x02a2a1), + (hex!("0100000000333333334444444455000001df0000000000006dc0"), 0x02a341), + (hex!("0100000000333333334444444455000001e00000000000001e10"), 0x02a3e1), + (hex!("0100000000333333334444444455000001e00000000000007010"), 0x02a481), + (hex!("0100000000333333334444444455000001e10000000000001e20"), 0x02a521), + (hex!("0100000000333333334444444455000001e10000000000005720"), 0x02a5c1), + (hex!("0100000000333333334444444455000001e10000000000006830"), 0x02a661), + (hex!("0100000000333333334444444455000001e20000000000001e30"), 0x02a701), + (hex!("0100000000333333334444444455000001e20000000000005100"), 0x02a7a1), + (hex!("0100000000333333334444444455000001e30000000000001e40"), 0x02a841), + (hex!("0100000000333333334444444455000001e40000000000001e50"), 0x02a8e1), + (hex!("0100000000333333334444444455000001e40000000000003f30"), 0x02a981), + (hex!("0100000000333333334444444455000001e40000000000005220"), 0x02aa21), + (hex!("0100000000333333334444444455000001e50000000000001e60"), 0x02aac1), + (hex!("0100000000333333334444444455000001e50000000000006f60"), 0x02ab61), + (hex!("0100000000333333334444444455000001e60000000000001e70"), 0x02ac01), + (hex!("0100000000333333334444444455000001e60000000000006c80"), 0x02aca1), + (hex!("0100000000333333334444444455000001e70000000000001e80"), 0x02ad41), + (hex!("0100000000333333334444444455000001e80000000000001e90"), 0x02ade1), + (hex!("0100000000333333334444444455000001e80000000000004e30"), 0x02ae81), + (hex!("0100000000333333334444444455000001e90000000000001ea0"), 0x02af21), + (hex!("0100000000333333334444444455000001e90000000000005470"), 0x02afc1), + (hex!("0100000000333333334444444455000001ea0000000000001eb0"), 0x02b061), + (hex!("0100000000333333334444444455000001ea0000000000007980"), 0x02b101), + (hex!("0100000000333333334444444455000001eb0000000000001ec0"), 0x02b1a1), + (hex!("0100000000333333334444444455000001eb0000000000004390"), 0x02b241), + (hex!("0100000000333333334444444455000001eb0000000000005970"), 0x02b2e1), + (hex!("0100000000333333334444444455000001ec0000000000001ed0"), 0x02b381), + (hex!("0100000000333333334444444455000001ec0000000000005d50"), 0x02b421), + (hex!("0100000000333333334444444455000001ec00000000000076e0"), 0x02b4c1), + (hex!("0100000000333333334444444455000001ed0000000000001ee0"), 0x02b561), + (hex!("0100000000333333334444444455000001ed0000000000006190"), 0x02b601), + (hex!("0100000000333333334444444455000001ee0000000000001ef0"), 0x02b6a1), + (hex!("0100000000333333334444444455000001ee0000000000004900"), 0x02b741), + (hex!("0100000000333333334444444455000001ef0000000000001f00"), 0x02b7e1), + (hex!("0100000000333333334444444455000001ef0000000000006c60"), 0x02b881), + (hex!("0100000000333333334444444455000001f00000000000001f10"), 0x02b921), + (hex!("0100000000333333334444444455000001f00000000000006950"), 0x02b9c1), + (hex!("0100000000333333334444444455000001f10000000000001f20"), 0x02ba61), + (hex!("0100000000333333334444444455000001f10000000000006400"), 0x02bb01), + (hex!("0100000000333333334444444455000001f20000000000001f30"), 0x02bba1), + (hex!("0100000000333333334444444455000001f20000000000006f00"), 0x02bc41), + (hex!("0100000000333333334444444455000001f20000000000007b10"), 0x02bce1), + (hex!("0100000000333333334444444455000001f30000000000001f40"), 0x02bd81), + (hex!("0100000000333333334444444455000001f40000000000001f50"), 0x02be21), + (hex!("0100000000333333334444444455000001f50000000000001f60"), 0x02bec1), + (hex!("0100000000333333334444444455000001f500000000000044f0"), 0x02bf61), + (hex!("0100000000333333334444444455000001f60000000000001f70"), 0x02c001), + (hex!("0100000000333333334444444455000001f70000000000001f80"), 0x02c0a1), + (hex!("0100000000333333334444444455000001f70000000000004ad0"), 0x02c141), + (hex!("0100000000333333334444444455000001f80000000000001f90"), 0x02c1e1), + (hex!("0100000000333333334444444455000001f90000000000001fa0"), 0x02c281), + (hex!("0100000000333333334444444455000001f90000000000003f60"), 0x02c321), + (hex!("0100000000333333334444444455000001f90000000000004a80"), 0x02c3c1), + (hex!("0100000000333333334444444455000001fa0000000000001fb0"), 0x02c461), + (hex!("0100000000333333334444444455000001fa0000000000006f90"), 0x02c501), + (hex!("0100000000333333334444444455000001fb0000000000001fc0"), 0x02c5a1), + (hex!("0100000000333333334444444455000001fc0000000000001fd0"), 0x02c641), + (hex!("0100000000333333334444444455000001fc0000000000004a90"), 0x02c6e1), + (hex!("0100000000333333334444444455000001fd0000000000001fe0"), 0x02c781), + (hex!("0100000000333333334444444455000001fd0000000000005f50"), 0x02c821), + (hex!("0100000000333333334444444455000001fe0000000000001ff0"), 0x02c8c1), + (hex!("0100000000333333334444444455000001ff0000000000002000"), 0x02c961), + (hex!("0100000000333333334444444455000002000000000000002010"), 0x02ca01), + (hex!("0100000000333333334444444455000002000000000000005f00"), 0x02caa1), + (hex!("0100000000333333334444444455000002000000000000006840"), 0x02cb41), + (hex!("0100000000333333334444444455000002010000000000002020"), 0x02cbe1), + (hex!("0100000000333333334444444455000002020000000000002030"), 0x02cc81), + (hex!("0100000000333333334444444455000002030000000000002040"), 0x02cd21), + (hex!("0100000000333333334444444455000002040000000000002050"), 0x02cdc1), + (hex!("01000000003333333344444444550000020400000000000051f0"), 0x02ce61), + (hex!("0100000000333333334444444455000002050000000000002060"), 0x02cf01), + (hex!("0100000000333333334444444455000002060000000000002070"), 0x02cfa1), + (hex!("0100000000333333334444444455000002060000000000005c80"), 0x02d041), + (hex!("01000000003333333344444444550000020600000000000061d0"), 0x02d0e1), + (hex!("01000000003333333344444444550000020600000000000078c0"), 0x02d181), + (hex!("0100000000333333334444444455000002070000000000002080"), 0x02d221), + (hex!("0100000000333333334444444455000002070000000000006ba0"), 0x02d2c1), + (hex!("0100000000333333334444444455000002080000000000002090"), 0x02d361), + (hex!("01000000003333333344444444550000020900000000000020a0"), 0x02d401), + (hex!("01000000003333333344444444550000020900000000000067a0"), 0x02d4a1), + (hex!("01000000003333333344444444550000020a00000000000020b0"), 0x02d541), + (hex!("01000000003333333344444444550000020a0000000000004950"), 0x02d5e1), + (hex!("01000000003333333344444444550000020a0000000000004de0"), 0x02d681), + (hex!("01000000003333333344444444550000020b00000000000020c0"), 0x02d721), + (hex!("01000000003333333344444444550000020b0000000000004b00"), 0x02d7c1), + (hex!("01000000003333333344444444550000020c00000000000020d0"), 0x02d861), + (hex!("01000000003333333344444444550000020d00000000000020e0"), 0x02d901), + (hex!("01000000003333333344444444550000020e00000000000020f0"), 0x02d9a1), + (hex!("01000000003333333344444444550000020f0000000000002100"), 0x02da41), + (hex!("0100000000333333334444444455000002100000000000002110"), 0x02dae1), + (hex!("0100000000333333334444444455000002110000000000002120"), 0x02db81), + (hex!("0100000000333333334444444455000002110000000000004490"), 0x02dc21), + (hex!("0100000000333333334444444455000002120000000000002130"), 0x02dcc1), + (hex!("0100000000333333334444444455000002130000000000002140"), 0x02dd61), + (hex!("01000000003333333344444444550000021300000000000046d0"), 0x02de01), + (hex!("01000000003333333344444444550000021300000000000046e0"), 0x02dea1), + (hex!("0100000000333333334444444455000002130000000000004b70"), 0x02df41), + (hex!("0100000000333333334444444455000002140000000000002150"), 0x02dfe1), + (hex!("0100000000333333334444444455000002140000000000006c50"), 0x02e081), + (hex!("0100000000333333334444444455000002150000000000002160"), 0x02e121), + (hex!("01000000003333333344444444550000021500000000000043c0"), 0x02e1c1), + (hex!("0100000000333333334444444455000002160000000000002170"), 0x02e261), + (hex!("01000000003333333344444444550000021600000000000055b0"), 0x02e301), + (hex!("0100000000333333334444444455000002160000000000006150"), 0x02e3a1), + (hex!("0100000000333333334444444455000002170000000000002180"), 0x02e441), + (hex!("01000000003333333344444444550000021700000000000053b0"), 0x02e4e1), + (hex!("0100000000333333334444444455000002170000000000007460"), 0x02e581), + (hex!("0100000000333333334444444455000002180000000000002190"), 0x02e621), + (hex!("01000000003333333344444444550000021900000000000021a0"), 0x02e6c1), + (hex!("01000000003333333344444444550000021a00000000000021b0"), 0x02e761), + (hex!("01000000003333333344444444550000021a0000000000007650"), 0x02e801), + (hex!("01000000003333333344444444550000021b00000000000021c0"), 0x02e8a1), + (hex!("01000000003333333344444444550000021b0000000000004b20"), 0x02e941), + (hex!("01000000003333333344444444550000021c00000000000021d0"), 0x02e9e1), + (hex!("01000000003333333344444444550000021c0000000000007610"), 0x02ea81), + (hex!("01000000003333333344444444550000021d00000000000021e0"), 0x02eb21), + (hex!("01000000003333333344444444550000021d0000000000005f40"), 0x02ebc1), + (hex!("01000000003333333344444444550000021e00000000000021f0"), 0x02ec61), + (hex!("01000000003333333344444444550000021e0000000000005a50"), 0x02ed01), + (hex!("01000000003333333344444444550000021e0000000000005ff0"), 0x02eda1), + (hex!("01000000003333333344444444550000021f0000000000002200"), 0x02ee41), + (hex!("01000000003333333344444444550000021f00000000000043a0"), 0x02eee1), + (hex!("01000000003333333344444444550000021f0000000000004cb0"), 0x02ef81), + (hex!("01000000003333333344444444550000021f0000000000004e00"), 0x02f021), + (hex!("0100000000333333334444444455000002200000000000002210"), 0x02f0c1), + (hex!("0100000000333333334444444455000002210000000000002220"), 0x02f161), + (hex!("0100000000333333334444444455000002210000000000006290"), 0x02f201), + (hex!("0100000000333333334444444455000002210000000000007230"), 0x02f2a1), + (hex!("0100000000333333334444444455000002220000000000002230"), 0x02f341), + (hex!("0100000000333333334444444455000002220000000000006ea0"), 0x02f3e1), + (hex!("0100000000333333334444444455000002230000000000002240"), 0x02f481), + (hex!("0100000000333333334444444455000002230000000000004710"), 0x02f521), + (hex!("0100000000333333334444444455000002240000000000002250"), 0x02f5c1), + (hex!("0100000000333333334444444455000002250000000000002260"), 0x02f661), + (hex!("0100000000333333334444444455000002260000000000002270"), 0x02f701), + (hex!("0100000000333333334444444455000002260000000000005b40"), 0x02f7a1), + (hex!("0100000000333333334444444455000002260000000000006300"), 0x02f841), + (hex!("0100000000333333334444444455000002270000000000002280"), 0x02f8e1), + (hex!("0100000000333333334444444455000002270000000000005b80"), 0x02f981), + (hex!("0100000000333333334444444455000002280000000000002290"), 0x02fa21), + (hex!("0100000000333333334444444455000002280000000000003ed0"), 0x02fac1), + (hex!("0100000000333333334444444455000002280000000000004550"), 0x02fb61), + (hex!("01000000003333333344444444550000022800000000000077d0"), 0x02fc01), + (hex!("01000000003333333344444444550000022900000000000022a0"), 0x02fca1), + (hex!("0100000000333333334444444455000002290000000000006480"), 0x02fd41), + (hex!("01000000003333333344444444550000022a00000000000022b0"), 0x02fde1), + (hex!("01000000003333333344444444550000022a0000000000005450"), 0x02fe81), + (hex!("01000000003333333344444444550000022b00000000000022c0"), 0x02ff21), + (hex!("01000000003333333344444444550000022b0000000000006dd0"), 0x02ffc1), + (hex!("01000000003333333344444444550000022c00000000000022d0"), 0x030061), + (hex!("01000000003333333344444444550000022c0000000000006890"), 0x030101), + (hex!("01000000003333333344444444550000022d00000000000022e0"), 0x0301a1), + (hex!("01000000003333333344444444550000022e00000000000022f0"), 0x030241), + (hex!("01000000003333333344444444550000022e0000000000004f20"), 0x0302e1), + (hex!("01000000003333333344444444550000022f0000000000002300"), 0x030381), + (hex!("01000000003333333344444444550000022f0000000000005260"), 0x030421), + (hex!("01000000003333333344444444550000022f00000000000053f0"), 0x0304c1), + (hex!("0100000000333333334444444455000002300000000000002310"), 0x030561), + (hex!("01000000003333333344444444550000023000000000000050e0"), 0x030601), + (hex!("0100000000333333334444444455000002310000000000002320"), 0x0306a1), + (hex!("0100000000333333334444444455000002310000000000007800"), 0x030741), + (hex!("0100000000333333334444444455000002320000000000002330"), 0x0307e1), + (hex!("0100000000333333334444444455000002330000000000002340"), 0x030881), + (hex!("0100000000333333334444444455000002330000000000004d70"), 0x030921), + (hex!("0100000000333333334444444455000002330000000000005cf0"), 0x0309c1), + (hex!("0100000000333333334444444455000002340000000000002350"), 0x030a61), + (hex!("0100000000333333334444444455000002350000000000002360"), 0x030b01), + (hex!("0100000000333333334444444455000002350000000000006970"), 0x030ba1), + (hex!("0100000000333333334444444455000002360000000000002370"), 0x030c41), + (hex!("0100000000333333334444444455000002360000000000005270"), 0x030ce1), + (hex!("0100000000333333334444444455000002370000000000002380"), 0x030d81), + (hex!("0100000000333333334444444455000002370000000000005d70"), 0x030e21), + (hex!("0100000000333333334444444455000002380000000000002390"), 0x030ec1), + (hex!("01000000003333333344444444550000023800000000000069a0"), 0x030f61), + (hex!("01000000003333333344444444550000023900000000000023a0"), 0x031001), + (hex!("01000000003333333344444444550000023900000000000052e0"), 0x0310a1), + (hex!("0100000000333333334444444455000002390000000000005a10"), 0x031141), + (hex!("0100000000333333334444444455000002390000000000007440"), 0x0311e1), + (hex!("01000000003333333344444444550000023a00000000000023b0"), 0x031281), + (hex!("01000000003333333344444444550000023a0000000000003f00"), 0x031321), + (hex!("01000000003333333344444444550000023a0000000000004430"), 0x0313c1), + (hex!("01000000003333333344444444550000023a0000000000007070"), 0x031461), + (hex!("01000000003333333344444444550000023a00000000000074a0"), 0x031501), + (hex!("01000000003333333344444444550000023b00000000000023c0"), 0x0315a1), + (hex!("01000000003333333344444444550000023b0000000000004730"), 0x031641), + (hex!("01000000003333333344444444550000023b00000000000068b0"), 0x0316e1), + (hex!("01000000003333333344444444550000023c00000000000023d0"), 0x031781), + (hex!("01000000003333333344444444550000023c0000000000004680"), 0x031821), + (hex!("01000000003333333344444444550000023d00000000000023e0"), 0x0318c1), + (hex!("01000000003333333344444444550000023d00000000000059a0"), 0x031961), + (hex!("01000000003333333344444444550000023e00000000000023f0"), 0x031a01), + (hex!("01000000003333333344444444550000023f0000000000002400"), 0x031aa1), + (hex!("0100000000333333334444444455000002400000000000002410"), 0x031b41), + (hex!("0100000000333333334444444455000002400000000000004920"), 0x031be1), + (hex!("01000000003333333344444444550000024000000000000066e0"), 0x031c81), + (hex!("01000000003333333344444444550000024000000000000076f0"), 0x031d21), + (hex!("01000000003333333344444444550000024000000000000078e0"), 0x031dc1), + (hex!("0100000000333333334444444455000002410000000000002420"), 0x031e61), + (hex!("0100000000333333334444444455000002420000000000002430"), 0x031f01), + (hex!("0100000000333333334444444455000002420000000000006590"), 0x031fa1), + (hex!("0100000000333333334444444455000002430000000000002440"), 0x032041), + (hex!("0100000000333333334444444455000002430000000000004d00"), 0x0320e1), + (hex!("0100000000333333334444444455000002440000000000002450"), 0x032181), + (hex!("0100000000333333334444444455000002440000000000005f80"), 0x032221), + (hex!("0100000000333333334444444455000002450000000000002460"), 0x0322c1), + (hex!("0100000000333333334444444455000002450000000000004940"), 0x032361), + (hex!("0100000000333333334444444455000002460000000000002470"), 0x032401), + (hex!("0100000000333333334444444455000002470000000000002480"), 0x0324a1), + (hex!("0100000000333333334444444455000002470000000000004dd0"), 0x032541), + (hex!("0100000000333333334444444455000002470000000000005930"), 0x0325e1), + (hex!("01000000003333333344444444550000024700000000000061b0"), 0x032681), + (hex!("0100000000333333334444444455000002470000000000007740"), 0x032721), + (hex!("0100000000333333334444444455000002480000000000002490"), 0x0327c1), + (hex!("0100000000333333334444444455000002480000000000004890"), 0x032861), + (hex!("01000000003333333344444444550000024900000000000024a0"), 0x032901), + (hex!("01000000003333333344444444550000024a00000000000024b0"), 0x0329a1), + (hex!("01000000003333333344444444550000024b00000000000024c0"), 0x032a41), + (hex!("01000000003333333344444444550000024c00000000000024d0"), 0x032ae1), + (hex!("01000000003333333344444444550000024d00000000000024e0"), 0x032b81), + (hex!("01000000003333333344444444550000024d0000000000004070"), 0x032c21), + (hex!("01000000003333333344444444550000024e00000000000024f0"), 0x032cc1), + (hex!("01000000003333333344444444550000024e00000000000066a0"), 0x032d61), + (hex!("01000000003333333344444444550000024e0000000000006ab0"), 0x032e01), + (hex!("01000000003333333344444444550000024f0000000000002500"), 0x032ea1), + (hex!("0100000000333333334444444455000002500000000000002510"), 0x032f41), + (hex!("0100000000333333334444444455000002510000000000002520"), 0x032fe1), + (hex!("0100000000333333334444444455000002510000000000007320"), 0x033081), + (hex!("0100000000333333334444444455000002520000000000002530"), 0x033121), + (hex!("0100000000333333334444444455000002520000000000006410"), 0x0331c1), + (hex!("0100000000333333334444444455000002530000000000002540"), 0x033261), + (hex!("0100000000333333334444444455000002530000000000005110"), 0x033301), + (hex!("0100000000333333334444444455000002540000000000002550"), 0x0333a1), + (hex!("01000000003333333344444444550000025400000000000040c0"), 0x033441), + (hex!("0100000000333333334444444455000002540000000000006a40"), 0x0334e1), + (hex!("0100000000333333334444444455000002550000000000002560"), 0x033581), + (hex!("0100000000333333334444444455000002550000000000005190"), 0x033621), + (hex!("0100000000333333334444444455000002560000000000002570"), 0x0336c1), + (hex!("01000000003333333344444444550000025600000000000061f0"), 0x033761), + (hex!("0100000000333333334444444455000002570000000000002580"), 0x033801), + (hex!("0100000000333333334444444455000002580000000000002590"), 0x0338a1), + (hex!("01000000003333333344444444550000025800000000000043d0"), 0x033941), + (hex!("01000000003333333344444444550000025900000000000025a0"), 0x0339e1), + (hex!("0100000000333333334444444455000002590000000000006bb0"), 0x033a81), + (hex!("01000000003333333344444444550000025a00000000000025b0"), 0x033b21), + (hex!("01000000003333333344444444550000025a0000000000005fb0"), 0x033bc1), + (hex!("01000000003333333344444444550000025a00000000000064c0"), 0x033c61), + (hex!("01000000003333333344444444550000025b00000000000025c0"), 0x033d01), + (hex!("01000000003333333344444444550000025b0000000000005c10"), 0x033da1), + (hex!("01000000003333333344444444550000025c00000000000025d0"), 0x033e41), + (hex!("01000000003333333344444444550000025c0000000000007d00"), 0x033ee1), + (hex!("01000000003333333344444444550000025d00000000000025e0"), 0x033f81), + (hex!("01000000003333333344444444550000025e00000000000025f0"), 0x034021), + (hex!("01000000003333333344444444550000025e00000000000045e0"), 0x0340c1), + (hex!("01000000003333333344444444550000025e0000000000006ee0"), 0x034161), + (hex!("01000000003333333344444444550000025f0000000000002600"), 0x034201), + (hex!("01000000003333333344444444550000025f00000000000050b0"), 0x0342a1), + (hex!("01000000003333333344444444550000025f0000000000007690"), 0x034341), + (hex!("0100000000333333334444444455000002600000000000002610"), 0x0343e1), + (hex!("0100000000333333334444444455000002600000000000007b60"), 0x034481), + (hex!("0100000000333333334444444455000002610000000000002620"), 0x034521), + (hex!("0100000000333333334444444455000002620000000000002630"), 0x0345c1), + (hex!("0100000000333333334444444455000002630000000000002640"), 0x034661), + (hex!("0100000000333333334444444455000002640000000000002650"), 0x034701), + (hex!("0100000000333333334444444455000002650000000000002660"), 0x0347a1), + (hex!("0100000000333333334444444455000002650000000000006180"), 0x034841), + (hex!("0100000000333333334444444455000002660000000000002670"), 0x0348e1), + (hex!("0100000000333333334444444455000002660000000000005430"), 0x034981), + (hex!("0100000000333333334444444455000002660000000000007a60"), 0x034a21), + (hex!("0100000000333333334444444455000002670000000000002680"), 0x034ac1), + (hex!("01000000003333333344444444550000026700000000000077f0"), 0x034b61), + (hex!("0100000000333333334444444455000002680000000000002690"), 0x034c01), + (hex!("01000000003333333344444444550000026900000000000026a0"), 0x034ca1), + (hex!("01000000003333333344444444550000026a00000000000026b0"), 0x034d41), + (hex!("01000000003333333344444444550000026a0000000000007530"), 0x034de1), + (hex!("01000000003333333344444444550000026b00000000000026c0"), 0x034e81), + (hex!("01000000003333333344444444550000026b00000000000058b0"), 0x034f21), + (hex!("01000000003333333344444444550000026b00000000000066b0"), 0x034fc1), + (hex!("01000000003333333344444444550000026b0000000000006b10"), 0x035061), + (hex!("01000000003333333344444444550000026c00000000000026d0"), 0x035101), + (hex!("01000000003333333344444444550000026d00000000000026e0"), 0x0351a1), + (hex!("01000000003333333344444444550000026d0000000000004210"), 0x035241), + (hex!("01000000003333333344444444550000026d0000000000005490"), 0x0352e1), + (hex!("01000000003333333344444444550000026d0000000000005e60"), 0x035381), + (hex!("01000000003333333344444444550000026d00000000000068e0"), 0x035421), + (hex!("01000000003333333344444444550000026d0000000000007020"), 0x0354c1), + (hex!("01000000003333333344444444550000026d0000000000007300"), 0x035561), + (hex!("01000000003333333344444444550000026e00000000000026f0"), 0x035601), + (hex!("01000000003333333344444444550000026f0000000000002700"), 0x0356a1), + (hex!("01000000003333333344444444550000026f0000000000004910"), 0x035741), + (hex!("0100000000333333334444444455000002700000000000002710"), 0x0357e1), + (hex!("0100000000333333334444444455000002710000000000002720"), 0x035881), + (hex!("01000000003333333344444444550000027100000000000050c0"), 0x035921), + (hex!("0100000000333333334444444455000002720000000000002730"), 0x0359c1), + (hex!("0100000000333333334444444455000002730000000000002740"), 0x035a61), + (hex!("0100000000333333334444444455000002740000000000002750"), 0x035b01), + (hex!("0100000000333333334444444455000002740000000000007490"), 0x035ba1), + (hex!("0100000000333333334444444455000002750000000000002760"), 0x035c41), + (hex!("0100000000333333334444444455000002760000000000002770"), 0x035ce1), + (hex!("0100000000333333334444444455000002760000000000004790"), 0x035d81), + (hex!("0100000000333333334444444455000002770000000000002780"), 0x035e21), + (hex!("01000000003333333344444444550000027700000000000050a0"), 0x035ec1), + (hex!("0100000000333333334444444455000002780000000000002790"), 0x035f61), + (hex!("0100000000333333334444444455000002780000000000004330"), 0x036001), + (hex!("0100000000333333334444444455000002780000000000006b00"), 0x0360a1), + (hex!("01000000003333333344444444550000027900000000000027a0"), 0x036141), + (hex!("01000000003333333344444444550000027a00000000000027b0"), 0x0361e1), + (hex!("01000000003333333344444444550000027b00000000000027c0"), 0x036281), + (hex!("01000000003333333344444444550000027b0000000000004930"), 0x036321), + (hex!("01000000003333333344444444550000027b0000000000006250"), 0x0363c1), + (hex!("01000000003333333344444444550000027c00000000000027d0"), 0x036461), + (hex!("01000000003333333344444444550000027d00000000000027e0"), 0x036501), + (hex!("01000000003333333344444444550000027d0000000000005ce0"), 0x0365a1), + (hex!("01000000003333333344444444550000027d0000000000005fe0"), 0x036641), + (hex!("01000000003333333344444444550000027e00000000000027f0"), 0x0366e1), + (hex!("01000000003333333344444444550000027f0000000000002800"), 0x036781), + (hex!("01000000003333333344444444550000027f0000000000003e90"), 0x036821), + (hex!("01000000003333333344444444550000027f0000000000007910"), 0x0368c1), + (hex!("0100000000333333334444444455000002800000000000002810"), 0x036961), + (hex!("0100000000333333334444444455000002800000000000004990"), 0x036a01), + (hex!("0100000000333333334444444455000002800000000000006160"), 0x036aa1), + (hex!("0100000000333333334444444455000002800000000000006740"), 0x036b41), + (hex!("0100000000333333334444444455000002810000000000002820"), 0x036be1), + (hex!("0100000000333333334444444455000002820000000000002830"), 0x036c81), + (hex!("0100000000333333334444444455000002820000000000005170"), 0x036d21), + (hex!("0100000000333333334444444455000002830000000000002840"), 0x036dc1), + (hex!("0100000000333333334444444455000002840000000000002850"), 0x036e61), + (hex!("0100000000333333334444444455000002840000000000004810"), 0x036f01), + (hex!("0100000000333333334444444455000002840000000000006aa0"), 0x036fa1), + (hex!("0100000000333333334444444455000002850000000000002860"), 0x037041), + (hex!("0100000000333333334444444455000002860000000000002870"), 0x0370e1), + (hex!("0100000000333333334444444455000002860000000000005080"), 0x037181), + (hex!("0100000000333333334444444455000002870000000000002880"), 0x037221), + (hex!("0100000000333333334444444455000002870000000000004e60"), 0x0372c1), + (hex!("0100000000333333334444444455000002880000000000002890"), 0x037361), + (hex!("0100000000333333334444444455000002880000000000005060"), 0x037401), + (hex!("0100000000333333334444444455000002880000000000006f20"), 0x0374a1), + (hex!("01000000003333333344444444550000028900000000000028a0"), 0x037541), + (hex!("01000000003333333344444444550000028900000000000047e0"), 0x0375e1), + (hex!("01000000003333333344444444550000028a00000000000028b0"), 0x037681), + (hex!("01000000003333333344444444550000028a0000000000005ab0"), 0x037721), + (hex!("01000000003333333344444444550000028a0000000000007130"), 0x0377c1), + (hex!("01000000003333333344444444550000028a0000000000007660"), 0x037861), + (hex!("01000000003333333344444444550000028b00000000000028c0"), 0x037901), + (hex!("01000000003333333344444444550000028b00000000000054e0"), 0x0379a1), + (hex!("01000000003333333344444444550000028c00000000000028d0"), 0x037a41), + (hex!("01000000003333333344444444550000028c00000000000046f0"), 0x037ae1), + (hex!("01000000003333333344444444550000028c00000000000061a0"), 0x037b81), + (hex!("01000000003333333344444444550000028d00000000000028e0"), 0x037c21), + (hex!("01000000003333333344444444550000028e00000000000028f0"), 0x037cc1), + (hex!("01000000003333333344444444550000028e0000000000004130"), 0x037d61), + (hex!("01000000003333333344444444550000028f0000000000002900"), 0x037e01), + (hex!("01000000003333333344444444550000028f0000000000007510"), 0x037ea1), + (hex!("0100000000333333334444444455000002900000000000002910"), 0x037f41), + (hex!("0100000000333333334444444455000002900000000000004a40"), 0x037fe1), + (hex!("0100000000333333334444444455000002910000000000002920"), 0x038081), + (hex!("0100000000333333334444444455000002920000000000002930"), 0x038121), + (hex!("0100000000333333334444444455000002920000000000004e90"), 0x0381c1), + (hex!("0100000000333333334444444455000002930000000000002940"), 0x038261), + (hex!("0100000000333333334444444455000002930000000000006880"), 0x038301), + (hex!("0100000000333333334444444455000002940000000000002950"), 0x0383a1), + (hex!("0100000000333333334444444455000002940000000000007bc0"), 0x038441), + (hex!("0100000000333333334444444455000002950000000000002960"), 0x0384e1), + (hex!("0100000000333333334444444455000002960000000000002970"), 0x038581), + (hex!("01000000003333333344444444550000029600000000000059d0"), 0x038621), + (hex!("0100000000333333334444444455000002970000000000002980"), 0x0386c1), + (hex!("0100000000333333334444444455000002970000000000004a50"), 0x038761), + (hex!("0100000000333333334444444455000002970000000000005f20"), 0x038801), + (hex!("01000000003333333344444444550000029700000000000068d0"), 0x0388a1), + (hex!("0100000000333333334444444455000002980000000000002990"), 0x038941), + (hex!("0100000000333333334444444455000002980000000000004370"), 0x0389e1), + (hex!("0100000000333333334444444455000002980000000000004420"), 0x038a81), + (hex!("01000000003333333344444444550000029900000000000029a0"), 0x038b21), + (hex!("01000000003333333344444444550000029a00000000000029b0"), 0x038bc1), + (hex!("01000000003333333344444444550000029a0000000000006010"), 0x038c61), + (hex!("01000000003333333344444444550000029a0000000000006980"), 0x038d01), + (hex!("01000000003333333344444444550000029b00000000000029c0"), 0x038da1), + (hex!("01000000003333333344444444550000029c00000000000029d0"), 0x038e41), + (hex!("01000000003333333344444444550000029c0000000000007480"), 0x038ee1), + (hex!("01000000003333333344444444550000029d00000000000029e0"), 0x038f81), + (hex!("01000000003333333344444444550000029d0000000000005030"), 0x039021), + (hex!("01000000003333333344444444550000029d0000000000007780"), 0x0390c1), + (hex!("01000000003333333344444444550000029d0000000000007a50"), 0x039161), + (hex!("01000000003333333344444444550000029e00000000000029f0"), 0x039201), + (hex!("01000000003333333344444444550000029e00000000000074b0"), 0x0392a1), + (hex!("01000000003333333344444444550000029f0000000000002a00"), 0x039341), + (hex!("0100000000333333334444444455000002a00000000000002a10"), 0x0393e1), + (hex!("0100000000333333334444444455000002a10000000000002a20"), 0x039481), + (hex!("0100000000333333334444444455000002a20000000000002a30"), 0x039521), + (hex!("0100000000333333334444444455000002a20000000000004c50"), 0x0395c1), + (hex!("0100000000333333334444444455000002a20000000000006f10"), 0x039661), + (hex!("0100000000333333334444444455000002a30000000000002a40"), 0x039701), + (hex!("0100000000333333334444444455000002a40000000000002a50"), 0x0397a1), + (hex!("0100000000333333334444444455000002a40000000000005d60"), 0x039841), + (hex!("0100000000333333334444444455000002a50000000000002a60"), 0x0398e1), + (hex!("0100000000333333334444444455000002a50000000000005440"), 0x039981), + (hex!("0100000000333333334444444455000002a50000000000005890"), 0x039a21), + (hex!("0100000000333333334444444455000002a60000000000002a70"), 0x039ac1), + (hex!("0100000000333333334444444455000002a70000000000002a80"), 0x039b61), + (hex!("0100000000333333334444444455000002a700000000000054a0"), 0x039c01), + (hex!("0100000000333333334444444455000002a70000000000007280"), 0x039ca1), + (hex!("0100000000333333334444444455000002a80000000000002a90"), 0x039d41), + (hex!("0100000000333333334444444455000002a90000000000002aa0"), 0x039de1), + (hex!("0100000000333333334444444455000002aa0000000000002ab0"), 0x039e81), + (hex!("0100000000333333334444444455000002ab0000000000002ac0"), 0x039f21), + (hex!("0100000000333333334444444455000002ab0000000000006c90"), 0x039fc1), + (hex!("0100000000333333334444444455000002ac0000000000002ad0"), 0x03a061), + (hex!("0100000000333333334444444455000002ac0000000000006db0"), 0x03a101), + (hex!("0100000000333333334444444455000002ad0000000000002ae0"), 0x03a1a1), + (hex!("0100000000333333334444444455000002ad00000000000065e0"), 0x03a241), + (hex!("0100000000333333334444444455000002ad0000000000007b40"), 0x03a2e1), + (hex!("0100000000333333334444444455000002ae0000000000002af0"), 0x03a381), + (hex!("0100000000333333334444444455000002ae0000000000004d20"), 0x03a421), + (hex!("0100000000333333334444444455000002ae0000000000006f30"), 0x03a4c1), + (hex!("0100000000333333334444444455000002af0000000000002b00"), 0x03a561), + (hex!("0100000000333333334444444455000002b00000000000002b10"), 0x03a601), + (hex!("0100000000333333334444444455000002b00000000000004560"), 0x03a6a1), + (hex!("0100000000333333334444444455000002b00000000000005800"), 0x03a741), + (hex!("0100000000333333334444444455000002b00000000000005a60"), 0x03a7e1), + (hex!("0100000000333333334444444455000002b10000000000002b20"), 0x03a881), + (hex!("0100000000333333334444444455000002b10000000000007b30"), 0x03a921), + (hex!("0100000000333333334444444455000002b20000000000002b30"), 0x03a9c1), + (hex!("0100000000333333334444444455000002b20000000000004440"), 0x03aa61), + (hex!("0100000000333333334444444455000002b20000000000004f80"), 0x03ab01), + (hex!("0100000000333333334444444455000002b20000000000005020"), 0x03aba1), + (hex!("0100000000333333334444444455000002b30000000000002b40"), 0x03ac41), + (hex!("0100000000333333334444444455000002b40000000000002b50"), 0x03ace1), + (hex!("0100000000333333334444444455000002b50000000000002b60"), 0x03ad81), + (hex!("0100000000333333334444444455000002b500000000000059e0"), 0x03ae21), + (hex!("0100000000333333334444444455000002b60000000000002b70"), 0x03aec1), + (hex!("0100000000333333334444444455000002b70000000000002b80"), 0x03af61), + (hex!("0100000000333333334444444455000002b80000000000002b90"), 0x03b001), + (hex!("0100000000333333334444444455000002b80000000000004590"), 0x03b0a1), + (hex!("0100000000333333334444444455000002b800000000000047d0"), 0x03b141), + (hex!("0100000000333333334444444455000002b80000000000006030"), 0x03b1e1), + (hex!("0100000000333333334444444455000002b80000000000006a20"), 0x03b281), + (hex!("0100000000333333334444444455000002b80000000000006a90"), 0x03b321), + (hex!("0100000000333333334444444455000002b90000000000002ba0"), 0x03b3c1), + (hex!("0100000000333333334444444455000002ba0000000000002bb0"), 0x03b461), + (hex!("0100000000333333334444444455000002ba0000000000006e80"), 0x03b501), + (hex!("0100000000333333334444444455000002bb0000000000002bc0"), 0x03b5a1), + (hex!("0100000000333333334444444455000002bc0000000000002bd0"), 0x03b641), + (hex!("0100000000333333334444444455000002bc0000000000004b30"), 0x03b6e1), + (hex!("0100000000333333334444444455000002bd0000000000002be0"), 0x03b781), + (hex!("0100000000333333334444444455000002bd0000000000005e10"), 0x03b821), + (hex!("0100000000333333334444444455000002be0000000000002bf0"), 0x03b8c1), + (hex!("0100000000333333334444444455000002bf0000000000002c00"), 0x03b961), + (hex!("0100000000333333334444444455000002c00000000000002c10"), 0x03ba01), + (hex!("0100000000333333334444444455000002c10000000000002c20"), 0x03baa1), + (hex!("0100000000333333334444444455000002c10000000000003ef0"), 0x03bb41), + (hex!("0100000000333333334444444455000002c20000000000002c30"), 0x03bbe1), + (hex!("0100000000333333334444444455000002c200000000000056e0"), 0x03bc81), + (hex!("0100000000333333334444444455000002c30000000000002c40"), 0x03bd21), + (hex!("0100000000333333334444444455000002c30000000000004b60"), 0x03bdc1), + (hex!("0100000000333333334444444455000002c40000000000002c50"), 0x03be61), + (hex!("0100000000333333334444444455000002c400000000000045f0"), 0x03bf01), + (hex!("0100000000333333334444444455000002c40000000000005290"), 0x03bfa1), + (hex!("0100000000333333334444444455000002c50000000000002c60"), 0x03c041), + (hex!("0100000000333333334444444455000002c60000000000002c70"), 0x03c0e1), + (hex!("0100000000333333334444444455000002c60000000000006ae0"), 0x03c181), + (hex!("0100000000333333334444444455000002c70000000000002c80"), 0x03c221), + (hex!("0100000000333333334444444455000002c70000000000005680"), 0x03c2c1), + (hex!("0100000000333333334444444455000002c70000000000006e10"), 0x03c361), + (hex!("0100000000333333334444444455000002c80000000000002c90"), 0x03c401), + (hex!("0100000000333333334444444455000002c90000000000002ca0"), 0x03c4a1), + (hex!("0100000000333333334444444455000002ca0000000000002cb0"), 0x03c541), + (hex!("0100000000333333334444444455000002cb0000000000002cc0"), 0x03c5e1), + (hex!("0100000000333333334444444455000002cc0000000000002cd0"), 0x03c681), + (hex!("0100000000333333334444444455000002cc0000000000005b50"), 0x03c721), + (hex!("0100000000333333334444444455000002cd0000000000002ce0"), 0x03c7c1), + (hex!("0100000000333333334444444455000002ce0000000000002cf0"), 0x03c861), + (hex!("0100000000333333334444444455000002ce00000000000043f0"), 0x03c901), + (hex!("0100000000333333334444444455000002ce0000000000006420"), 0x03c9a1), + (hex!("0100000000333333334444444455000002cf0000000000002d00"), 0x03ca41), + (hex!("0100000000333333334444444455000002d00000000000002d10"), 0x03cae1), + (hex!("0100000000333333334444444455000002d10000000000002d20"), 0x03cb81), + (hex!("0100000000333333334444444455000002d10000000000005370"), 0x03cc21), + (hex!("0100000000333333334444444455000002d20000000000002d30"), 0x03ccc1), + (hex!("0100000000333333334444444455000002d20000000000005ef0"), 0x03cd61), + (hex!("0100000000333333334444444455000002d20000000000006570"), 0x03ce01), + (hex!("0100000000333333334444444455000002d30000000000002d40"), 0x03cea1), + (hex!("0100000000333333334444444455000002d30000000000007360"), 0x03cf41), + (hex!("0100000000333333334444444455000002d40000000000002d50"), 0x03cfe1), + (hex!("0100000000333333334444444455000002d400000000000079a0"), 0x03d081), + (hex!("0100000000333333334444444455000002d50000000000002d60"), 0x03d121), + (hex!("0100000000333333334444444455000002d50000000000004250"), 0x03d1c1), + (hex!("0100000000333333334444444455000002d50000000000006050"), 0x03d261), + (hex!("0100000000333333334444444455000002d60000000000002d70"), 0x03d301), + (hex!("0100000000333333334444444455000002d60000000000007080"), 0x03d3a1), + (hex!("0100000000333333334444444455000002d70000000000002d80"), 0x03d441), + (hex!("0100000000333333334444444455000002d80000000000002d90"), 0x03d4e1), + (hex!("0100000000333333334444444455000002d80000000000007110"), 0x03d581), + (hex!("0100000000333333334444444455000002d800000000000073c0"), 0x03d621), + (hex!("0100000000333333334444444455000002d800000000000075a0"), 0x03d6c1), + (hex!("0100000000333333334444444455000002d90000000000002da0"), 0x03d761), + (hex!("0100000000333333334444444455000002d90000000000004860"), 0x03d801), + (hex!("0100000000333333334444444455000002d90000000000006b60"), 0x03d8a1), + (hex!("0100000000333333334444444455000002da0000000000002db0"), 0x03d941), + (hex!("0100000000333333334444444455000002da0000000000006630"), 0x03d9e1), + (hex!("0100000000333333334444444455000002db0000000000002dc0"), 0x03da81), + (hex!("0100000000333333334444444455000002dc0000000000002dd0"), 0x03db21), + (hex!("0100000000333333334444444455000002dc0000000000004830"), 0x03dbc1), + (hex!("0100000000333333334444444455000002dd0000000000002de0"), 0x03dc61), + (hex!("0100000000333333334444444455000002de0000000000002df0"), 0x03dd01), + (hex!("0100000000333333334444444455000002de0000000000004f00"), 0x03dda1), + (hex!("0100000000333333334444444455000002df0000000000002e00"), 0x03de41), + (hex!("0100000000333333334444444455000002e00000000000002e10"), 0x03dee1), + (hex!("0100000000333333334444444455000002e10000000000002e20"), 0x03df81), + (hex!("0100000000333333334444444455000002e10000000000006e90"), 0x03e021), + (hex!("0100000000333333334444444455000002e20000000000002e30"), 0x03e0c1), + (hex!("0100000000333333334444444455000002e200000000000053e0"), 0x03e161), + (hex!("0100000000333333334444444455000002e30000000000002e40"), 0x03e201), + (hex!("0100000000333333334444444455000002e30000000000006020"), 0x03e2a1), + (hex!("0100000000333333334444444455000002e30000000000006540"), 0x03e341), + (hex!("0100000000333333334444444455000002e40000000000002e50"), 0x03e3e1), + (hex!("0100000000333333334444444455000002e50000000000002e60"), 0x03e481), + (hex!("0100000000333333334444444455000002e50000000000005180"), 0x03e521), + (hex!("0100000000333333334444444455000002e50000000000007bf0"), 0x03e5c1), + (hex!("0100000000333333334444444455000002e60000000000002e70"), 0x03e661), + (hex!("0100000000333333334444444455000002e60000000000005350"), 0x03e701), + (hex!("0100000000333333334444444455000002e60000000000007960"), 0x03e7a1), + (hex!("0100000000333333334444444455000002e70000000000002e80"), 0x03e841), + (hex!("0100000000333333334444444455000002e80000000000002e90"), 0x03e8e1), + (hex!("0100000000333333334444444455000002e90000000000002ea0"), 0x03e981), + (hex!("0100000000333333334444444455000002ea0000000000002eb0"), 0x03ea21), + (hex!("0100000000333333334444444455000002eb0000000000002ec0"), 0x03eac1), + (hex!("0100000000333333334444444455000002ec0000000000002ed0"), 0x03eb61), + (hex!("0100000000333333334444444455000002ec0000000000006c10"), 0x03ec01), + (hex!("0100000000333333334444444455000002ed0000000000002ee0"), 0x03eca1), + (hex!("0100000000333333334444444455000002ed0000000000005590"), 0x03ed41), + (hex!("0100000000333333334444444455000002ed0000000000005cd0"), 0x03ede1), + (hex!("0100000000333333334444444455000002ed0000000000006910"), 0x03ee81), + (hex!("0100000000333333334444444455000002ee0000000000002ef0"), 0x03ef21), + (hex!("0100000000333333334444444455000002ef0000000000002f00"), 0x03efc1), + (hex!("0100000000333333334444444455000002ef0000000000004ed0"), 0x03f061), + (hex!("0100000000333333334444444455000002f00000000000002f10"), 0x03f101), + (hex!("0100000000333333334444444455000002f00000000000004cf0"), 0x03f1a1), + (hex!("0100000000333333334444444455000002f00000000000005d10"), 0x03f241), + (hex!("0100000000333333334444444455000002f00000000000006860"), 0x03f2e1), + (hex!("0100000000333333334444444455000002f00000000000006b50"), 0x03f381), + (hex!("0100000000333333334444444455000002f00000000000007100"), 0x03f421), + (hex!("0100000000333333334444444455000002f00000000000007aa0"), 0x03f4c1), + (hex!("0100000000333333334444444455000002f10000000000002f20"), 0x03f561), + (hex!("0100000000333333334444444455000002f20000000000002f30"), 0x03f601), + (hex!("0100000000333333334444444455000002f200000000000044b0"), 0x03f6a1), + (hex!("0100000000333333334444444455000002f30000000000002f40"), 0x03f741), + (hex!("0100000000333333334444444455000002f300000000000075b0"), 0x03f7e1), + (hex!("0100000000333333334444444455000002f40000000000002f50"), 0x03f881), + (hex!("0100000000333333334444444455000002f400000000000060f0"), 0x03f921), + (hex!("0100000000333333334444444455000002f50000000000002f60"), 0x03f9c1), + (hex!("0100000000333333334444444455000002f50000000000007210"), 0x03fa61), + (hex!("0100000000333333334444444455000002f60000000000002f70"), 0x03fb01), + (hex!("0100000000333333334444444455000002f60000000000006610"), 0x03fba1), + (hex!("0100000000333333334444444455000002f70000000000002f80"), 0x03fc41), + (hex!("0100000000333333334444444455000002f70000000000007560"), 0x03fce1), + (hex!("0100000000333333334444444455000002f80000000000002f90"), 0x03fd81), + (hex!("0100000000333333334444444455000002f80000000000006320"), 0x03fe21), + (hex!("0100000000333333334444444455000002f90000000000002fa0"), 0x03fec1), + (hex!("0100000000333333334444444455000002f90000000000006e50"), 0x03ff61), + (hex!("0100000000333333334444444455000002fa0000000000002fb0"), 0x040001), + (hex!("0100000000333333334444444455000002fb0000000000002fc0"), 0x0400a1), + (hex!("0100000000333333334444444455000002fb0000000000004780"), 0x040141), + (hex!("0100000000333333334444444455000002fc0000000000002fd0"), 0x0401e1), + (hex!("0100000000333333334444444455000002fd0000000000002fe0"), 0x040281), + (hex!("0100000000333333334444444455000002fd0000000000005600"), 0x040321), + (hex!("0100000000333333334444444455000002fd0000000000006c00"), 0x0403c1), + (hex!("0100000000333333334444444455000002fe0000000000002ff0"), 0x040461), + (hex!("0100000000333333334444444455000002ff0000000000003000"), 0x040501), + (hex!("0100000000333333334444444455000003000000000000003010"), 0x0405a1), + (hex!("0100000000333333334444444455000003000000000000004080"), 0x040641), + (hex!("0100000000333333334444444455000003010000000000003020"), 0x0406e1), + (hex!("0100000000333333334444444455000003010000000000006340"), 0x040781), + (hex!("0100000000333333334444444455000003020000000000003030"), 0x040821), + (hex!("0100000000333333334444444455000003020000000000005b00"), 0x0408c1), + (hex!("0100000000333333334444444455000003020000000000007b20"), 0x040961), + (hex!("0100000000333333334444444455000003030000000000003040"), 0x040a01), + (hex!("01000000003333333344444444550000030300000000000056b0"), 0x040aa1), + (hex!("0100000000333333334444444455000003030000000000006280"), 0x040b41), + (hex!("0100000000333333334444444455000003030000000000007ad0"), 0x040be1), + (hex!("0100000000333333334444444455000003040000000000003050"), 0x040c81), + (hex!("0100000000333333334444444455000003040000000000005c50"), 0x040d21), + (hex!("0100000000333333334444444455000003050000000000003060"), 0x040dc1), + (hex!("01000000003333333344444444550000030500000000000072e0"), 0x040e61), + (hex!("0100000000333333334444444455000003060000000000003070"), 0x040f01), + (hex!("0100000000333333334444444455000003060000000000004360"), 0x040fa1), + (hex!("0100000000333333334444444455000003060000000000004380"), 0x041041), + (hex!("0100000000333333334444444455000003060000000000004820"), 0x0410e1), + (hex!("0100000000333333334444444455000003060000000000006d10"), 0x041181), + (hex!("0100000000333333334444444455000003070000000000003080"), 0x041221), + (hex!("0100000000333333334444444455000003070000000000004450"), 0x0412c1), + (hex!("0100000000333333334444444455000003080000000000003090"), 0x041361), + (hex!("0100000000333333334444444455000003080000000000005ad0"), 0x041401), + (hex!("01000000003333333344444444550000030900000000000030a0"), 0x0414a1), + (hex!("01000000003333333344444444550000030a00000000000030b0"), 0x041541), + (hex!("01000000003333333344444444550000030a0000000000007760"), 0x0415e1), + (hex!("01000000003333333344444444550000030b00000000000030c0"), 0x041681), + (hex!("01000000003333333344444444550000030b0000000000007a80"), 0x041721), + (hex!("01000000003333333344444444550000030c00000000000030d0"), 0x0417c1), + (hex!("01000000003333333344444444550000030d00000000000030e0"), 0x041861), + (hex!("01000000003333333344444444550000030d0000000000003eb0"), 0x041901), + (hex!("01000000003333333344444444550000030e00000000000030f0"), 0x0419a1), + (hex!("01000000003333333344444444550000030f0000000000003100"), 0x041a41), + (hex!("01000000003333333344444444550000030f0000000000004690"), 0x041ae1), + (hex!("01000000003333333344444444550000030f0000000000006900"), 0x041b81), + (hex!("0100000000333333334444444455000003100000000000003110"), 0x041c21), + (hex!("01000000003333333344444444550000031000000000000058a0"), 0x041cc1), + (hex!("0100000000333333334444444455000003110000000000003120"), 0x041d61), + (hex!("0100000000333333334444444455000003110000000000004200"), 0x041e01), + (hex!("0100000000333333334444444455000003120000000000003130"), 0x041ea1), + (hex!("0100000000333333334444444455000003130000000000003140"), 0x041f41), + (hex!("0100000000333333334444444455000003130000000000004d50"), 0x041fe1), + (hex!("0100000000333333334444444455000003130000000000005400"), 0x042081), + (hex!("0100000000333333334444444455000003130000000000005520"), 0x042121), + (hex!("0100000000333333334444444455000003140000000000003150"), 0x0421c1), + (hex!("0100000000333333334444444455000003140000000000006450"), 0x042261), + (hex!("0100000000333333334444444455000003150000000000003160"), 0x042301), + (hex!("01000000003333333344444444550000031500000000000062d0"), 0x0423a1), + (hex!("0100000000333333334444444455000003160000000000003170"), 0x042441), + (hex!("0100000000333333334444444455000003160000000000004c40"), 0x0424e1), + (hex!("0100000000333333334444444455000003160000000000007c80"), 0x042581), + (hex!("0100000000333333334444444455000003170000000000003180"), 0x042621), + (hex!("0100000000333333334444444455000003170000000000004400"), 0x0426c1), + (hex!("0100000000333333334444444455000003170000000000005090"), 0x042761), + (hex!("0100000000333333334444444455000003170000000000006cb0"), 0x042801), + (hex!("0100000000333333334444444455000003180000000000003190"), 0x0428a1), + (hex!("0100000000333333334444444455000003180000000000006560"), 0x042941), + (hex!("01000000003333333344444444550000031900000000000031a0"), 0x0429e1), + (hex!("01000000003333333344444444550000031900000000000052d0"), 0x042a81), + (hex!("01000000003333333344444444550000031900000000000057e0"), 0x042b21), + (hex!("01000000003333333344444444550000031a00000000000031b0"), 0x042bc1), + (hex!("01000000003333333344444444550000031a00000000000071e0"), 0x042c61), + (hex!("01000000003333333344444444550000031b00000000000031c0"), 0x042d01), + (hex!("01000000003333333344444444550000031c00000000000031d0"), 0x042da1), + (hex!("01000000003333333344444444550000031c0000000000004480"), 0x042e41), + (hex!("01000000003333333344444444550000031c0000000000005790"), 0x042ee1), + (hex!("01000000003333333344444444550000031c0000000000007be0"), 0x042f81), + (hex!("01000000003333333344444444550000031d00000000000031e0"), 0x043021), + (hex!("01000000003333333344444444550000031d0000000000005560"), 0x0430c1), + (hex!("01000000003333333344444444550000031e00000000000031f0"), 0x043161), + (hex!("01000000003333333344444444550000031f0000000000003200"), 0x043201), + (hex!("01000000003333333344444444550000031f0000000000004190"), 0x0432a1), + (hex!("0100000000333333334444444455000003200000000000003210"), 0x043341), + (hex!("0100000000333333334444444455000003210000000000003220"), 0x0433e1), + (hex!("0100000000333333334444444455000003220000000000003230"), 0x043481), + (hex!("0100000000333333334444444455000003230000000000003240"), 0x043521), + (hex!("01000000003333333344444444550000032300000000000069d0"), 0x0435c1), + (hex!("0100000000333333334444444455000003240000000000003250"), 0x043661), + (hex!("0100000000333333334444444455000003250000000000003260"), 0x043701), + (hex!("01000000003333333344444444550000032500000000000042b0"), 0x0437a1), + (hex!("01000000003333333344444444550000032500000000000064e0"), 0x043841), + (hex!("0100000000333333334444444455000003260000000000003270"), 0x0438e1), + (hex!("0100000000333333334444444455000003270000000000003280"), 0x043981), + (hex!("0100000000333333334444444455000003270000000000005b20"), 0x043a21), + (hex!("0100000000333333334444444455000003270000000000006330"), 0x043ac1), + (hex!("0100000000333333334444444455000003270000000000006810"), 0x043b61), + (hex!("0100000000333333334444444455000003280000000000003290"), 0x043c01), + (hex!("01000000003333333344444444550000032900000000000032a0"), 0x043ca1), + (hex!("01000000003333333344444444550000032900000000000056f0"), 0x043d41), + (hex!("0100000000333333334444444455000003290000000000005e20"), 0x043de1), + (hex!("0100000000333333334444444455000003290000000000005e70"), 0x043e81), + (hex!("01000000003333333344444444550000032a00000000000032b0"), 0x043f21), + (hex!("01000000003333333344444444550000032b00000000000032c0"), 0x043fc1), + (hex!("01000000003333333344444444550000032b0000000000005500"), 0x044061), + (hex!("01000000003333333344444444550000032b0000000000005a20"), 0x044101), + (hex!("01000000003333333344444444550000032c00000000000032d0"), 0x0441a1), + (hex!("01000000003333333344444444550000032c0000000000004060"), 0x044241), + (hex!("01000000003333333344444444550000032c0000000000004760"), 0x0442e1), + (hex!("01000000003333333344444444550000032d00000000000032e0"), 0x044381), + (hex!("01000000003333333344444444550000032d00000000000068a0"), 0x044421), + (hex!("01000000003333333344444444550000032e00000000000032f0"), 0x0444c1), + (hex!("01000000003333333344444444550000032f0000000000003300"), 0x044561), + (hex!("0100000000333333334444444455000003300000000000003310"), 0x044601), + (hex!("0100000000333333334444444455000003300000000000006e40"), 0x0446a1), + (hex!("0100000000333333334444444455000003310000000000003320"), 0x044741), + (hex!("0100000000333333334444444455000003310000000000004620"), 0x0447e1), + (hex!("0100000000333333334444444455000003320000000000003330"), 0x044881), + (hex!("0100000000333333334444444455000003330000000000003340"), 0x044921), + (hex!("0100000000333333334444444455000003330000000000004b80"), 0x0449c1), + (hex!("0100000000333333334444444455000003340000000000003350"), 0x044a61), + (hex!("0100000000333333334444444455000003350000000000003360"), 0x044b01), + (hex!("0100000000333333334444444455000003360000000000003370"), 0x044ba1), + (hex!("0100000000333333334444444455000003370000000000003380"), 0x044c41), + (hex!("0100000000333333334444444455000003380000000000003390"), 0x044ce1), + (hex!("01000000003333333344444444550000033900000000000033a0"), 0x044d81), + (hex!("0100000000333333334444444455000003390000000000006b90"), 0x044e21), + (hex!("01000000003333333344444444550000033a00000000000033b0"), 0x044ec1), + (hex!("01000000003333333344444444550000033a0000000000007420"), 0x044f61), + (hex!("01000000003333333344444444550000033b00000000000033c0"), 0x045001), + (hex!("01000000003333333344444444550000033b0000000000007620"), 0x0450a1), + (hex!("01000000003333333344444444550000033c00000000000033d0"), 0x045141), + (hex!("01000000003333333344444444550000033c0000000000006b30"), 0x0451e1), + (hex!("01000000003333333344444444550000033d00000000000033e0"), 0x045281), + (hex!("01000000003333333344444444550000033e00000000000033f0"), 0x045321), + (hex!("01000000003333333344444444550000033e00000000000048b0"), 0x0453c1), + (hex!("01000000003333333344444444550000033e0000000000004e70"), 0x045461), + (hex!("01000000003333333344444444550000033f0000000000003400"), 0x045501), + (hex!("01000000003333333344444444550000033f0000000000006380"), 0x0455a1), + (hex!("0100000000333333334444444455000003400000000000003410"), 0x045641), + (hex!("0100000000333333334444444455000003410000000000003420"), 0x0456e1), + (hex!("0100000000333333334444444455000003410000000000006090"), 0x045781), + (hex!("0100000000333333334444444455000003420000000000003430"), 0x045821), + (hex!("01000000003333333344444444550000034200000000000073d0"), 0x0458c1), + (hex!("0100000000333333334444444455000003430000000000003440"), 0x045961), + (hex!("0100000000333333334444444455000003430000000000006370"), 0x045a01), + (hex!("01000000003333333344444444550000034300000000000075c0"), 0x045aa1), + (hex!("0100000000333333334444444455000003440000000000003450"), 0x045b41), + (hex!("0100000000333333334444444455000003450000000000003460"), 0x045be1), + (hex!("0100000000333333334444444455000003460000000000003470"), 0x045c81), + (hex!("01000000003333333344444444550000034600000000000055f0"), 0x045d21), + (hex!("0100000000333333334444444455000003470000000000003480"), 0x045dc1), + (hex!("0100000000333333334444444455000003470000000000003fe0"), 0x045e61), + (hex!("0100000000333333334444444455000003480000000000003490"), 0x045f01), + (hex!("0100000000333333334444444455000003480000000000007990"), 0x045fa1), + (hex!("01000000003333333344444444550000034900000000000034a0"), 0x046041), + (hex!("0100000000333333334444444455000003490000000000004410"), 0x0460e1), + (hex!("01000000003333333344444444550000034a00000000000034b0"), 0x046181), + (hex!("01000000003333333344444444550000034a00000000000062a0"), 0x046221), + (hex!("01000000003333333344444444550000034a0000000000007260"), 0x0462c1), + (hex!("01000000003333333344444444550000034b00000000000034c0"), 0x046361), + (hex!("01000000003333333344444444550000034b0000000000005760"), 0x046401), + (hex!("01000000003333333344444444550000034b0000000000006200"), 0x0464a1), + (hex!("01000000003333333344444444550000034c00000000000034d0"), 0x046541), + (hex!("01000000003333333344444444550000034d00000000000034e0"), 0x0465e1), + (hex!("01000000003333333344444444550000034e00000000000034f0"), 0x046681), + (hex!("01000000003333333344444444550000034e0000000000007790"), 0x046721), + (hex!("01000000003333333344444444550000034f0000000000003500"), 0x0467c1), + (hex!("0100000000333333334444444455000003500000000000003510"), 0x046861), + (hex!("0100000000333333334444444455000003510000000000003520"), 0x046901), + (hex!("0100000000333333334444444455000003520000000000003530"), 0x0469a1), + (hex!("01000000003333333344444444550000035200000000000056a0"), 0x046a41), + (hex!("0100000000333333334444444455000003530000000000003540"), 0x046ae1), + (hex!("0100000000333333334444444455000003540000000000003550"), 0x046b81), + (hex!("01000000003333333344444444550000035400000000000047b0"), 0x046c21), + (hex!("0100000000333333334444444455000003550000000000003560"), 0x046cc1), + (hex!("0100000000333333334444444455000003550000000000004500"), 0x046d61), + (hex!("0100000000333333334444444455000003560000000000003570"), 0x046e01), + (hex!("0100000000333333334444444455000003560000000000004fc0"), 0x046ea1), + (hex!("0100000000333333334444444455000003560000000000007160"), 0x046f41), + (hex!("0100000000333333334444444455000003560000000000007400"), 0x046fe1), + (hex!("0100000000333333334444444455000003570000000000003580"), 0x047081), + (hex!("0100000000333333334444444455000003580000000000003590"), 0x047121), + (hex!("0100000000333333334444444455000003580000000000005a80"), 0x0471c1), + (hex!("01000000003333333344444444550000035900000000000035a0"), 0x047261), + (hex!("01000000003333333344444444550000035900000000000073b0"), 0x047301), + (hex!("01000000003333333344444444550000035a00000000000035b0"), 0x0473a1), + (hex!("01000000003333333344444444550000035a0000000000004c20"), 0x047441), + (hex!("01000000003333333344444444550000035b00000000000035c0"), 0x0474e1), + (hex!("01000000003333333344444444550000035b0000000000005120"), 0x047581), + (hex!("01000000003333333344444444550000035c00000000000035d0"), 0x047621), + (hex!("01000000003333333344444444550000035c0000000000004300"), 0x0476c1), + (hex!("01000000003333333344444444550000035c0000000000005a40"), 0x047761), + (hex!("01000000003333333344444444550000035c0000000000006620"), 0x047801), + (hex!("01000000003333333344444444550000035c0000000000006ed0"), 0x0478a1), + (hex!("01000000003333333344444444550000035d00000000000035e0"), 0x047941), + (hex!("01000000003333333344444444550000035d0000000000005df0"), 0x0479e1), + (hex!("01000000003333333344444444550000035e00000000000035f0"), 0x047a81), + (hex!("01000000003333333344444444550000035f0000000000003600"), 0x047b21), + (hex!("01000000003333333344444444550000035f00000000000058d0"), 0x047bc1), + (hex!("0100000000333333334444444455000003600000000000003610"), 0x047c61), + (hex!("0100000000333333334444444455000003600000000000007b90"), 0x047d01), + (hex!("0100000000333333334444444455000003610000000000003620"), 0x047da1), + (hex!("0100000000333333334444444455000003610000000000006ad0"), 0x047e41), + (hex!("0100000000333333334444444455000003620000000000003630"), 0x047ee1), + (hex!("01000000003333333344444444550000036200000000000063a0"), 0x047f81), + (hex!("0100000000333333334444444455000003630000000000003640"), 0x048021), + (hex!("0100000000333333334444444455000003630000000000007250"), 0x0480c1), + (hex!("0100000000333333334444444455000003640000000000003650"), 0x048161), + (hex!("0100000000333333334444444455000003640000000000005510"), 0x048201), + (hex!("0100000000333333334444444455000003640000000000007850"), 0x0482a1), + (hex!("0100000000333333334444444455000003650000000000003660"), 0x048341), + (hex!("0100000000333333334444444455000003660000000000003670"), 0x0483e1), + (hex!("0100000000333333334444444455000003660000000000004650"), 0x048481), + (hex!("01000000003333333344444444550000036600000000000050d0"), 0x048521), + (hex!("0100000000333333334444444455000003660000000000006eb0"), 0x0485c1), + (hex!("0100000000333333334444444455000003670000000000003680"), 0x048661), + (hex!("01000000003333333344444444550000036700000000000071f0"), 0x048701), + (hex!("0100000000333333334444444455000003680000000000003690"), 0x0487a1), + (hex!("01000000003333333344444444550000036900000000000036a0"), 0x048841), + (hex!("0100000000333333334444444455000003690000000000005c70"), 0x0488e1), + (hex!("01000000003333333344444444550000036a00000000000036b0"), 0x048981), + (hex!("01000000003333333344444444550000036a00000000000071b0"), 0x048a21), + (hex!("01000000003333333344444444550000036b00000000000036c0"), 0x048ac1), + (hex!("01000000003333333344444444550000036b0000000000004670"), 0x048b61), + (hex!("01000000003333333344444444550000036c00000000000036d0"), 0x048c01), + (hex!("01000000003333333344444444550000036c0000000000004750"), 0x048ca1), + (hex!("01000000003333333344444444550000036c0000000000006fa0"), 0x048d41), + (hex!("01000000003333333344444444550000036d00000000000036e0"), 0x048de1), + (hex!("01000000003333333344444444550000036d0000000000003f70"), 0x048e81), + (hex!("01000000003333333344444444550000036d0000000000004b90"), 0x048f21), + (hex!("01000000003333333344444444550000036d00000000000057a0"), 0x048fc1), + (hex!("01000000003333333344444444550000036e00000000000036f0"), 0x049061), + (hex!("01000000003333333344444444550000036e00000000000075d0"), 0x049101), + (hex!("01000000003333333344444444550000036f0000000000003700"), 0x0491a1), + (hex!("0100000000333333334444444455000003700000000000003710"), 0x049241), + (hex!("0100000000333333334444444455000003700000000000005aa0"), 0x0492e1), + (hex!("0100000000333333334444444455000003710000000000003720"), 0x049381), + (hex!("0100000000333333334444444455000003710000000000005130"), 0x049421), + (hex!("0100000000333333334444444455000003710000000000006fc0"), 0x0494c1), + (hex!("0100000000333333334444444455000003710000000000007b00"), 0x049561), + (hex!("0100000000333333334444444455000003720000000000003730"), 0x049601), + (hex!("01000000003333333344444444550000037200000000000054d0"), 0x0496a1), + (hex!("0100000000333333334444444455000003730000000000003740"), 0x049741), + (hex!("0100000000333333334444444455000003730000000000004220"), 0x0497e1), + (hex!("0100000000333333334444444455000003740000000000003750"), 0x049881), + (hex!("0100000000333333334444444455000003740000000000004720"), 0x049921), + (hex!("0100000000333333334444444455000003750000000000003760"), 0x0499c1), + (hex!("0100000000333333334444444455000003750000000000004110"), 0x049a61), + (hex!("0100000000333333334444444455000003760000000000003770"), 0x049b01), + (hex!("0100000000333333334444444455000003770000000000003780"), 0x049ba1), + (hex!("0100000000333333334444444455000003780000000000003790"), 0x049c41), + (hex!("0100000000333333334444444455000003780000000000004b40"), 0x049ce1), + (hex!("0100000000333333334444444455000003780000000000005660"), 0x049d81), + (hex!("0100000000333333334444444455000003780000000000005ea0"), 0x049e21), + (hex!("01000000003333333344444444550000037900000000000037a0"), 0x049ec1), + (hex!("01000000003333333344444444550000037a00000000000037b0"), 0x049f61), + (hex!("01000000003333333344444444550000037b00000000000037c0"), 0x04a001), + (hex!("01000000003333333344444444550000037c00000000000037d0"), 0x04a0a1), + (hex!("01000000003333333344444444550000037c0000000000004340"), 0x04a141), + (hex!("01000000003333333344444444550000037c0000000000005230"), 0x04a1e1), + (hex!("01000000003333333344444444550000037d00000000000037e0"), 0x04a281), + (hex!("01000000003333333344444444550000037d00000000000051e0"), 0x04a321), + (hex!("01000000003333333344444444550000037e00000000000037f0"), 0x04a3c1), + (hex!("01000000003333333344444444550000037e0000000000004090"), 0x04a461), + (hex!("01000000003333333344444444550000037e0000000000005c20"), 0x04a501), + (hex!("01000000003333333344444444550000037f0000000000003800"), 0x04a5a1), + (hex!("0100000000333333334444444455000003800000000000003810"), 0x04a641), + (hex!("0100000000333333334444444455000003800000000000007630"), 0x04a6e1), + (hex!("0100000000333333334444444455000003810000000000003820"), 0x04a781), + (hex!("0100000000333333334444444455000003820000000000003830"), 0x04a821), + (hex!("0100000000333333334444444455000003820000000000004170"), 0x04a8c1), + (hex!("0100000000333333334444444455000003830000000000003840"), 0x04a961), + (hex!("0100000000333333334444444455000003840000000000003850"), 0x04aa01), + (hex!("0100000000333333334444444455000003850000000000003860"), 0x04aaa1), + (hex!("0100000000333333334444444455000003850000000000004180"), 0x04ab41), + (hex!("0100000000333333334444444455000003850000000000005c90"), 0x04abe1), + (hex!("0100000000333333334444444455000003850000000000005da0"), 0x04ac81), + (hex!("0100000000333333334444444455000003850000000000006ff0"), 0x04ad21), + (hex!("0100000000333333334444444455000003860000000000003870"), 0x04adc1), + (hex!("01000000003333333344444444550000038600000000000065c0"), 0x04ae61), + (hex!("0100000000333333334444444455000003870000000000003880"), 0x04af01), + (hex!("0100000000333333334444444455000003870000000000007cc0"), 0x04afa1), + (hex!("0100000000333333334444444455000003880000000000003890"), 0x04b041), + (hex!("01000000003333333344444444550000038900000000000038a0"), 0x04b0e1), + (hex!("01000000003333333344444444550000038a00000000000038b0"), 0x04b181), + (hex!("01000000003333333344444444550000038a00000000000073e0"), 0x04b221), + (hex!("01000000003333333344444444550000038b00000000000038c0"), 0x04b2c1), + (hex!("01000000003333333344444444550000038c00000000000038d0"), 0x04b361), + (hex!("01000000003333333344444444550000038d00000000000038e0"), 0x04b401), + (hex!("01000000003333333344444444550000038d00000000000069f0"), 0x04b4a1), + (hex!("01000000003333333344444444550000038d0000000000007680"), 0x04b541), + (hex!("01000000003333333344444444550000038e00000000000038f0"), 0x04b5e1), + (hex!("01000000003333333344444444550000038f0000000000003900"), 0x04b681), + (hex!("01000000003333333344444444550000038f00000000000045b0"), 0x04b721), + (hex!("01000000003333333344444444550000038f0000000000007180"), 0x04b7c1), + (hex!("0100000000333333334444444455000003900000000000003910"), 0x04b861), + (hex!("0100000000333333334444444455000003910000000000003920"), 0x04b901), + (hex!("0100000000333333334444444455000003910000000000004a20"), 0x04b9a1), + (hex!("0100000000333333334444444455000003920000000000003930"), 0x04ba41), + (hex!("01000000003333333344444444550000039200000000000059b0"), 0x04bae1), + (hex!("0100000000333333334444444455000003930000000000003940"), 0x04bb81), + (hex!("0100000000333333334444444455000003930000000000006cc0"), 0x04bc21), + (hex!("0100000000333333334444444455000003940000000000003950"), 0x04bcc1), + (hex!("01000000003333333344444444550000039400000000000056c0"), 0x04bd61), + (hex!("0100000000333333334444444455000003950000000000003960"), 0x04be01), + (hex!("0100000000333333334444444455000003950000000000004cc0"), 0x04bea1), + (hex!("0100000000333333334444444455000003950000000000007720"), 0x04bf41), + (hex!("0100000000333333334444444455000003960000000000003970"), 0x04bfe1), + (hex!("0100000000333333334444444455000003960000000000004da0"), 0x04c081), + (hex!("0100000000333333334444444455000003960000000000004df0"), 0x04c121), + (hex!("0100000000333333334444444455000003960000000000004f30"), 0x04c1c1), + (hex!("01000000003333333344444444550000039600000000000050f0"), 0x04c261), + (hex!("0100000000333333334444444455000003960000000000007940"), 0x04c301), + (hex!("0100000000333333334444444455000003970000000000003980"), 0x04c3a1), + (hex!("0100000000333333334444444455000003970000000000005850"), 0x04c441), + (hex!("0100000000333333334444444455000003970000000000007bd0"), 0x04c4e1), + (hex!("0100000000333333334444444455000003980000000000003990"), 0x04c581), + (hex!("0100000000333333334444444455000003980000000000004c00"), 0x04c621), + (hex!("0100000000333333334444444455000003980000000000005580"), 0x04c6c1), + (hex!("01000000003333333344444444550000039900000000000039a0"), 0x04c761), + (hex!("0100000000333333334444444455000003990000000000005820"), 0x04c801), + (hex!("01000000003333333344444444550000039a00000000000039b0"), 0x04c8a1), + (hex!("01000000003333333344444444550000039b00000000000039c0"), 0x04c941), + (hex!("01000000003333333344444444550000039b0000000000004c10"), 0x04c9e1), + (hex!("01000000003333333344444444550000039b0000000000006460"), 0x04ca81), + (hex!("01000000003333333344444444550000039c00000000000039d0"), 0x04cb21), + (hex!("01000000003333333344444444550000039d00000000000039e0"), 0x04cbc1), + (hex!("01000000003333333344444444550000039d00000000000044c0"), 0x04cc61), + (hex!("01000000003333333344444444550000039d00000000000049e0"), 0x04cd01), + (hex!("01000000003333333344444444550000039e00000000000039f0"), 0x04cda1), + (hex!("01000000003333333344444444550000039f0000000000003a00"), 0x04ce41), + (hex!("0100000000333333334444444455000003a00000000000003a10"), 0x04cee1), + (hex!("0100000000333333334444444455000003a10000000000003a20"), 0x04cf81), + (hex!("0100000000333333334444444455000003a10000000000006a80"), 0x04d021), + (hex!("0100000000333333334444444455000003a20000000000003a30"), 0x04d0c1), + (hex!("0100000000333333334444444455000003a200000000000062b0"), 0x04d161), + (hex!("0100000000333333334444444455000003a30000000000003a40"), 0x04d201), + (hex!("0100000000333333334444444455000003a30000000000006ce0"), 0x04d2a1), + (hex!("0100000000333333334444444455000003a40000000000003a50"), 0x04d341), + (hex!("0100000000333333334444444455000003a50000000000003a60"), 0x04d3e1), + (hex!("0100000000333333334444444455000003a60000000000003a70"), 0x04d481), + (hex!("0100000000333333334444444455000003a60000000000007750"), 0x04d521), + (hex!("0100000000333333334444444455000003a70000000000003a80"), 0x04d5c1), + (hex!("0100000000333333334444444455000003a70000000000005b10"), 0x04d661), + (hex!("0100000000333333334444444455000003a80000000000003a90"), 0x04d701), + (hex!("0100000000333333334444444455000003a80000000000006c20"), 0x04d7a1), + (hex!("0100000000333333334444444455000003a90000000000003aa0"), 0x04d841), + (hex!("0100000000333333334444444455000003a90000000000005b70"), 0x04d8e1), + (hex!("0100000000333333334444444455000003a900000000000070e0"), 0x04d981), + (hex!("0100000000333333334444444455000003aa0000000000003ab0"), 0x04da21), + (hex!("0100000000333333334444444455000003aa00000000000049f0"), 0x04dac1), + (hex!("0100000000333333334444444455000003aa0000000000004d60"), 0x04db61), + (hex!("0100000000333333334444444455000003ab0000000000003ac0"), 0x04dc01), + (hex!("0100000000333333334444444455000003ac0000000000003ad0"), 0x04dca1), + (hex!("0100000000333333334444444455000003ac0000000000004580"), 0x04dd41), + (hex!("0100000000333333334444444455000003ad0000000000003ae0"), 0x04dde1), + (hex!("0100000000333333334444444455000003ae0000000000003af0"), 0x04de81), + (hex!("0100000000333333334444444455000003af0000000000003b00"), 0x04df21), + (hex!("0100000000333333334444444455000003b00000000000003b10"), 0x04dfc1), + (hex!("0100000000333333334444444455000003b10000000000003b20"), 0x04e061), + (hex!("0100000000333333334444444455000003b10000000000003fd0"), 0x04e101), + (hex!("0100000000333333334444444455000003b20000000000003b30"), 0x04e1a1), + (hex!("0100000000333333334444444455000003b30000000000003b40"), 0x04e241), + (hex!("0100000000333333334444444455000003b40000000000003b50"), 0x04e2e1), + (hex!("0100000000333333334444444455000003b40000000000007450"), 0x04e381), + (hex!("0100000000333333334444444455000003b50000000000003b60"), 0x04e421), + (hex!("0100000000333333334444444455000003b60000000000003b70"), 0x04e4c1), + (hex!("0100000000333333334444444455000003b70000000000003b80"), 0x04e561), + (hex!("0100000000333333334444444455000003b70000000000006d50"), 0x04e601), + (hex!("0100000000333333334444444455000003b80000000000003b90"), 0x04e6a1), + (hex!("0100000000333333334444444455000003b800000000000057c0"), 0x04e741), + (hex!("0100000000333333334444444455000003b800000000000078a0"), 0x04e7e1), + (hex!("0100000000333333334444444455000003b90000000000003ba0"), 0x04e881), + (hex!("0100000000333333334444444455000003b90000000000006750"), 0x04e921), + (hex!("0100000000333333334444444455000003ba0000000000003bb0"), 0x04e9c1), + (hex!("0100000000333333334444444455000003ba0000000000007a10"), 0x04ea61), + (hex!("0100000000333333334444444455000003ba0000000000007a20"), 0x04eb01), + (hex!("0100000000333333334444444455000003bb0000000000003bc0"), 0x04eba1), + (hex!("0100000000333333334444444455000003bb0000000000005bc0"), 0x04ec41), + (hex!("0100000000333333334444444455000003bc0000000000003bd0"), 0x04ece1), + (hex!("0100000000333333334444444455000003bc0000000000005e80"), 0x04ed81), + (hex!("0100000000333333334444444455000003bc0000000000007ab0"), 0x04ee21), + (hex!("0100000000333333334444444455000003bd0000000000003be0"), 0x04eec1), + (hex!("0100000000333333334444444455000003bd00000000000049b0"), 0x04ef61), + (hex!("0100000000333333334444444455000003be0000000000003bf0"), 0x04f001), + (hex!("0100000000333333334444444455000003be0000000000005780"), 0x04f0a1), + (hex!("0100000000333333334444444455000003be0000000000007930"), 0x04f141), + (hex!("0100000000333333334444444455000003bf0000000000003c00"), 0x04f1e1), + (hex!("0100000000333333334444444455000003bf0000000000005de0"), 0x04f281), + (hex!("0100000000333333334444444455000003bf00000000000060b0"), 0x04f321), + (hex!("0100000000333333334444444455000003bf00000000000060c0"), 0x04f3c1), + (hex!("0100000000333333334444444455000003bf0000000000006a50"), 0x04f461), + (hex!("0100000000333333334444444455000003c00000000000003c10"), 0x04f501), + (hex!("0100000000333333334444444455000003c00000000000004030"), 0x04f5a1), + (hex!("0100000000333333334444444455000003c10000000000003c20"), 0x04f641), + (hex!("0100000000333333334444444455000003c20000000000003c30"), 0x04f6e1), + (hex!("0100000000333333334444444455000003c200000000000040b0"), 0x04f781), + (hex!("0100000000333333334444444455000003c30000000000003c40"), 0x04f821), + (hex!("0100000000333333334444444455000003c40000000000003c50"), 0x04f8c1), + (hex!("0100000000333333334444444455000003c40000000000005ba0"), 0x04f961), + (hex!("0100000000333333334444444455000003c50000000000003c60"), 0x04fa01), + (hex!("0100000000333333334444444455000003c60000000000003c70"), 0x04faa1), + (hex!("0100000000333333334444444455000003c70000000000003c80"), 0x04fb41), + (hex!("0100000000333333334444444455000003c70000000000004270"), 0x04fbe1), + (hex!("0100000000333333334444444455000003c80000000000003c90"), 0x04fc81), + (hex!("0100000000333333334444444455000003c80000000000006e70"), 0x04fd21), + (hex!("0100000000333333334444444455000003c90000000000003ca0"), 0x04fdc1), + (hex!("0100000000333333334444444455000003ca0000000000003cb0"), 0x04fe61), + (hex!("0100000000333333334444444455000003ca0000000000006e20"), 0x04ff01), + (hex!("0100000000333333334444444455000003ca0000000000007c20"), 0x04ffa1), + (hex!("0100000000333333334444444455000003cb0000000000003cc0"), 0x050041), + (hex!("0100000000333333334444444455000003cc0000000000003cd0"), 0x0500e1), + (hex!("0100000000333333334444444455000003cc0000000000006120"), 0x050181), + (hex!("0100000000333333334444444455000003cc0000000000007950"), 0x050221), + (hex!("0100000000333333334444444455000003cd0000000000003ce0"), 0x0502c1), + (hex!("0100000000333333334444444455000003ce0000000000003cf0"), 0x050361), + (hex!("0100000000333333334444444455000003cf0000000000003d00"), 0x050401), + (hex!("0100000000333333334444444455000003d00000000000003d10"), 0x0504a1), + (hex!("0100000000333333334444444455000003d10000000000003d20"), 0x050541), + (hex!("0100000000333333334444444455000003d10000000000005e50"), 0x0505e1), + (hex!("0100000000333333334444444455000003d10000000000007880"), 0x050681), + (hex!("0100000000333333334444444455000003d20000000000003d30"), 0x050721), + (hex!("0100000000333333334444444455000003d20000000000005d00"), 0x0507c1), + (hex!("0100000000333333334444444455000003d30000000000003d40"), 0x050861), + (hex!("0100000000333333334444444455000003d30000000000005d40"), 0x050901), + (hex!("0100000000333333334444444455000003d300000000000063f0"), 0x0509a1), + (hex!("0100000000333333334444444455000003d40000000000003d50"), 0x050a41), + (hex!("0100000000333333334444444455000003d40000000000005700"), 0x050ae1), + (hex!("0100000000333333334444444455000003d400000000000078f0"), 0x050b81), + (hex!("0100000000333333334444444455000003d50000000000003d60"), 0x050c21), + (hex!("0100000000333333334444444455000003d60000000000003d70"), 0x050cc1), + (hex!("0100000000333333334444444455000003d70000000000003d80"), 0x050d61), + (hex!("0100000000333333334444444455000003d80000000000003d90"), 0x050e01), + (hex!("0100000000333333334444444455000003d80000000000006690"), 0x050ea1), + (hex!("0100000000333333334444444455000003d90000000000003da0"), 0x050f41), + (hex!("0100000000333333334444444455000003d900000000000076d0"), 0x050fe1), + (hex!("0100000000333333334444444455000003da0000000000003db0"), 0x051081), + (hex!("0100000000333333334444444455000003db0000000000003dc0"), 0x051121), + (hex!("0100000000333333334444444455000003db0000000000004a30"), 0x0511c1), + (hex!("0100000000333333334444444455000003db0000000000005390"), 0x051261), + (hex!("0100000000333333334444444455000003dc0000000000003dd0"), 0x051301), + (hex!("0100000000333333334444444455000003dc0000000000006d60"), 0x0513a1), + (hex!("0100000000333333334444444455000003dd0000000000003de0"), 0x051441), + (hex!("0100000000333333334444444455000003de0000000000003df0"), 0x0514e1), + (hex!("0100000000333333334444444455000003df0000000000003e00"), 0x051581), + (hex!("0100000000333333334444444455000003df0000000000005240"), 0x051621), + (hex!("0100000000333333334444444455000003df0000000000005610"), 0x0516c1), + (hex!("0100000000333333334444444455000003e00000000000003e10"), 0x051761), + (hex!("0100000000333333334444444455000003e00000000000006500"), 0x051801), + (hex!("0100000000333333334444444455000003e10000000000003e20"), 0x0518a1), + (hex!("0100000000333333334444444455000003e10000000000006a10"), 0x051941), + (hex!("0100000000333333334444444455000003e10000000000007c10"), 0x0519e1), + (hex!("0100000000333333334444444455000003e20000000000003e30"), 0x051a81), + (hex!("0100000000333333334444444455000003e20000000000006310"), 0x051b21), + (hex!("0100000000333333334444444455000003e30000000000003e40"), 0x051bc1), + (hex!("0100000000333333334444444455000003e40000000000003e50"), 0x051c61), + (hex!("0100000000333333334444444455000003e40000000000006780"), 0x051d01), + (hex!("0100000000333333334444444455000003e40000000000007ce0"), 0x051da1), + (hex!("0100000000333333334444444455000003e50000000000003e60"), 0x051e41), + (hex!("0100000000333333334444444455000003e60000000000003e70"), 0x051ee1), + (hex!("0100000000333333334444444455000003e60000000000005040"), 0x051f81), + (hex!("0100000000333333334444444455000003e60000000000005bf0"), 0x052021), + (hex!("0100000000333333334444444455000003e70000000000003e80"), 0x0520c1), + (hex!("0100000000333333334444444455000003e70000000000003f50"), 0x052161), ]; diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 1275250bf0..2cb1e55b26 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -345,14 +345,19 @@ impl InMemoryLayer { let cursor = inner.file.block_cursor(); - let mut keys: Vec<(&Key, &VecMap)> = inner.index.iter().collect(); - keys.sort_by_key(|k| k.0); + // Sort the keys because delta layer writer expects them sorted. + // + // NOTE: this sort can take up significant time if the layer has millions of + // keys. To speed up all the comparisons we convert the key to i128 and + // keep the value as a reference. + let mut keys: Vec<_> = inner.index.iter().map(|(k, m)| (k.to_i128(), m)).collect(); + keys.sort_unstable_by_key(|k| k.0); let ctx = RequestContextBuilder::extend(ctx) .page_content_kind(PageContentKind::InMemoryLayer) .build(); for (key, vec_map) in keys.iter() { - let key = **key; + let key = Key::from_i128(*key); // Write all page versions for (lsn, pos) in vec_map.as_slice() { cursor.read_blob_into_buf(*pos, &mut buf, &ctx).await?; From e6470ee92efec67a4bc845bb149a7b06260456dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 6 Nov 2023 15:00:07 +0100 Subject: [PATCH 26/29] Add API description for safekeeper copy endpoint (#5770) Adds a yaml API description for a new endpoint that allows creation of a new timeline as the copy of an existing one. Part of #5282 --- safekeeper/src/http/openapi_spec.yaml | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/safekeeper/src/http/openapi_spec.yaml b/safekeeper/src/http/openapi_spec.yaml index 51ce7589a0..a617e0310c 100644 --- a/safekeeper/src/http/openapi_spec.yaml +++ b/safekeeper/src/http/openapi_spec.yaml @@ -86,6 +86,41 @@ paths: default: $ref: "#/components/responses/GenericError" + /v1/tenant/{tenant_id}/timeline/{source_timeline_id}/copy: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex + - name: source_timeline_id + in: path + required: true + schema: + type: string + format: hex + + post: + tags: + - "Timeline" + summary: Register new timeline as copy of existing timeline + description: "" + operationId: v1CopyTenantTimeline + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/TimelineCopyRequest" + responses: + "201": + description: Timeline created + # TODO: return timeline info? + "403": + $ref: "#/components/responses/ForbiddenError" + default: + $ref: "#/components/responses/GenericError" + /v1/tenant/{tenant_id}/timeline/{timeline_id}: parameters: @@ -210,6 +245,18 @@ components: type: integer minimum: 0 + TimelineCopyRequest: + type: object + required: + - target_timeline_id + - until_lsn + properties: + target_timeline_id: + type: string + format: hex + until_lsn: + type: string + SkTimelineInfo: type: object required: From 85cd97af61ffc92f0e4c7ad7ee4992af0e98dce2 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 6 Nov 2023 14:03:22 +0000 Subject: [PATCH 27/29] pageserver: add `InProgress` tenant map state, use a sync lock for the map (#5367) ## Problem Follows on from #5299 - We didn't have a generic way to protect a tenant undergoing changes: `Tenant` had states, but for our arbitrary transitions between secondary/attached, we need a general way to say "reserve this tenant ID, and don't allow any other ops on it, but don't try and report it as being in any particular state". - The TenantsMap structure was behind an async RwLock, but it was never correct to hold it across await points: that would block any other changes for all tenants. ## Summary of changes - Add the `TenantSlot::InProgress` value. This means: - Incoming administrative operations on the tenant should retry later - Anything trying to read the live state of the tenant (e.g. a page service reader) should retry later or block. - Store TenantsMap in `std::sync::RwLock` - Provide an extended `get_active_tenant_with_timeout` for page_service to use, which will wait on InProgress slots as well as non-active tenants. Closes: https://github.com/neondatabase/neon/issues/5378 --------- Co-authored-by: Christian Schwarz --- libs/utils/src/lib.rs | 3 + libs/utils/src/timeout.rs | 37 + pageserver/src/consumption_metrics.rs | 2 +- pageserver/src/consumption_metrics/metrics.rs | 1 - pageserver/src/disk_usage_eviction_task.rs | 2 +- pageserver/src/http/routes.rs | 77 +- pageserver/src/metrics.rs | 29 + pageserver/src/page_service.rs | 134 +- pageserver/src/tenant.rs | 63 +- pageserver/src/tenant/delete.rs | 52 +- pageserver/src/tenant/mgr.rs | 1203 ++++++++++++----- .../src/tenant/timeline/eviction_task.rs | 15 +- test_runner/fixtures/neon_fixtures.py | 17 + test_runner/regress/test_neon_cli.py | 7 + .../regress/test_pageserver_restart.py | 5 + test_runner/regress/test_tenant_delete.py | 8 + test_runner/regress/test_tenant_detach.py | 76 +- 17 files changed, 1142 insertions(+), 589 deletions(-) create mode 100644 libs/utils/src/timeout.rs diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 1e34034023..ff5d774285 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -77,6 +77,9 @@ pub mod completion; /// Reporting utilities pub mod error; +/// async timeout helper +pub mod timeout; + pub mod sync; /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages diff --git a/libs/utils/src/timeout.rs b/libs/utils/src/timeout.rs new file mode 100644 index 0000000000..11fa417242 --- /dev/null +++ b/libs/utils/src/timeout.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use tokio_util::sync::CancellationToken; + +pub enum TimeoutCancellableError { + Timeout, + Cancelled, +} + +/// Wrap [`tokio::time::timeout`] with a CancellationToken. +/// +/// This wrapper is appropriate for any long running operation in a task +/// that ought to respect a CancellationToken (which means most tasks). +/// +/// The only time you should use a bare tokio::timeout is when the future `F` +/// itself respects a CancellationToken: otherwise, always use this wrapper +/// with your CancellationToken to ensure that your task does not hold up +/// graceful shutdown. +pub async fn timeout_cancellable( + duration: Duration, + cancel: &CancellationToken, + future: F, +) -> Result +where + F: std::future::Future, +{ + tokio::select!( + r = tokio::time::timeout(duration, future) => { + r.map_err(|_| TimeoutCancellableError::Timeout) + + }, + _ = cancel.cancelled() => { + Err(TimeoutCancellableError::Cancelled) + + } + ) +} diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 061045eb76..9e8377c1f1 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -266,7 +266,7 @@ async fn calculate_synthetic_size_worker( continue; } - if let Ok(tenant) = mgr::get_tenant(tenant_id, true).await { + if let Ok(tenant) = mgr::get_tenant(tenant_id, true) { // TODO should we use concurrent_background_tasks_rate_limit() here, like the other background tasks? // We can put in some prioritization for consumption metrics. // Same for the loop that fetches computed metrics. diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index c22f218976..2989e15e8e 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -202,7 +202,6 @@ pub(super) async fn collect_all_metrics( None } else { crate::tenant::mgr::get_tenant(id, true) - .await .ok() .map(|tenant| (id, tenant)) } diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index bc4bf51862..642cafad28 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -545,7 +545,7 @@ async fn collect_eviction_candidates( if cancel.is_cancelled() { return Ok(EvictionCandidates::Cancelled); } - let tenant = match tenant::mgr::get_tenant(*tenant_id, true).await { + let tenant = match tenant::mgr::get_tenant(*tenant_id, true) { Ok(tenant) => tenant, Err(e) => { // this can happen if tenant has lifecycle transition after we fetched it diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index db728f243e..820b8eae46 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -35,7 +35,8 @@ use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; use crate::tenant::config::{LocationConf, TenantConfOpt}; use crate::tenant::mgr::{ - GetTenantError, SetNewTenantConfigError, TenantMapInsertError, TenantStateError, + GetTenantError, SetNewTenantConfigError, TenantMapError, TenantMapInsertError, TenantSlotError, + TenantSlotUpsertError, TenantStateError, }; use crate::tenant::size::ModelInputs; use crate::tenant::storage_layer::LayerAccessStatsReset; @@ -146,28 +147,59 @@ impl From for ApiError { impl From for ApiError { fn from(tmie: TenantMapInsertError) -> ApiError { match tmie { - TenantMapInsertError::StillInitializing | TenantMapInsertError::ShuttingDown => { - ApiError::ResourceUnavailable(format!("{tmie}").into()) - } - TenantMapInsertError::TenantAlreadyExists(id, state) => { - ApiError::Conflict(format!("tenant {id} already exists, state: {state:?}")) - } - TenantMapInsertError::TenantExistsSecondary(id) => { - ApiError::Conflict(format!("tenant {id} already exists as secondary")) - } + TenantMapInsertError::SlotError(e) => e.into(), + TenantMapInsertError::SlotUpsertError(e) => e.into(), TenantMapInsertError::Other(e) => ApiError::InternalServerError(e), } } } +impl From for ApiError { + fn from(e: TenantSlotError) -> ApiError { + use TenantSlotError::*; + match e { + NotFound(tenant_id) => { + ApiError::NotFound(anyhow::anyhow!("NotFound: tenant {tenant_id}").into()) + } + e @ (AlreadyExists(_, _) | Conflict(_)) => ApiError::Conflict(format!("{e}")), + InProgress => { + ApiError::ResourceUnavailable("Tenant is being modified concurrently".into()) + } + MapState(e) => e.into(), + } + } +} + +impl From for ApiError { + fn from(e: TenantSlotUpsertError) -> ApiError { + use TenantSlotUpsertError::*; + match e { + InternalError(e) => ApiError::InternalServerError(anyhow::anyhow!("{e}")), + MapState(e) => e.into(), + } + } +} + +impl From for ApiError { + fn from(e: TenantMapError) -> ApiError { + use TenantMapError::*; + match e { + StillInitializing | ShuttingDown => { + ApiError::ResourceUnavailable(format!("{e}").into()) + } + } + } +} + impl From for ApiError { fn from(tse: TenantStateError) -> ApiError { match tse { - TenantStateError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid).into()), TenantStateError::IsStopping(_) => { ApiError::ResourceUnavailable("Tenant is stopping".into()) } - _ => ApiError::InternalServerError(anyhow::Error::new(tse)), + TenantStateError::SlotError(e) => e.into(), + TenantStateError::SlotUpsertError(e) => e.into(), + TenantStateError::Other(e) => ApiError::InternalServerError(anyhow!(e)), } } } @@ -188,6 +220,7 @@ impl From for ApiError { // (We can produce this variant only in `mgr::get_tenant(..., active=true)` calls). ApiError::ResourceUnavailable("Tenant not yet active".into()) } + GetTenantError::MapState(e) => ApiError::ResourceUnavailable(format!("{e}").into()), } } } @@ -242,6 +275,9 @@ impl From for ApiError { Get(g) => ApiError::from(g), e @ AlreadyInProgress => ApiError::Conflict(e.to_string()), Timeline(t) => ApiError::from(t), + NotAttached => ApiError::NotFound(anyhow::anyhow!("Tenant is not attached").into()), + SlotError(e) => e.into(), + SlotUpsertError(e) => e.into(), Other(o) => ApiError::InternalServerError(o), e @ InvalidState(_) => ApiError::PreconditionFailed(e.to_string().into_boxed_str()), } @@ -368,7 +404,7 @@ async fn timeline_create_handler( let state = get_state(&request); async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; match tenant.create_timeline( new_timeline_id, request_data.ancestor_timeline_id.map(TimelineId::from), @@ -418,7 +454,7 @@ async fn timeline_list_handler( let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let response_data = async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; let timelines = tenant.list_timelines(); let mut response_data = Vec::with_capacity(timelines.len()); @@ -457,7 +493,7 @@ async fn timeline_detail_handler( let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let timeline_info = async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; let timeline = tenant .get_timeline(timeline_id, false) @@ -713,7 +749,7 @@ async fn tenant_status( check_permission(&request, Some(tenant_id))?; let tenant_info = async { - let tenant = mgr::get_tenant(tenant_id, false).await?; + let tenant = mgr::get_tenant(tenant_id, false)?; // Calculate total physical size of all timelines let mut current_physical_size = 0; @@ -776,7 +812,7 @@ async fn tenant_size_handler( let headers = request.headers(); let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; // this can be long operation let inputs = tenant @@ -1033,7 +1069,7 @@ async fn get_tenant_config_handler( let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; - let tenant = mgr::get_tenant(tenant_id, false).await?; + let tenant = mgr::get_tenant(tenant_id, false)?; let response = HashMap::from([ ( @@ -1092,7 +1128,7 @@ async fn put_tenant_location_config_handler( .await { match e { - TenantStateError::NotFound(_) => { + TenantStateError::SlotError(TenantSlotError::NotFound(_)) => { // This API is idempotent: a NotFound on a detach is fine. } _ => return Err(e.into()), @@ -1130,7 +1166,6 @@ async fn handle_tenant_break( let tenant_id: TenantId = parse_request_param(&r, "tenant_id")?; let tenant = crate::tenant::mgr::get_tenant(tenant_id, true) - .await .map_err(|_| ApiError::Conflict(String::from("no active tenant found")))?; tenant.set_broken("broken from test".to_owned()).await; @@ -1435,7 +1470,7 @@ async fn active_timeline_of_active_tenant( tenant_id: TenantId, timeline_id: TimelineId, ) -> Result, ApiError> { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; tenant .get_timeline(timeline_id, true) .map_err(|e| ApiError::NotFound(e.into())) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 3e15be67dc..429ab801d9 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -962,6 +962,32 @@ static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy = Lazy .expect("failed to define a metric") }); +pub(crate) struct TenantManagerMetrics { + pub(crate) tenant_slots: UIntGauge, + pub(crate) tenant_slot_writes: IntCounter, + pub(crate) unexpected_errors: IntCounter, +} + +pub(crate) static TENANT_MANAGER: Lazy = Lazy::new(|| { + TenantManagerMetrics { + tenant_slots: register_uint_gauge!( + "pageserver_tenant_manager_slots", + "How many slots currently exist, including all attached, secondary and in-progress operations", + ) + .expect("failed to define a metric"), + tenant_slot_writes: register_int_counter!( + "pageserver_tenant_manager_slot_writes", + "Writes to a tenant slot, including all of create/attach/detach/delete" + ) + .expect("failed to define a metric"), + unexpected_errors: register_int_counter!( + "pageserver_tenant_manager_unexpected_errors_total", + "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug." + ) + .expect("failed to define a metric"), +} +}); + pub(crate) struct DeletionQueueMetrics { pub(crate) keys_submitted: IntCounter, pub(crate) keys_dropped: IntCounter, @@ -1884,6 +1910,9 @@ pub fn preinitialize_metrics() { // Deletion queue stats Lazy::force(&DELETION_QUEUE); + // Tenant manager stats + Lazy::force(&TENANT_MANAGER); + // countervecs [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT] .into_iter() diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 334aee3dd1..6086d0b063 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -14,7 +14,6 @@ use async_compression::tokio::write::GzipEncoder; use bytes::Buf; use bytes::Bytes; use futures::Stream; -use pageserver_api::models::TenantState; use pageserver_api::models::{ PagestreamBeMessage, PagestreamDbSizeRequest, PagestreamDbSizeResponse, PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse, @@ -55,16 +54,20 @@ use crate::metrics; use crate::metrics::LIVE_CONNECTIONS_COUNT; use crate::task_mgr; use crate::task_mgr::TaskKind; -use crate::tenant; use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::mgr; -use crate::tenant::mgr::GetTenantError; -use crate::tenant::{Tenant, Timeline}; +use crate::tenant::mgr::get_active_tenant_with_timeout; +use crate::tenant::mgr::GetActiveTenantError; +use crate::tenant::Timeline; use crate::trace::Tracer; use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID; use postgres_ffi::BLCKSZ; +// How long we may block waiting for a [`TenantSlot::InProgress`]` and/or a [`Tenant`] which +// is not yet in state [`TenantState::Active`]. +const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(5000); + /// Read the end of a tar archive. /// /// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each. @@ -378,7 +381,12 @@ impl PageServerHandler { debug_assert_current_span_has_tenant_and_timeline_id(); // Make request tracer if needed - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = mgr::get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; let mut tracer = if tenant.get_trace_read_requests() { let connection_id = ConnectionId::generate(); let path = tenant @@ -529,7 +537,12 @@ impl PageServerHandler { // Create empty timeline info!("creating new timeline"); - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; let timeline = tenant .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx) .await?; @@ -587,7 +600,9 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let last_record_lsn = timeline.get_last_record_lsn(); if last_record_lsn != start_lsn { return Err(QueryError::Other( @@ -795,7 +810,9 @@ impl PageServerHandler { let started = std::time::Instant::now(); // check that the timeline exists - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); if let Some(lsn) = lsn { // Backup was requested at a particular LSN. Wait for it to arrive. @@ -894,6 +911,25 @@ impl PageServerHandler { .expect("claims presence already checked"); check_permission(claims, tenant_id) } + + /// Shorthand for getting a reference to a Timeline of an Active tenant. + async fn get_active_tenant_timeline( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> Result, GetActiveTimelineError> { + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await + .map_err(GetActiveTimelineError::Tenant)?; + let timeline = tenant + .get_timeline(timeline_id, true) + .map_err(|e| GetActiveTimelineError::Timeline(anyhow::anyhow!(e)))?; + Ok(timeline) + } } #[async_trait::async_trait] @@ -1051,7 +1087,9 @@ where .record("timeline_id", field::display(timeline_id)); self.check_permission(Some(tenant_id))?; - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let end_of_timeline = timeline.get_last_record_rlsn(); @@ -1235,7 +1273,12 @@ where self.check_permission(Some(tenant_id))?; - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; pgb.write_message_noflush(&BeMessage::RowDescription(&[ RowDescriptor::int8_col(b"checkpoint_distance"), RowDescriptor::int8_col(b"checkpoint_timeout"), @@ -1281,67 +1324,13 @@ where } } -#[derive(thiserror::Error, Debug)] -enum GetActiveTenantError { - #[error( - "Timed out waiting {wait_time:?} for tenant active state. Latest state: {latest_state:?}" - )] - WaitForActiveTimeout { - latest_state: TenantState, - wait_time: Duration, - }, - #[error(transparent)] - NotFound(GetTenantError), - #[error(transparent)] - WaitTenantActive(tenant::WaitToBecomeActiveError), -} - impl From for QueryError { fn from(e: GetActiveTenantError) -> Self { match e { GetActiveTenantError::WaitForActiveTimeout { .. } => QueryError::Disconnected( ConnectionError::Io(io::Error::new(io::ErrorKind::TimedOut, e.to_string())), ), - GetActiveTenantError::WaitTenantActive(e) => QueryError::Other(anyhow::Error::new(e)), - GetActiveTenantError::NotFound(e) => QueryError::Other(anyhow::Error::new(e)), - } - } -} - -/// Get active tenant. -/// -/// If the tenant is Loading, waits for it to become Active, for up to 30 s. That -/// ensures that queries don't fail immediately after pageserver startup, because -/// all tenants are still loading. -async fn get_active_tenant_with_timeout( - tenant_id: TenantId, - _ctx: &RequestContext, /* require get a context to support cancellation in the future */ -) -> Result, GetActiveTenantError> { - let tenant = match mgr::get_tenant(tenant_id, false).await { - Ok(tenant) => tenant, - Err(e @ GetTenantError::NotFound(_)) => return Err(GetActiveTenantError::NotFound(e)), - Err(GetTenantError::NotActive(_)) => { - unreachable!("we're calling get_tenant with active_only=false") - } - Err(GetTenantError::Broken(_)) => { - unreachable!("we're calling get_tenant with active_only=false") - } - }; - let wait_time = Duration::from_secs(30); - match tokio::time::timeout(wait_time, tenant.wait_to_become_active()).await { - Ok(Ok(())) => Ok(tenant), - // no .context(), the error message is good enough and some tests depend on it - Ok(Err(e)) => Err(GetActiveTenantError::WaitTenantActive(e)), - Err(_) => { - let latest_state = tenant.current_state(); - if latest_state == TenantState::Active { - Ok(tenant) - } else { - Err(GetActiveTenantError::WaitForActiveTimeout { - latest_state, - wait_time, - }) - } + e => QueryError::Other(anyhow::anyhow!(e)), } } } @@ -1362,18 +1351,3 @@ impl From for QueryError { } } } - -/// Shorthand for getting a reference to a Timeline of an Active tenant. -async fn get_active_tenant_timeline( - tenant_id: TenantId, - timeline_id: TimelineId, - ctx: &RequestContext, -) -> Result, GetActiveTimelineError> { - let tenant = get_active_tenant_with_timeout(tenant_id, ctx) - .await - .map_err(GetActiveTimelineError::Tenant)?; - let timeline = tenant - .get_timeline(timeline_id, true) - .map_err(|e| GetActiveTimelineError::Timeline(anyhow::anyhow!(e)))?; - Ok(timeline) -} diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index ad538a82fd..a738633d5e 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -55,6 +55,8 @@ use self::config::TenantConf; use self::delete::DeleteTenantFlow; use self::metadata::LoadMetadataError; use self::metadata::TimelineMetadata; +use self::mgr::GetActiveTenantError; +use self::mgr::GetTenantError; use self::mgr::TenantsMap; use self::remote_timeline_client::RemoteTimelineClient; use self::timeline::uninit::TimelineUninitMark; @@ -263,6 +265,12 @@ pub struct Tenant { pub(crate) gate: Gate, } +impl std::fmt::Debug for Tenant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ({})", self.tenant_id, self.current_state()) + } +} + pub(crate) enum WalRedoManager { Prod(PostgresRedoManager), #[cfg(test)] @@ -368,34 +376,6 @@ impl Debug for SetStoppingError { } } -#[derive(Debug, thiserror::Error)] -pub(crate) enum WaitToBecomeActiveError { - WillNotBecomeActive { - tenant_id: TenantId, - state: TenantState, - }, - TenantDropped { - tenant_id: TenantId, - }, -} - -impl std::fmt::Display for WaitToBecomeActiveError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - WaitToBecomeActiveError::WillNotBecomeActive { tenant_id, state } => { - write!( - f, - "Tenant {} will not become active. Current state: {:?}", - tenant_id, state - ) - } - WaitToBecomeActiveError::TenantDropped { tenant_id } => { - write!(f, "Tenant {tenant_id} will not become active (dropped)") - } - } - } -} - #[derive(thiserror::Error, Debug)] pub enum CreateTimelineError { #[error("a timeline with the given ID already exists")] @@ -537,7 +517,7 @@ impl Tenant { resources: TenantSharedResources, attached_conf: AttachedTenantConf, init_order: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { @@ -1850,6 +1830,7 @@ impl Tenant { } Err(SetStoppingError::AlreadyStopping(other)) => { // give caller the option to wait for this this shutdown + info!("Tenant::shutdown: AlreadyStopping"); return Err(other); } }; @@ -2048,7 +2029,7 @@ impl Tenant { self.state.subscribe() } - pub(crate) async fn wait_to_become_active(&self) -> Result<(), WaitToBecomeActiveError> { + pub(crate) async fn wait_to_become_active(&self) -> Result<(), GetActiveTenantError> { let mut receiver = self.state.subscribe(); loop { let current_state = receiver.borrow_and_update().clone(); @@ -2056,11 +2037,9 @@ impl Tenant { TenantState::Loading | TenantState::Attaching | TenantState::Activating(_) => { // in these states, there's a chance that we can reach ::Active receiver.changed().await.map_err( - |_e: tokio::sync::watch::error::RecvError| { - WaitToBecomeActiveError::TenantDropped { - tenant_id: self.tenant_id, - } - }, + |_e: tokio::sync::watch::error::RecvError| + // Tenant existed but was dropped: report it as non-existent + GetActiveTenantError::NotFound(GetTenantError::NotFound(self.tenant_id)) )?; } TenantState::Active { .. } => { @@ -2068,10 +2047,7 @@ impl Tenant { } TenantState::Broken { .. } | TenantState::Stopping { .. } => { // There's no chance the tenant can transition back into ::Active - return Err(WaitToBecomeActiveError::WillNotBecomeActive { - tenant_id: self.tenant_id, - state: current_state, - }); + return Err(GetActiveTenantError::WillNotBecomeActive(current_state)); } } } @@ -2137,6 +2113,9 @@ where } impl Tenant { + pub fn get_tenant_id(&self) -> TenantId { + self.tenant_id + } pub fn tenant_specific_overrides(&self) -> TenantConfOpt { self.tenant_conf.read().unwrap().tenant_conf } @@ -4266,11 +4245,7 @@ mod tests { metadata_bytes[8] ^= 1; std::fs::write(metadata_path, metadata_bytes)?; - let err = harness - .try_load_local(&ctx) - .await - .err() - .expect("should fail"); + let err = harness.try_load_local(&ctx).await.expect_err("should fail"); // get all the stack with all .context, not only the last one let message = format!("{err:#}"); let expected = "failed to load metadata"; diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 87b48e4bee..066f239ff0 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -21,7 +21,7 @@ use crate::{ }; use super::{ - mgr::{GetTenantError, TenantsMap}, + mgr::{GetTenantError, TenantSlotError, TenantSlotUpsertError, TenantsMap}, remote_timeline_client::{FAILED_REMOTE_OP_RETRIES, FAILED_UPLOAD_WARN_THRESHOLD}, span, timeline::delete::DeleteTimelineFlow, @@ -33,12 +33,21 @@ pub(crate) enum DeleteTenantError { #[error("GetTenant {0}")] Get(#[from] GetTenantError), + #[error("Tenant not attached")] + NotAttached, + #[error("Invalid state {0}. Expected Active or Broken")] InvalidState(TenantState), #[error("Tenant deletion is already in progress")] AlreadyInProgress, + #[error("Tenant map slot error {0}")] + SlotError(#[from] TenantSlotError), + + #[error("Tenant map slot upsert error {0}")] + SlotUpsertError(#[from] TenantSlotUpsertError), + #[error("Timeline {0}")] Timeline(#[from] DeleteTimelineError), @@ -273,12 +282,12 @@ impl DeleteTenantFlow { pub(crate) async fn run( conf: &'static PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, - tenant_id: TenantId, + tenants: &'static std::sync::RwLock, + tenant: Arc, ) -> Result<(), DeleteTenantError> { span::debug_assert_current_span_has_tenant_id(); - let (tenant, mut guard) = Self::prepare(tenants, tenant_id).await?; + let mut guard = Self::prepare(&tenant).await?; if let Err(e) = Self::run_inner(&mut guard, conf, remote_storage.as_ref(), &tenant).await { tenant.set_broken(format!("{e:#}")).await; @@ -378,7 +387,7 @@ impl DeleteTenantFlow { guard: DeletionGuard, tenant: &Arc, preload: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, init_order: Option, ctx: &RequestContext, ) -> Result<(), DeleteTenantError> { @@ -405,15 +414,8 @@ impl DeleteTenantFlow { } async fn prepare( - tenants: &tokio::sync::RwLock, - tenant_id: TenantId, - ) -> Result<(Arc, tokio::sync::OwnedMutexGuard), DeleteTenantError> { - let m = tenants.read().await; - - let tenant = m - .get(&tenant_id) - .ok_or(GetTenantError::NotFound(tenant_id))?; - + tenant: &Arc, + ) -> Result, DeleteTenantError> { // FIXME: unsure about active only. Our init jobs may not be cancellable properly, // so at least for now allow deletions only for active tenants. TODO recheck // Broken and Stopping is needed for retries. @@ -447,14 +449,14 @@ impl DeleteTenantFlow { ))); } - Ok((Arc::clone(tenant), guard)) + Ok(guard) } fn schedule_background( guard: OwnedMutexGuard, conf: &'static PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, tenant: Arc, ) { let tenant_id = tenant.tenant_id; @@ -487,7 +489,7 @@ impl DeleteTenantFlow { mut guard: OwnedMutexGuard, conf: &PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, tenant: &Arc, ) -> Result<(), DeleteTenantError> { // Tree sort timelines, schedule delete for them. Mention retries from the console side. @@ -535,10 +537,18 @@ impl DeleteTenantFlow { .await .context("cleanup_remaining_fs_traces")?; - let mut locked = tenants.write().await; - if locked.remove(&tenant.tenant_id).is_none() { - warn!("Tenant got removed from tenants map during deletion"); - }; + { + let mut locked = tenants.write().unwrap(); + if locked.remove(&tenant.tenant_id).is_none() { + warn!("Tenant got removed from tenants map during deletion"); + }; + + // FIXME: we should not be modifying this from outside of mgr.rs. + // This will go away when we simplify deletion (https://github.com/neondatabase/neon/issues/5080) + crate::metrics::TENANT_MANAGER + .tenant_slots + .set(locked.len() as u64); + } *guard = Self::Finished; diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 33fdc76f8d..576bcea2ce 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -3,13 +3,16 @@ use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; use rand::{distributions::Alphanumeric, Rng}; -use std::collections::{hash_map, HashMap}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::ops::Deref; use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::fs; +use utils::timeout::{timeout_cancellable, TimeoutCancellableError}; use anyhow::Context; use once_cell::sync::Lazy; -use tokio::sync::RwLock; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::*; @@ -23,6 +26,7 @@ use crate::control_plane_client::{ ControlPlaneClient, ControlPlaneGenerationsApi, RetryForeverError, }; use crate::deletion_queue::DeletionQueueClient; +use crate::metrics::TENANT_MANAGER as METRICS; use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; use crate::tenant::delete::DeleteTenantFlow; @@ -47,10 +51,22 @@ use super::TenantSharedResources; /// that way we avoid having to carefully switch a tenant's ingestion etc on and off during /// its lifetime, and we can preserve some important safety invariants like `Tenant` always /// having a properly acquired generation (Secondary doesn't need a generation) -#[derive(Clone)] pub(crate) enum TenantSlot { Attached(Arc), Secondary, + /// In this state, other administrative operations acting on the TenantId should + /// block, or return a retry indicator equivalent to HTTP 503. + InProgress(utils::completion::Barrier), +} + +impl std::fmt::Debug for TenantSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Attached(tenant) => write!(f, "Attached({})", tenant.current_state()), + Self::Secondary => write!(f, "Secondary"), + Self::InProgress(_) => write!(f, "InProgress"), + } + } } impl TenantSlot { @@ -59,14 +75,7 @@ impl TenantSlot { match self { Self::Attached(t) => Some(t), Self::Secondary => None, - } - } - - /// Consume self and return the `Tenant` that was in this slot if attached, else None - fn into_attached(self) -> Option> { - match self { - Self::Attached(t) => Some(t), - Self::Secondary => None, + Self::InProgress(_) => None, } } } @@ -77,7 +86,7 @@ pub(crate) enum TenantsMap { /// [`init_tenant_mgr`] is not done yet. Initializing, /// [`init_tenant_mgr`] is done, all on-disk tenants have been loaded. - /// New tenants can be added using [`tenant_map_insert`]. + /// New tenants can be added using [`tenant_map_acquire_slot`]. Open(HashMap), /// The pageserver has entered shutdown mode via [`shutdown_all_tenants`]. /// Existing tenants are still accessible, but no new tenants can be created. @@ -97,19 +106,17 @@ impl TenantsMap { } } - /// Get the contents of the map at this tenant ID, even if it is in secondary state. - pub(crate) fn get_slot(&self, tenant_id: &TenantId) -> Option<&TenantSlot> { + pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option { match self { TenantsMap::Initializing => None, - TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.get(tenant_id), + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.remove(tenant_id), } } - pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option> { + + pub(crate) fn len(&self) -> usize { match self { - TenantsMap::Initializing => None, - TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => { - m.remove(tenant_id).and_then(TenantSlot::into_attached) - } + TenantsMap::Initializing => 0, + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.len(), } } } @@ -147,7 +154,8 @@ async fn safe_rename_tenant_dir(path: impl AsRef) -> std::io::Result> = Lazy::new(|| RwLock::new(TenantsMap::Initializing)); +static TENANTS: Lazy> = + Lazy::new(|| std::sync::RwLock::new(TenantsMap::Initializing)); /// Create a directory, including parents. This does no fsyncs and makes /// no guarantees about the persistence of the resulting metadata: for @@ -456,8 +464,9 @@ pub async fn init_tenant_mgr( info!("Processed {} local tenants at startup", tenants.len()); - let mut tenants_map = TENANTS.write().await; + let mut tenants_map = TENANTS.write().unwrap(); assert!(matches!(&*tenants_map, &TenantsMap::Initializing)); + METRICS.tenant_slots.set(tenants.len() as u64); *tenants_map = TenantsMap::Open(tenants); Ok(()) } @@ -472,7 +481,7 @@ pub(crate) fn tenant_spawn( resources: TenantSharedResources, location_conf: AttachedTenantConf, init_order: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { @@ -533,12 +542,12 @@ pub(crate) async fn shutdown_all_tenants() { shutdown_all_tenants0(&TENANTS).await } -async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { +async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { use utils::completion; - // Prevent new tenants from being created. - let tenants_to_shut_down = { - let mut m = tenants.write().await; + // Atomically, 1. extract the list of tenants to shut down and 2. prevent creation of new tenants. + let (in_progress_ops, tenants_to_shut_down) = { + let mut m = tenants.write().unwrap(); match &mut *m { TenantsMap::Initializing => { *m = TenantsMap::ShuttingDown(HashMap::default()); @@ -546,9 +555,28 @@ async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { return; } TenantsMap::Open(tenants) => { - let tenants_clone = tenants.clone(); - *m = TenantsMap::ShuttingDown(std::mem::take(tenants)); - tenants_clone + let mut shutdown_state = HashMap::new(); + let mut in_progress_ops = Vec::new(); + let mut tenants_to_shut_down = Vec::new(); + + for (k, v) in tenants.drain() { + match v { + TenantSlot::Attached(t) => { + tenants_to_shut_down.push(t.clone()); + shutdown_state.insert(k, TenantSlot::Attached(t)); + } + TenantSlot::Secondary => { + shutdown_state.insert(k, TenantSlot::Secondary); + } + TenantSlot::InProgress(notify) => { + // InProgress tenants are not visible in TenantsMap::ShuttingDown: we will + // wait for their notifications to fire in this function. + in_progress_ops.push(notify); + } + } + } + *m = TenantsMap::ShuttingDown(shutdown_state); + (in_progress_ops, tenants_to_shut_down) } TenantsMap::ShuttingDown(_) => { // TODO: it is possible that detach and shutdown happen at the same time. as a @@ -559,25 +587,31 @@ async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { } }; + info!( + "Waiting for {} InProgress tenants and {} Attached tenants to shut down", + in_progress_ops.len(), + tenants_to_shut_down.len() + ); + + for barrier in in_progress_ops { + barrier.wait().await; + } + + info!( + "InProgress tenants shut down, waiting for {} Attached tenants to shut down", + tenants_to_shut_down.len() + ); let started_at = std::time::Instant::now(); let mut join_set = JoinSet::new(); - for (tenant_id, tenant) in tenants_to_shut_down { + for tenant in tenants_to_shut_down { + let tenant_id = tenant.get_tenant_id(); join_set.spawn( async move { let freeze_and_flush = true; let res = { let (_guard, shutdown_progress) = completion::channel(); - match tenant { - TenantSlot::Attached(t) => { - t.shutdown(shutdown_progress, freeze_and_flush).await - } - TenantSlot::Secondary => { - // TODO: once secondary mode downloads are implemented, - // ensure they have all stopped before we reach this point. - Ok(()) - } - } + tenant.shutdown(shutdown_progress, freeze_and_flush).await }; if let Err(other_progress) = res { @@ -649,42 +683,35 @@ pub(crate) async fn create_tenant( resources: TenantSharedResources, ctx: &RequestContext, ) -> Result, TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let location_conf = LocationConf::attached_single(tenant_conf, generation); + let location_conf = LocationConf::attached_single(tenant_conf, generation); - // We're holding the tenants lock in write mode while doing local IO. - // If this section ever becomes contentious, introduce a new `TenantState::Creating` - // and do the work in that state. - super::create_tenant_files(conf, &location_conf, &tenant_id).await?; + let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + let tenant_path = super::create_tenant_files(conf, &location_conf, &tenant_id).await?; - // TODO: tenant directory remains on disk if we bail out from here on. - // See https://github.com/neondatabase/neon/issues/4233 + let created_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Create, + ctx, + )?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 - let tenant_path = conf.tenant_path(&tenant_id); + let created_tenant_id = created_tenant.tenant_id(); + if tenant_id != created_tenant_id { + return Err(TenantMapInsertError::Other(anyhow::anyhow!( + "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {created_tenant_id})", + ))); + } - let created_tenant = tenant_spawn( - conf, - tenant_id, - &tenant_path, - resources, - AttachedTenantConf::try_from(location_conf)?, - None, - &TENANTS, - SpawnMode::Create, - ctx, - )?; - // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. - // See https://github.com/neondatabase/neon/issues/4233 + slot_guard.upsert(TenantSlot::Attached(created_tenant.clone()))?; - let crated_tenant_id = created_tenant.tenant_id(); - anyhow::ensure!( - tenant_id == crated_tenant_id, - "loaded created tenant has unexpected tenant id \ - (expect {tenant_id} != actual {crated_tenant_id})", - ); - Ok(created_tenant) - }) - .await + Ok(created_tenant) } #[derive(Debug, thiserror::Error)] @@ -701,7 +728,7 @@ pub(crate) async fn set_new_tenant_config( tenant_id: TenantId, ) -> Result<(), SetNewTenantConfigError> { info!("configuring tenant {tenant_id}"); - let tenant = get_tenant(tenant_id, true).await?; + let tenant = get_tenant(tenant_id, true)?; // This is a legacy API that only operates on attached tenants: the preferred // API to use is the location_config/ endpoint, which lets the caller provide @@ -727,32 +754,49 @@ pub(crate) async fn upsert_location( ) -> Result<(), anyhow::Error> { info!("configuring tenant location {tenant_id} to state {new_location_config:?}"); - let mut existing_tenant = match get_tenant(tenant_id, false).await { - Ok(t) => Some(t), - Err(GetTenantError::NotFound(_)) => None, - Err(e) => anyhow::bail!(e), - }; + // Special case fast-path for updates to Tenant: if our upsert is only updating configuration, + // then we do not need to set the slot to InProgress, we can just call into the + // existng tenant. + { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Write)?; + match (&new_location_config.mode, peek_slot) { + (LocationMode::Attached(attach_conf), Some(TenantSlot::Attached(tenant))) => { + if attach_conf.generation == tenant.generation { + // A transition from Attached to Attached in the same generation, we may + // take our fast path and just provide the updated configuration + // to the tenant. + tenant.set_new_location_config(AttachedTenantConf::try_from( + new_location_config, + )?); - // If we need to shut down a Tenant, do that first - let shutdown_tenant = match (&new_location_config.mode, &existing_tenant) { - (LocationMode::Secondary(_), Some(t)) => Some(t), - (LocationMode::Attached(attach_conf), Some(t)) => { - if attach_conf.generation != t.generation { - Some(t) - } else { - None + // Persist the new config in the background, to avoid holding up any + // locks while we do so. + // TODO + + return Ok(()); + } else { + // Different generations, fall through to general case + } + } + _ => { + // Not an Attached->Attached transition, fall through to general case } } - _ => None, - }; + } - // TODO: currently we risk concurrent operations interfering with the tenant - // while we await shutdown, but we also should not hold the TenantsMap lock - // across the whole operation. Before we start using this function in production, - // a follow-on change will revise how concurrency is handled in TenantsMap. - // (https://github.com/neondatabase/neon/issues/5378) + // General case for upserts to TenantsMap, excluding the case above: we will substitute an + // InProgress value to the slot while we make whatever changes are required. The state for + // the tenant is inaccessible to the outside world while we are doing this, but that is sensible: + // the state is ill-defined while we're in transition. Transitions are async, but fast: we do + // not do significant I/O, and shutdowns should be prompt via cancellation tokens. + let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::Any)?; - if let Some(tenant) = shutdown_tenant { + if let Some(TenantSlot::Attached(tenant)) = slot_guard.get_old_value() { + // The case where we keep a Tenant alive was covered above in the special case + // for Attached->Attached transitions in the same generation. By this point, + // if we see an attached tenant we know it will be discarded and should be + // shut down. let (_guard, progress) = utils::completion::channel(); match tenant.get_attach_mode() { @@ -774,82 +818,62 @@ pub(crate) async fn upsert_location( barrier.wait().await; } } - existing_tenant = None; + slot_guard.drop_old_value().expect("We just shut it down"); } - if let Some(tenant) = existing_tenant { - // Update the existing tenant - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; - tenant.set_new_location_config(AttachedTenantConf::try_from(new_location_config)?); - } else { - // Upsert a fresh TenantSlot into TenantsMap. Do it within the map write lock, - // and re-check that the state of anything we are replacing is as expected. - tenant_map_upsert_slot(tenant_id, |old_value| async move { - if let Some(TenantSlot::Attached(t)) = old_value { - if !matches!(t.current_state(), TenantState::Stopping { .. }) { - anyhow::bail!("Tenant state changed during location configuration update"); - } - } + let tenant_path = conf.tenant_path(&tenant_id); + let new_slot = match &new_location_config.mode { + LocationMode::Secondary(_) => { let tenant_path = conf.tenant_path(&tenant_id); + // Directory doesn't need to be fsync'd because if we crash it can + // safely be recreated next time this tenant location is configured. + unsafe_create_dir_all(&tenant_path) + .await + .with_context(|| format!("Creating {tenant_path}"))?; - let new_slot = match &new_location_config.mode { - LocationMode::Secondary(_) => { - // Directory doesn't need to be fsync'd because if we crash it can - // safely be recreated next time this tenant location is configured. - unsafe_create_dir_all(&tenant_path) - .await - .with_context(|| format!("Creating {tenant_path}"))?; + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; + TenantSlot::Secondary + } + LocationMode::Attached(_attach_config) => { + let timelines_path = conf.timelines_path(&tenant_id); - TenantSlot::Secondary - } - LocationMode::Attached(_attach_config) => { - // FIXME: should avoid doing this disk I/O inside the TenantsMap lock, - // we have the same problem in load_tenant/attach_tenant. Probably - // need a lock in TenantSlot to fix this. - let timelines_path = conf.timelines_path(&tenant_id); + // Directory doesn't need to be fsync'd because we do not depend on + // it to exist after crashes: it may be recreated when tenant is + // re-attached, see https://github.com/neondatabase/neon/issues/5550 + unsafe_create_dir_all(&timelines_path) + .await + .with_context(|| format!("Creating {timelines_path}"))?; - // Directory doesn't need to be fsync'd because we do not depend on - // it to exist after crashes: it may be recreated when tenant is - // re-attached, see https://github.com/neondatabase/neon/issues/5550 - unsafe_create_dir_all(&timelines_path) - .await - .with_context(|| format!("Creating {timelines_path}"))?; + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; + let tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + TenantSharedResources { + broker_client, + remote_storage, + deletion_queue_client, + }, + AttachedTenantConf::try_from(new_location_config)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + )?; - let tenant = tenant_spawn( - conf, - tenant_id, - &tenant_path, - TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client, - }, - AttachedTenantConf::try_from(new_location_config)?, - None, - &TENANTS, - SpawnMode::Normal, - ctx, - )?; + TenantSlot::Attached(tenant) + } + }; - TenantSlot::Attached(tenant) - } - }; + slot_guard.upsert(new_slot)?; - Ok(new_slot) - }) - .await?; - } Ok(()) } @@ -864,34 +888,167 @@ pub(crate) enum GetTenantError { /// is a stuck error state #[error("Tenant is broken: {0}")] Broken(String), + + // Initializing or shutting down: cannot authoritatively say whether we have this tenant + #[error("Tenant map is not available: {0}")] + MapState(#[from] TenantMapError), } /// Gets the tenant from the in-memory data, erroring if it's absent or is not fitting to the query. /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. /// /// This method is cancel-safe. -pub(crate) async fn get_tenant( +pub(crate) fn get_tenant( tenant_id: TenantId, active_only: bool, ) -> Result, GetTenantError> { - let m = TENANTS.read().await; - let tenant = m - .get(&tenant_id) - .ok_or(GetTenantError::NotFound(tenant_id))?; + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read)?; - match tenant.current_state() { - TenantState::Broken { - reason, - backtrace: _, - } if active_only => Err(GetTenantError::Broken(reason)), - TenantState::Active => Ok(Arc::clone(tenant)), - _ => { - if active_only { - Err(GetTenantError::NotActive(tenant_id)) - } else { - Ok(Arc::clone(tenant)) + match peek_slot { + Some(TenantSlot::Attached(tenant)) => match tenant.current_state() { + TenantState::Broken { + reason, + backtrace: _, + } if active_only => Err(GetTenantError::Broken(reason)), + TenantState::Active => Ok(Arc::clone(tenant)), + _ => { + if active_only { + Err(GetTenantError::NotActive(tenant_id)) + } else { + Ok(Arc::clone(tenant)) + } + } + }, + Some(TenantSlot::InProgress(_)) => Err(GetTenantError::NotActive(tenant_id)), + None | Some(TenantSlot::Secondary) => Err(GetTenantError::NotFound(tenant_id)), + } +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum GetActiveTenantError { + /// We may time out either while TenantSlot is InProgress, or while the Tenant + /// is in a non-Active state + #[error( + "Timed out waiting {wait_time:?} for tenant active state. Latest state: {latest_state:?}" + )] + WaitForActiveTimeout { + latest_state: Option, + wait_time: Duration, + }, + + /// The TenantSlot is absent, or in secondary mode + #[error(transparent)] + NotFound(#[from] GetTenantError), + + /// Cancellation token fired while we were waiting + #[error("cancelled")] + Cancelled, + + /// Tenant exists, but is in a state that cannot become active (e.g. Stopping, Broken) + #[error("will not become active. Current state: {0}")] + WillNotBecomeActive(TenantState), +} + +/// Get a [`Tenant`] in its active state. If the tenant_id is currently in [`TenantSlot::InProgress`] +/// state, then wait for up to `timeout`. If the [`Tenant`] is not currently in [`TenantState::Active`], +/// then wait for up to `timeout` (minus however long we waited for the slot). +pub(crate) async fn get_active_tenant_with_timeout( + tenant_id: TenantId, + timeout: Duration, + cancel: &CancellationToken, +) -> Result, GetActiveTenantError> { + enum WaitFor { + Barrier(utils::completion::Barrier), + Tenant(Arc), + } + + let wait_start = Instant::now(); + let deadline = wait_start + timeout; + + let wait_for = { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read) + .map_err(GetTenantError::MapState)?; + match peek_slot { + Some(TenantSlot::Attached(tenant)) => { + match tenant.current_state() { + TenantState::Active => { + // Fast path: we don't need to do any async waiting. + return Ok(tenant.clone()); + } + _ => WaitFor::Tenant(tenant.clone()), + } + } + Some(TenantSlot::Secondary) => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotActive( + tenant_id, + ))) + } + Some(TenantSlot::InProgress(barrier)) => WaitFor::Barrier(barrier.clone()), + None => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotFound( + tenant_id, + ))) } } + }; + + let tenant = match wait_for { + WaitFor::Barrier(barrier) => { + tracing::debug!("Waiting for tenant InProgress state to pass..."); + timeout_cancellable( + deadline.duration_since(Instant::now()), + cancel, + barrier.wait(), + ) + .await + .map_err(|e| match e { + TimeoutCancellableError::Timeout => GetActiveTenantError::WaitForActiveTimeout { + latest_state: None, + wait_time: wait_start.elapsed(), + }, + TimeoutCancellableError::Cancelled => GetActiveTenantError::Cancelled, + })?; + { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_id, TenantSlotPeekMode::Read) + .map_err(GetTenantError::MapState)?; + match peek_slot { + Some(TenantSlot::Attached(tenant)) => tenant.clone(), + _ => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotActive( + tenant_id, + ))) + } + } + } + } + WaitFor::Tenant(tenant) => tenant, + }; + + tracing::debug!("Waiting for tenant to enter active state..."); + match timeout_cancellable( + deadline.duration_since(Instant::now()), + cancel, + tenant.wait_to_become_active(), + ) + .await + { + Ok(Ok(())) => Ok(tenant), + Ok(Err(e)) => Err(e), + Err(TimeoutCancellableError::Timeout) => { + let latest_state = tenant.current_state(); + if latest_state == TenantState::Active { + Ok(tenant) + } else { + Err(GetActiveTenantError::WaitForActiveTimeout { + latest_state: Some(latest_state), + wait_time: timeout, + }) + } + } + Err(TimeoutCancellableError::Cancelled) => Err(GetActiveTenantError::Cancelled), } } @@ -900,7 +1057,33 @@ pub(crate) async fn delete_tenant( remote_storage: Option, tenant_id: TenantId, ) -> Result<(), DeleteTenantError> { - DeleteTenantFlow::run(conf, remote_storage, &TENANTS, tenant_id).await + // We acquire a SlotGuard during this function to protect against concurrent + // changes while the ::prepare phase of DeleteTenantFlow executes, but then + // have to return the Tenant to the map while the background deletion runs. + // + // TODO: refactor deletion to happen outside the lifetime of a Tenant. + // Currently, deletion requires a reference to the tenants map in order to + // keep the Tenant in the map until deletion is complete, and then remove + // it at the end. + // + // See https://github.com/neondatabase/neon/issues/5080 + + let mut slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustExist)?; + + // unwrap is safe because we used MustExist mode when acquiring + let tenant = match slot_guard.get_old_value().as_ref().unwrap() { + TenantSlot::Attached(tenant) => tenant.clone(), + _ => { + // Express "not attached" as equivalent to "not found" + return Err(DeleteTenantError::NotAttached); + } + }; + + let result = DeleteTenantFlow::run(conf, remote_storage, &TENANTS, tenant).await; + + // The Tenant goes back into the map in Stopping state, it will eventually be removed by DeleteTenantFLow + slot_guard.revert(); + result } #[derive(Debug, thiserror::Error)] @@ -917,18 +1100,20 @@ pub(crate) async fn delete_timeline( timeline_id: TimelineId, _ctx: &RequestContext, ) -> Result<(), DeleteTimelineError> { - let tenant = get_tenant(tenant_id, true).await?; + let tenant = get_tenant(tenant_id, true)?; DeleteTimelineFlow::run(&tenant, timeline_id, false).await?; Ok(()) } #[derive(Debug, thiserror::Error)] pub(crate) enum TenantStateError { - #[error("Tenant {0} not found")] - NotFound(TenantId), #[error("Tenant {0} is stopping")] IsStopping(TenantId), #[error(transparent)] + SlotError(#[from] TenantSlotError), + #[error(transparent)] + SlotUpsertError(#[from] TenantSlotUpsertError), + #[error(transparent)] Other(#[from] anyhow::Error), } @@ -967,7 +1152,7 @@ pub(crate) async fn detach_tenant( async fn detach_tenant0( conf: &'static PageServerConf, - tenants: &tokio::sync::RwLock, + tenants: &std::sync::RwLock, tenant_id: TenantId, detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, @@ -988,7 +1173,12 @@ async fn detach_tenant0( // Ignored tenants are not present in memory and will bail the removal from memory operation. // Before returning the error, check for ignored tenant removal case — we only need to clean its local files then. - if detach_ignored && matches!(removal_result, Err(TenantStateError::NotFound(_))) { + if detach_ignored + && matches!( + removal_result, + Err(TenantStateError::SlotError(TenantSlotError::NotFound(_))) + ) + { let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); if tenant_ignore_mark.exists() { info!("Detaching an ignored tenant"); @@ -1011,31 +1201,44 @@ pub(crate) async fn load_tenant( deletion_queue_client: DeletionQueueClient, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let tenant_path = conf.tenant_path(&tenant_id); - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); - if tenant_ignore_mark.exists() { - std::fs::remove_file(&tenant_ignore_mark) - .with_context(|| format!("Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading"))?; - } + let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + let tenant_path = conf.tenant_path(&tenant_id); - let resources = TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client - }; + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); + if tenant_ignore_mark.exists() { + std::fs::remove_file(&tenant_ignore_mark).with_context(|| { + format!( + "Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading" + ) + })?; + } - let mut location_conf = Tenant::load_tenant_config(conf, &tenant_id).map_err( TenantMapInsertError::Other)?; - location_conf.attach_in_generation(generation); - Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + let resources = TenantSharedResources { + broker_client, + remote_storage, + deletion_queue_client, + }; - let new_tenant = tenant_spawn(conf, tenant_id, &tenant_path, resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx) - .with_context(|| { - format!("Failed to schedule tenant processing in path {tenant_path:?}") - })?; + let mut location_conf = + Tenant::load_tenant_config(conf, &tenant_id).map_err(TenantMapInsertError::Other)?; + location_conf.attach_in_generation(generation); - Ok(new_tenant) - }).await?; + Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + + let new_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + ) + .with_context(|| format!("Failed to schedule tenant processing in path {tenant_path:?}"))?; + + slot_guard.upsert(TenantSlot::Attached(new_tenant))?; Ok(()) } @@ -1048,7 +1251,7 @@ pub(crate) async fn ignore_tenant( async fn ignore_tenant0( conf: &'static PageServerConf, - tenants: &tokio::sync::RwLock, + tenants: &std::sync::RwLock, tenant_id: TenantId, ) -> Result<(), TenantStateError> { remove_tenant_from_memory(tenants, tenant_id, async { @@ -1076,7 +1279,7 @@ pub(crate) enum TenantMapListError { /// Get list of tenants, for the mgmt API /// pub(crate) async fn list_tenants() -> Result, TenantMapListError> { - let tenants = TENANTS.read().await; + let tenants = TENANTS.read().unwrap(); let m = match &*tenants { TenantsMap::Initializing => return Err(TenantMapListError::Initializing), TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m, @@ -1085,6 +1288,7 @@ pub(crate) async fn list_tenants() -> Result, Tenan .filter_map(|(id, tenant)| match tenant { TenantSlot::Attached(tenant) => Some((*id, tenant.current_state())), TenantSlot::Secondary => None, + TenantSlot::InProgress(_) => None, }) .collect()) } @@ -1101,101 +1305,436 @@ pub(crate) async fn attach_tenant( resources: TenantSharedResources, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let location_conf = LocationConf::attached_single(tenant_conf, generation); - let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; - // TODO: tenant directory remains on disk if we bail out from here on. - // See https://github.com/neondatabase/neon/issues/4233 + let slot_guard = tenant_map_acquire_slot(&tenant_id, TenantSlotAcquireMode::MustNotExist)?; + let location_conf = LocationConf::attached_single(tenant_conf, generation); + let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; + // TODO: tenant directory remains on disk if we bail out from here on. + // See https://github.com/neondatabase/neon/issues/4233 - let attached_tenant = tenant_spawn(conf, tenant_id, &tenant_dir, - resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx)?; - // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. - // See https://github.com/neondatabase/neon/issues/4233 + let attached_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_dir, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + )?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 - let attached_tenant_id = attached_tenant.tenant_id(); - anyhow::ensure!( - tenant_id == attached_tenant_id, + let attached_tenant_id = attached_tenant.tenant_id(); + if tenant_id != attached_tenant_id { + return Err(TenantMapInsertError::Other(anyhow::anyhow!( "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {attached_tenant_id})", - ); - Ok(attached_tenant) - }) - .await?; + ))); + } + + slot_guard.upsert(TenantSlot::Attached(attached_tenant))?; Ok(()) } #[derive(Debug, thiserror::Error)] pub(crate) enum TenantMapInsertError { - #[error("tenant map is still initializing")] - StillInitializing, - #[error("tenant map is shutting down")] - ShuttingDown, - #[error("tenant {0} already exists, state: {1:?}")] - TenantAlreadyExists(TenantId, TenantState), - #[error("tenant {0} already exists in secondary state")] - TenantExistsSecondary(TenantId), + #[error(transparent)] + SlotError(#[from] TenantSlotError), + #[error(transparent)] + SlotUpsertError(#[from] TenantSlotUpsertError), #[error(transparent)] Other(#[from] anyhow::Error), } -/// Give the given closure access to the tenants map entry for the given `tenant_id`, iff that -/// entry is vacant. The closure is responsible for creating the tenant object and inserting -/// it into the tenants map through the vacnt entry that it receives as argument. +/// Superset of TenantMapError: issues that can occur when acquiring a slot +/// for a particular tenant ID. +#[derive(Debug, thiserror::Error)] +pub enum TenantSlotError { + /// When acquiring a slot with the expectation that the tenant already exists. + #[error("Tenant {0} not found")] + NotFound(TenantId), + + /// When acquiring a slot with the expectation that the tenant does not already exist. + #[error("tenant {0} already exists, state: {1:?}")] + AlreadyExists(TenantId, TenantState), + + #[error("tenant {0} already exists in but is not attached")] + Conflict(TenantId), + + // Tried to read a slot that is currently being mutated by another administrative + // operation. + #[error("tenant has a state change in progress, try again later")] + InProgress, + + #[error(transparent)] + MapState(#[from] TenantMapError), +} + +/// Superset of TenantMapError: issues that can occur when using a SlotGuard +/// to insert a new value. +#[derive(Debug, thiserror::Error)] +pub enum TenantSlotUpsertError { + /// An error where the slot is in an unexpected state, indicating a code bug + #[error("Internal error updating Tenant")] + InternalError(Cow<'static, str>), + + #[error(transparent)] + MapState(#[from] TenantMapError), +} + +#[derive(Debug)] +enum TenantSlotDropError { + /// It is only legal to drop a TenantSlot if its contents are fully shut down + NotShutdown, +} + +/// Errors that can happen any time we are walking the tenant map to try and acquire +/// the TenantSlot for a particular tenant. +#[derive(Debug, thiserror::Error)] +pub enum TenantMapError { + // Tried to read while initializing + #[error("tenant map is still initializing")] + StillInitializing, + + // Tried to read while shutting down + #[error("tenant map is shutting down")] + ShuttingDown, +} + +/// Guards a particular tenant_id's content in the TenantsMap. While this +/// structure exists, the TenantsMap will contain a [`TenantSlot::InProgress`] +/// for this tenant, which acts as a marker for any operations targeting +/// this tenant to retry later, or wait for the InProgress state to end. /// -/// NB: the closure should return quickly because the current implementation of tenants map -/// serializes access through an `RwLock`. -async fn tenant_map_insert( +/// This structure enforces the important invariant that we do not have overlapping +/// tasks that will try use local storage for a the same tenant ID: we enforce that +/// the previous contents of a slot have been shut down before the slot can be +/// left empty or used for something else +/// +/// Holders of a SlotGuard should explicitly dispose of it, using either `upsert` +/// to provide a new value, or `revert` to put the slot back into its initial +/// state. If the SlotGuard is dropped without calling either of these, then +/// we will leave the slot empty if our `old_value` is already shut down, else +/// we will replace the slot with `old_value` (equivalent to doing a revert). +/// +/// The `old_value` may be dropped before the SlotGuard is dropped, by calling +/// `drop_old_value`. It is an error to call this without shutting down +/// the conents of `old_value`. +pub struct SlotGuard { tenant_id: TenantId, - insert_fn: F, -) -> Result, TenantMapInsertError> -where - F: FnOnce() -> R, - R: std::future::Future>>, -{ - let mut guard = TENANTS.write().await; - let m = match &mut *guard { - TenantsMap::Initializing => return Err(TenantMapInsertError::StillInitializing), - TenantsMap::ShuttingDown(_) => return Err(TenantMapInsertError::ShuttingDown), - TenantsMap::Open(m) => m, - }; - match m.entry(tenant_id) { - hash_map::Entry::Occupied(e) => match e.get() { - TenantSlot::Attached(t) => Err(TenantMapInsertError::TenantAlreadyExists( - tenant_id, - t.current_state(), - )), - TenantSlot::Secondary => Err(TenantMapInsertError::TenantExistsSecondary(tenant_id)), - }, - hash_map::Entry::Vacant(v) => match insert_fn().await { - Ok(tenant) => { - v.insert(TenantSlot::Attached(tenant.clone())); - Ok(tenant) + old_value: Option, + upserted: bool, + + /// [`TenantSlot::InProgress`] carries the corresponding Barrier: it will + /// release any waiters as soon as this SlotGuard is dropped. + _completion: utils::completion::Completion, +} + +unsafe impl Send for SlotGuard {} +unsafe impl Sync for SlotGuard {} + +impl SlotGuard { + fn new( + tenant_id: TenantId, + old_value: Option, + completion: utils::completion::Completion, + ) -> Self { + Self { + tenant_id, + old_value, + upserted: false, + _completion: completion, + } + } + + /// Take any value that was present in the slot before we acquired ownership + /// of it: in state transitions, this will be the old state. + fn get_old_value(&mut self) -> &Option { + &self.old_value + } + + /// Emplace a new value in the slot. This consumes the guard, and after + /// returning, the slot is no longer protected from concurrent changes. + fn upsert(mut self, new_value: TenantSlot) -> Result<(), TenantSlotUpsertError> { + if !self.old_value_is_shutdown() { + // This is a bug: callers should never try to drop an old value without + // shutting it down + return Err(TenantSlotUpsertError::InternalError( + "Old TenantSlot value not shut down".into(), + )); + } + + let replaced = { + let mut locked = TENANTS.write().unwrap(); + + if let TenantSlot::InProgress(_) = new_value { + // It is never expected to try and upsert InProgress via this path: it should + // only be written via the tenant_map_acquire_slot path. If we hit this it's a bug. + return Err(TenantSlotUpsertError::InternalError( + "Attempt to upsert an InProgress state".into(), + )); } - Err(e) => Err(TenantMapInsertError::Other(e)), - }, + + let m = match &mut *locked { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing.into()), + TenantsMap::ShuttingDown(_) => { + return Err(TenantMapError::ShuttingDown.into()); + } + TenantsMap::Open(m) => m, + }; + + let replaced = m.insert(self.tenant_id, new_value); + self.upserted = true; + + METRICS.tenant_slots.set(m.len() as u64); + + replaced + }; + + // Sanity check: on an upsert we should always be replacing an InProgress marker + match replaced { + Some(TenantSlot::InProgress(_)) => { + // Expected case: we find our InProgress in the map: nothing should have + // replaced it because the code that acquires slots will not grant another + // one for the same TenantId. + Ok(()) + } + None => { + METRICS.unexpected_errors.inc(); + error!( + tenant_id = %self.tenant_id, + "Missing InProgress marker during tenant upsert, this is a bug." + ); + Err(TenantSlotUpsertError::InternalError( + "Missing InProgress marker during tenant upsert".into(), + )) + } + Some(slot) => { + METRICS.unexpected_errors.inc(); + error!(tenant_id=%self.tenant_id, "Unexpected contents of TenantSlot during upsert, this is a bug. Contents: {:?}", slot); + Err(TenantSlotUpsertError::InternalError( + "Unexpected contents of TenantSlot".into(), + )) + } + } + } + + /// Replace the InProgress slot with whatever was in the guard when we started + fn revert(mut self) { + if let Some(value) = self.old_value.take() { + match self.upsert(value) { + Err(TenantSlotUpsertError::InternalError(_)) => { + // We already logged the error, nothing else we can do. + } + Err(TenantSlotUpsertError::MapState(_)) => { + // If the map is shutting down, we need not replace anything + } + Ok(()) => {} + } + } + } + + /// We may never drop our old value until it is cleanly shut down: otherwise we might leave + /// rogue background tasks that would write to the local tenant directory that this guard + /// is responsible for protecting + fn old_value_is_shutdown(&self) -> bool { + match self.old_value.as_ref() { + Some(TenantSlot::Attached(tenant)) => { + // TODO: PR #5711 will add a gate that enables properly checking that + // shutdown completed. + matches!( + tenant.current_state(), + TenantState::Stopping { .. } | TenantState::Broken { .. } + ) + } + Some(TenantSlot::Secondary) => { + // TODO: when adding secondary mode tenants, this will check for shutdown + // in the same way that we do for `Tenant` above + true + } + Some(TenantSlot::InProgress(_)) => { + // A SlotGuard cannot be constructed for a slot that was already InProgress + unreachable!() + } + None => true, + } + } + + /// The guard holder is done with the old value of the slot: they are obliged to already + /// shut it down before we reach this point. + fn drop_old_value(&mut self) -> Result<(), TenantSlotDropError> { + if !self.old_value_is_shutdown() { + Err(TenantSlotDropError::NotShutdown) + } else { + self.old_value.take(); + Ok(()) + } } } -async fn tenant_map_upsert_slot<'a, F, R>( - tenant_id: TenantId, - upsert_fn: F, -) -> Result<(), TenantMapInsertError> -where - F: FnOnce(Option) -> R, - R: std::future::Future>, -{ - let mut guard = TENANTS.write().await; - let m = match &mut *guard { - TenantsMap::Initializing => return Err(TenantMapInsertError::StillInitializing), - TenantsMap::ShuttingDown(_) => return Err(TenantMapInsertError::ShuttingDown), +impl Drop for SlotGuard { + fn drop(&mut self) { + if self.upserted { + return; + } + // Our old value is already shutdown, or it never existed: it is safe + // for us to fully release the TenantSlot back into an empty state + + let mut locked = TENANTS.write().unwrap(); + + let m = match &mut *locked { + TenantsMap::Initializing => { + // There is no map, this should never happen. + return; + } + TenantsMap::ShuttingDown(_) => { + // When we transition to shutdown, InProgress elements are removed + // from the map, so we do not need to clean up our Inprogress marker. + // See [`shutdown_all_tenants0`] + return; + } + TenantsMap::Open(m) => m, + }; + + use std::collections::hash_map::Entry; + match m.entry(self.tenant_id) { + Entry::Occupied(mut entry) => { + if !matches!(entry.get(), TenantSlot::InProgress(_)) { + METRICS.unexpected_errors.inc(); + error!(tenant_id=%self.tenant_id, "Unexpected contents of TenantSlot during drop, this is a bug. Contents: {:?}", entry.get()); + } + + if self.old_value_is_shutdown() { + entry.remove(); + } else { + entry.insert(self.old_value.take().unwrap()); + } + } + Entry::Vacant(_) => { + METRICS.unexpected_errors.inc(); + error!( + tenant_id = %self.tenant_id, + "Missing InProgress marker during SlotGuard drop, this is a bug." + ); + } + } + + METRICS.tenant_slots.set(m.len() as u64); + } +} + +enum TenantSlotPeekMode { + /// In Read mode, peek will be permitted to see the slots even if the pageserver is shutting down + Read, + /// In Write mode, trying to peek at a slot while the pageserver is shutting down is an error + Write, +} + +fn tenant_map_peek_slot<'a>( + tenants: &'a std::sync::RwLockReadGuard<'a, TenantsMap>, + tenant_id: &TenantId, + mode: TenantSlotPeekMode, +) -> Result, TenantMapError> { + let m = match tenants.deref() { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing), + TenantsMap::ShuttingDown(m) => match mode { + TenantSlotPeekMode::Read => m, + TenantSlotPeekMode::Write => { + return Err(TenantMapError::ShuttingDown); + } + }, TenantsMap::Open(m) => m, }; - match upsert_fn(m.remove(&tenant_id)).await { - Ok(upsert_val) => { - m.insert(tenant_id, upsert_val); - Ok(()) + Ok(m.get(tenant_id)) +} + +enum TenantSlotAcquireMode { + /// Acquire the slot irrespective of current state, or whether it already exists + Any, + /// Return an error if trying to acquire a slot and it doesn't already exist + MustExist, + /// Return an error if trying to acquire a slot and it already exists + MustNotExist, +} + +fn tenant_map_acquire_slot( + tenant_id: &TenantId, + mode: TenantSlotAcquireMode, +) -> Result { + tenant_map_acquire_slot_impl(tenant_id, &TENANTS, mode) +} + +fn tenant_map_acquire_slot_impl( + tenant_id: &TenantId, + tenants: &std::sync::RwLock, + mode: TenantSlotAcquireMode, +) -> Result { + use TenantSlotAcquireMode::*; + METRICS.tenant_slot_writes.inc(); + + let mut locked = tenants.write().unwrap(); + let span = tracing::info_span!("acquire_slot", %tenant_id); + let _guard = span.enter(); + + let m = match &mut *locked { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing.into()), + TenantsMap::ShuttingDown(_) => return Err(TenantMapError::ShuttingDown.into()), + TenantsMap::Open(m) => m, + }; + + use std::collections::hash_map::Entry; + let entry = m.entry(*tenant_id); + match entry { + Entry::Vacant(v) => match mode { + MustExist => { + tracing::debug!("Vacant && MustExist: return NotFound"); + Err(TenantSlotError::NotFound(*tenant_id)) + } + _ => { + let (completion, barrier) = utils::completion::channel(); + v.insert(TenantSlot::InProgress(barrier)); + tracing::debug!("Vacant, inserted InProgress"); + Ok(SlotGuard::new(*tenant_id, None, completion)) + } + }, + Entry::Occupied(mut o) => { + // Apply mode-driven checks + match (o.get(), mode) { + (TenantSlot::InProgress(_), _) => { + tracing::debug!("Occupied, failing for InProgress"); + Err(TenantSlotError::InProgress) + } + (slot, MustNotExist) => match slot { + TenantSlot::Attached(tenant) => { + tracing::debug!("Attached && MustNotExist, return AlreadyExists"); + Err(TenantSlotError::AlreadyExists( + *tenant_id, + tenant.current_state(), + )) + } + _ => { + // FIXME: the AlreadyExists error assumes that we have a Tenant + // to get the state from + tracing::debug!("Occupied & MustNotExist, return AlreadyExists"); + Err(TenantSlotError::AlreadyExists( + *tenant_id, + TenantState::Broken { + reason: "Present but not attached".to_string(), + backtrace: "".to_string(), + }, + )) + } + }, + _ => { + // Happy case: the slot was not in any state that violated our mode + let (completion, barrier) = utils::completion::channel(); + let old_value = o.insert(TenantSlot::InProgress(barrier)); + tracing::debug!("Occupied, replaced with InProgress"); + Ok(SlotGuard::new(*tenant_id, Some(old_value), completion)) + } + } } - Err(e) => Err(TenantMapInsertError::Other(e)), } } @@ -1204,7 +1743,7 @@ where /// If the cleanup fails, tenant will stay in memory in [`TenantState::Broken`] state, and another removal /// operation would be needed to remove it. async fn remove_tenant_from_memory( - tenants: &tokio::sync::RwLock, + tenants: &std::sync::RwLock, tenant_id: TenantId, tenant_cleanup: F, ) -> Result @@ -1213,20 +1752,14 @@ where { use utils::completion; - // It's important to keep the tenant in memory after the final cleanup, to avoid cleanup races. - // The exclusive lock here ensures we don't miss the tenant state updates before trying another removal. - // tenant-wde cleanup operations may take some time (removing the entire tenant directory), we want to - // avoid holding the lock for the entire process. - let tenant = { - match tenants - .write() - .await - .get_slot(&tenant_id) - .ok_or(TenantStateError::NotFound(tenant_id))? - { - TenantSlot::Attached(t) => Some(t.clone()), - TenantSlot::Secondary => None, - } + let mut slot_guard = + tenant_map_acquire_slot_impl(&tenant_id, tenants, TenantSlotAcquireMode::MustExist)?; + + // The SlotGuard allows us to manipulate the Tenant object without fear of some + // concurrent API request doing something else for the same tenant ID. + let attached_tenant = match slot_guard.get_old_value() { + Some(TenantSlot::Attached(t)) => Some(t), + _ => None, }; // allow pageserver shutdown to await for our completion @@ -1234,7 +1767,7 @@ where // If the tenant was attached, shut it down gracefully. For secondary // locations this part is not necessary - match tenant { + match &attached_tenant { Some(attached_tenant) => { // whenever we remove a tenant from memory, we don't want to flush and wait for upload let freeze_and_flush = false; @@ -1246,6 +1779,7 @@ where Err(_other) => { // if pageserver shutdown or other detach/ignore is already ongoing, we don't want to // wait for it but return an error right away because these are distinct requests. + slot_guard.revert(); return Err(TenantStateError::IsStopping(tenant_id)); } } @@ -1260,23 +1794,21 @@ where .with_context(|| format!("Failed to run cleanup for tenant {tenant_id}")) { Ok(hook_value) => { - let mut tenants_accessor = tenants.write().await; - if tenants_accessor.remove(&tenant_id).is_none() { - warn!("Tenant {tenant_id} got removed from memory before operation finished"); - } + // Success: drop the old TenantSlot::Attached. + slot_guard + .drop_old_value() + .expect("We just called shutdown"); + Ok(hook_value) } Err(e) => { - let tenants_accessor = tenants.read().await; - match tenants_accessor.get(&tenant_id) { - Some(tenant) => { - tenant.set_broken(e.to_string()).await; - } - None => { - warn!("Tenant {tenant_id} got removed from memory"); - return Err(TenantStateError::NotFound(tenant_id)); - } + // If we had a Tenant, set it to Broken and put it back in the TenantsMap + if let Some(attached_tenant) = attached_tenant { + attached_tenant.set_broken(e.to_string()).await; } + // Leave the broken tenant in the map + slot_guard.revert(); + Err(TenantStateError::Other(e)) } } @@ -1293,7 +1825,7 @@ pub(crate) async fn immediate_gc( gc_req: TimelineGcRequest, ctx: &RequestContext, ) -> Result>, ApiError> { - let guard = TENANTS.read().await; + let guard = TENANTS.read().unwrap(); let tenant = guard .get(&tenant_id) .map(Arc::clone) @@ -1346,14 +1878,12 @@ mod tests { use super::{super::harness::TenantHarness, TenantsMap}; - #[tokio::test(start_paused = true)] - async fn shutdown_joins_remove_tenant_from_memory() { - // the test is a bit ugly with the lockstep together with spawned tasks. the aim is to make - // sure `shutdown_all_tenants0` per-tenant processing joins in any active - // remove_tenant_from_memory calls, which is enforced by making the operation last until - // we've ran `shutdown_all_tenants0` for a long time. + #[tokio::test] + async fn shutdown_awaits_in_progress_tenant() { + // Test that if an InProgress tenant is in the map during shutdown, the shutdown will gracefully + // wait for it to complete before proceeding. - let (t, _ctx) = TenantHarness::create("shutdown_joins_detach") + let (t, _ctx) = TenantHarness::create("shutdown_awaits_in_progress_tenant") .unwrap() .load() .await; @@ -1366,13 +1896,14 @@ mod tests { let _e = info_span!("testing", tenant_id = %id).entered(); let tenants = HashMap::from([(id, TenantSlot::Attached(t.clone()))]); - let tenants = Arc::new(tokio::sync::RwLock::new(TenantsMap::Open(tenants))); + let tenants = Arc::new(std::sync::RwLock::new(TenantsMap::Open(tenants))); + + // Invoke remove_tenant_from_memory with a cleanup hook that blocks until we manually + // permit it to proceed: that will stick the tenant in InProgress let (until_cleanup_completed, can_complete_cleanup) = utils::completion::channel(); let (until_cleanup_started, cleanup_started) = utils::completion::channel(); - - // start a "detaching operation", which will take a while, until can_complete_cleanup - let cleanup_task = { + let mut remove_tenant_from_memory_task = { let jh = tokio::spawn({ let tenants = tenants.clone(); async move { @@ -1391,12 +1922,6 @@ mod tests { jh }; - let mut cleanup_progress = std::pin::pin!(t - .shutdown(utils::completion::Barrier::default(), false) - .await - .unwrap_err() - .wait()); - let mut shutdown_task = { let (until_shutdown_started, shutdown_started) = utils::completion::channel(); @@ -1409,37 +1934,17 @@ mod tests { shutdown_task }; - // if the joining in is removed from shutdown_all_tenants0, the shutdown_task should always - // get to complete within timeout and fail the test. it is expected to continue awaiting - // until completion or SIGKILL during normal shutdown. - // - // the timeout is long to cover anything that shutdown_task could be doing, but it is - // handled instantly because we use tokio's time pausing in this test. 100s is much more than - // what we get from systemd on shutdown (10s). - let long_time = std::time::Duration::from_secs(100); + let long_time = std::time::Duration::from_secs(15); tokio::select! { - _ = &mut shutdown_task => unreachable!("shutdown must continue, until_cleanup_completed is not dropped"), - _ = &mut cleanup_progress => unreachable!("cleanup progress must continue, until_cleanup_completed is not dropped"), + _ = &mut shutdown_task => unreachable!("shutdown should block on remove_tenant_from_memory completing"), + _ = &mut remove_tenant_from_memory_task => unreachable!("remove_tenant_from_memory_task should not complete until explicitly unblocked"), _ = tokio::time::sleep(long_time) => {}, } - // allow the remove_tenant_from_memory and thus eventually the shutdown to continue drop(until_cleanup_completed); - let (je, ()) = tokio::join!(shutdown_task, cleanup_progress); - je.expect("Tenant::shutdown shutdown not have panicked"); - cleanup_task - .await - .expect("no panicking") - .expect("remove_tenant_from_memory failed"); - - futures::future::poll_immediate( - t.shutdown(utils::completion::Barrier::default(), false) - .await - .unwrap_err() - .wait(), - ) - .await - .expect("the stopping progress must still be complete"); + // Now that we allow it to proceed, shutdown should complete immediately + remove_tenant_from_memory_task.await.unwrap().unwrap(); + shutdown_task.await.unwrap(); } } diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 52c53a5c3b..e3aad22e40 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -341,20 +341,7 @@ impl Timeline { // Make one of the tenant's timelines draw the short straw and run the calculation. // The others wait until the calculation is done so that they take into account the // imitated accesses that the winner made. - // - // It is critical we are responsive to cancellation here. Otherwise, we deadlock with - // tenant deletion (holds TENANTS in read mode) any other task that attempts to - // acquire TENANTS in write mode before we here call get_tenant. - // See https://github.com/neondatabase/neon/issues/5284. - let res = tokio::select! { - _ = cancel.cancelled() => { - return ControlFlow::Break(()); - } - res = crate::tenant::mgr::get_tenant(self.tenant_id, true) => { - res - } - }; - let tenant = match res { + let tenant = match crate::tenant::mgr::get_tenant(self.tenant_id, true) { Ok(t) => t, Err(_) => { return ControlFlow::Break(()); diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 740c05dcea..d00bf8c4d8 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -626,6 +626,8 @@ class NeonEnvBuilder: sk.stop(immediate=True) for pageserver in self.env.pageservers: + pageserver.assert_no_metric_errors() + pageserver.stop(immediate=True) if self.env.attachment_service is not None: @@ -1784,6 +1786,21 @@ class NeonPageserver(PgProtocol): assert not errors + def assert_no_metric_errors(self): + """ + Certain metrics should _always_ be zero: they track conditions that indicate a bug. + """ + if not self.running: + log.info(f"Skipping metrics check on pageserver {self.id}, it is not running") + return + + for metric in [ + "pageserver_tenant_manager_unexpected_errors_total", + "pageserver_deletion_queue_unexpected_errors_total", + ]: + value = self.http_client().get_metric_value(metric) + assert value == 0, f"Nonzero {metric} == {value}" + def log_contains(self, pattern: str) -> Optional[str]: """Check that the pageserver log contains a line that matches the given regex""" logfile = open(os.path.join(self.workdir, "pageserver.log"), "r") diff --git a/test_runner/regress/test_neon_cli.py b/test_runner/regress/test_neon_cli.py index 1b3984583a..de18ea0e6b 100644 --- a/test_runner/regress/test_neon_cli.py +++ b/test_runner/regress/test_neon_cli.py @@ -134,6 +134,9 @@ def test_cli_start_stop(neon_env_builder: NeonEnvBuilder): env.neon_cli.pageserver_stop(env.pageserver.id) env.neon_cli.safekeeper_stop() + # Keep NeonEnv state up to date, it usually owns starting/stopping services + env.pageserver.running = False + # Default start res = env.neon_cli.raw_cli(["start"]) res.check_returncode() @@ -155,6 +158,10 @@ def test_cli_start_stop_multi(neon_env_builder: NeonEnvBuilder): env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID) env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 1) + # Keep NeonEnv state up to date, it usually owns starting/stopping services + env.pageservers[0].running = False + env.pageservers[1].running = False + # Addressing a nonexistent ID throws with pytest.raises(RuntimeError): env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 100) diff --git a/test_runner/regress/test_pageserver_restart.py b/test_runner/regress/test_pageserver_restart.py index 4c51155236..fa1b131537 100644 --- a/test_runner/regress/test_pageserver_restart.py +++ b/test_runner/regress/test_pageserver_restart.py @@ -20,6 +20,8 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool) endpoint = env.endpoints.create_start("main") pageserver_http = env.pageserver.http_client() + assert pageserver_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + pg_conn = endpoint.connect() cur = pg_conn.cursor() @@ -52,6 +54,9 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool) env.pageserver.stop() env.pageserver.start() + # We reloaded our tenant + assert pageserver_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + cur.execute("SELECT count(*) FROM foo") assert cur.fetchone() == (100000,) diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index ae77197088..6e35890922 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -63,6 +63,9 @@ def test_tenant_delete_smoke( conf=MANY_SMALL_LAYERS_TENANT_CONFIG, ) + # Default tenant and the one we created + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 2 + # create two timelines one being the parent of another parent = None for timeline in ["first", "second"]: @@ -88,7 +91,9 @@ def test_tenant_delete_smoke( iterations = poll_for_remote_storage_iterations(remote_storage_kind) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 2 tenant_delete_wait_completed(ps_http, tenant_id, iterations) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 1 tenant_path = env.pageserver.tenant_dir(tenant_id) assert not tenant_path.exists() @@ -104,6 +109,9 @@ def test_tenant_delete_smoke( ), ) + # Deletion updates the tenant count: the one default tenant remains + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + class Check(enum.Enum): RETRY_WITHOUT_RESTART = enum.auto() diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 11e8a80e1d..03e78dda2b 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -26,6 +26,16 @@ from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until from prometheus_client.samples import Sample +# In tests that overlap endpoint activity with tenant attach/detach, there are +# a variety of warnings that the page service may emit when it cannot acquire +# an active tenant to serve a request +PERMIT_PAGE_SERVICE_ERRORS = [ + ".*page_service.*Tenant .* not found", + ".*page_service.*Tenant .* is not active", + ".*page_service.*cancelled", + ".*page_service.*will not become active.*", +] + def do_gc_target( pageserver_http: PageserverHttpClient, tenant_id: TenantId, timeline_id: TimelineId @@ -60,12 +70,7 @@ def test_tenant_reattach( # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: with endpoint.cursor() as cur: @@ -235,10 +240,7 @@ def test_tenant_reattach_while_busy( # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) endpoint = env.endpoints.create_start("main", tenant_id=tenant_id) @@ -259,7 +261,7 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() - env.pageserver.allowed_errors.append(".*NotFound: Tenant .*") + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # first check for non existing tenant tenant_id = TenantId.generate() @@ -271,19 +273,9 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): assert excinfo.value.status_code == 404 - # the error will be printed to the log too - env.pageserver.allowed_errors.append(".*NotFound: tenant *") - # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) - # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -345,12 +337,7 @@ def test_tenant_detach_ignored_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -401,12 +388,7 @@ def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -453,12 +435,7 @@ def test_detach_while_attaching( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # Create table, and insert some rows. Make it big enough that it doesn't fit in # shared_buffers, otherwise the SELECT after restart will just return answer @@ -593,12 +570,7 @@ def test_ignored_tenant_download_missing_layers(neon_env_builder: NeonEnvBuilder tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) data_id = 1 data_secret = "very secret secret" @@ -649,12 +621,7 @@ def test_load_attach_negatives(neon_env_builder: NeonEnvBuilder): tenant_id = env.initial_tenant - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) env.pageserver.allowed_errors.append(".*tenant .*? already exists, state:.*") with pytest.raises( @@ -693,12 +660,7 @@ def test_ignore_while_attaching( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) data_id = 1 data_secret = "very secret secret" From b09a8517059a74499c1fdfe37f9c2842a7cbb229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 6 Nov 2023 16:16:55 +0100 Subject: [PATCH 28/29] Make azure blob storage not do extra metadata requests (#5777) Load the metadata from the returned `GetBlobResponse` and avoid downloading it via a separate request. As it turns out, the SDK does return the metadata: https://github.com/Azure/azure-sdk-for-rust/issues/1439 . This PR will reduce the number of requests to Azure caused by downloads. Fixes #5571 --- libs/remote_storage/src/azure_blob.rs | 43 +++++++-------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index f0521d27c5..ae08e9b171 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -1,21 +1,18 @@ //! Azure Blob Storage wrapper +use std::collections::HashMap; use std::env; use std::num::NonZeroU32; use std::sync::Arc; -use std::{borrow::Cow, collections::HashMap, io::Cursor}; +use std::{borrow::Cow, io::Cursor}; use super::REMOTE_STORAGE_PREFIX_SEPARATOR; use anyhow::Result; use azure_core::request_options::{MaxResults, Metadata, Range}; -use azure_core::Header; use azure_identity::DefaultAzureCredential; use azure_storage::StorageCredentials; use azure_storage_blobs::prelude::ClientBuilder; -use azure_storage_blobs::{ - blob::operations::GetBlobBuilder, - prelude::{BlobClient, ContainerClient}, -}; +use azure_storage_blobs::{blob::operations::GetBlobBuilder, prelude::ContainerClient}; use futures_util::StreamExt; use http_types::StatusCode; use tokio::io::AsyncRead; @@ -112,16 +109,19 @@ impl AzureBlobStorage { async fn download_for_builder( &self, - metadata: StorageMetadata, builder: GetBlobBuilder, ) -> Result { let mut response = builder.into_stream(); + let mut metadata = HashMap::new(); // TODO give proper streaming response instead of buffering into RAM // https://github.com/neondatabase/neon/issues/5563 let mut buf = Vec::new(); while let Some(part) = response.next().await { let part = part.map_err(to_download_error)?; + if let Some(blob_meta) = part.blob.metadata { + metadata.extend(blob_meta.iter().map(|(k, v)| (k.to_owned(), v.to_owned()))); + } let data = part .data .collect() @@ -131,28 +131,9 @@ impl AzureBlobStorage { } Ok(Download { download_stream: Box::pin(Cursor::new(buf)), - metadata: Some(metadata), + metadata: Some(StorageMetadata(metadata)), }) } - // TODO get rid of this function once we have metadata included in the response - // https://github.com/Azure/azure-sdk-for-rust/issues/1439 - async fn get_metadata( - &self, - blob_client: &BlobClient, - ) -> Result { - let builder = blob_client.get_metadata(); - - let response = builder.into_future().await.map_err(to_download_error)?; - let mut map = HashMap::new(); - - for md in response.metadata.iter() { - map.insert( - md.name().as_str().to_string(), - md.value().as_str().to_string(), - ); - } - Ok(StorageMetadata(map)) - } async fn permit(&self, kind: RequestKind) -> tokio::sync::SemaphorePermit<'_> { self.concurrency_limiter @@ -269,11 +250,9 @@ impl RemoteStorage for AzureBlobStorage { let _permit = self.permit(RequestKind::Get).await; let blob_client = self.client.blob_client(self.relative_path_to_name(from)); - let metadata = self.get_metadata(&blob_client).await?; - let builder = blob_client.get(); - self.download_for_builder(metadata, builder).await + self.download_for_builder(builder).await } async fn download_byte_range( @@ -285,8 +264,6 @@ impl RemoteStorage for AzureBlobStorage { let _permit = self.permit(RequestKind::Get).await; let blob_client = self.client.blob_client(self.relative_path_to_name(from)); - let metadata = self.get_metadata(&blob_client).await?; - let mut builder = blob_client.get(); if let Some(end_exclusive) = end_exclusive { @@ -301,7 +278,7 @@ impl RemoteStorage for AzureBlobStorage { builder = builder.range(Range::new(start_inclusive, end_exclusive)); } - self.download_for_builder(metadata, builder).await + self.download_for_builder(builder).await } async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> { From ad5b02e17518445d390d6dab523d4efd13db2c69 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 6 Nov 2023 17:44:44 +0000 Subject: [PATCH 29/29] proxy: remove unsafe (#5805) ## Problem `unsafe {}` ## Summary of changes `CStr` has a method to parse the bytes up to a null byte, so we don't have to do it ourselves. --- proxy/src/parse.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/proxy/src/parse.rs b/proxy/src/parse.rs index cbd48d91e9..0d03574901 100644 --- a/proxy/src/parse.rs +++ b/proxy/src/parse.rs @@ -3,10 +3,9 @@ use std::ffi::CStr; pub fn split_cstr(bytes: &[u8]) -> Option<(&CStr, &[u8])> { - let pos = bytes.iter().position(|&x| x == 0)?; - let (cstr, other) = bytes.split_at(pos + 1); - // SAFETY: we've already checked that there's a terminator - Some((unsafe { CStr::from_bytes_with_nul_unchecked(cstr) }, other)) + let cstr = CStr::from_bytes_until_nul(bytes).ok()?; + let (_, other) = bytes.split_at(cstr.to_bytes_with_nul().len()); + Some((cstr, other)) } /// See .