diff --git a/.config/hakari.toml b/.config/hakari.toml index 15b939e86f..9913ecc9c0 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -22,5 +22,11 @@ platforms = [ # "x86_64-pc-windows-msvc", ] +[final-excludes] +# vm_monitor benefits from the same Cargo.lock as the rest of our artifacts, but +# it is built primarly in separate repo neondatabase/autoscaling and thus is excluded +# from depending on workspace-hack because most of the dependencies are not used. +workspace-members = ["vm_monitor"] + # Write out exact versions rather than a semver range. (Defaults to false.) # exact-versions = true diff --git a/.github/ISSUE_TEMPLATE/epic-template.md b/.github/ISSUE_TEMPLATE/epic-template.md index 7707e0aa67..019e6e7345 100644 --- a/.github/ISSUE_TEMPLATE/epic-template.md +++ b/.github/ISSUE_TEMPLATE/epic-template.md @@ -17,8 +17,9 @@ assignees: '' ## Implementation ideas -## Tasks -- [ ] +```[tasklist] +### Tasks +``` ## Other related tasks and Epics diff --git a/.github/PULL_REQUEST_TEMPLATE/release-pr.md b/.github/PULL_REQUEST_TEMPLATE/release-pr.md index 1e18fd5d44..44b3094c24 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-pr.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-pr.md @@ -3,7 +3,7 @@ **NB: this PR must be merged only by 'Create a merge commit'!** ### Checklist when preparing for release -- [ ] Read or refresh [the release flow guide](https://github.com/neondatabase/cloud/wiki/Release:-general-flow) +- [ ] Read or refresh [the release flow guide](https://www.notion.so/neondatabase/Release-general-flow-61f2e39fd45d4d14a70c7749604bd70b) - [ ] Ask in the [cloud Slack channel](https://neondb.slack.com/archives/C033A2WE6BZ) that you are going to rollout the release. Any blockers? - [ ] Does this release contain any db migrations? Destructive ones? What is the rollback plan? diff --git a/.github/actionlint.yml b/.github/actionlint.yml index e2ece5f230..362480f256 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -1,5 +1,7 @@ self-hosted-runner: labels: + - arm64 + - dev - gen3 - large - small diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e0f8afc954..caa7ebee05 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -172,10 +172,10 @@ jobs: # https://github.com/EmbarkStudios/cargo-deny - name: Check rust licenses/bans/advisories/sources if: ${{ !cancelled() }} - run: cargo deny check + run: cargo deny check --hide-inclusion-graph build-neon: - needs: [ check-permissions ] + needs: [ check-permissions, tag ] runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned @@ -187,6 +187,7 @@ jobs: env: BUILD_TYPE: ${{ matrix.build_type }} GIT_VERSION: ${{ github.event.pull_request.head.sha || github.sha }} + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} steps: - name: Fix git ownership @@ -585,10 +586,13 @@ jobs: id: upload-coverage-report-new env: BUCKET: neon-github-public-dev + # A differential coverage report is available only for PRs. + # (i.e. for pushes into main/release branches we have a regular coverage report) COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} run: | - BASELINE="$(git merge-base HEAD origin/main)" CURRENT="${COMMIT_SHA}" + BASELINE="$(git merge-base $BASE_SHA $CURRENT)" cp /tmp/coverage/report/lcov.info ./${CURRENT}.info @@ -723,6 +727,7 @@ jobs: --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} + --build-arg BUILD_TAG=${{ needs.tag.outputs.build-tag }} --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{needs.tag.outputs.build-tag}} --destination neondatabase/neon:${{needs.tag.outputs.build-tag}} @@ -847,7 +852,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.18.5 + VM_BUILDER_VERSION: v0.19.0 steps: - name: Checkout @@ -869,8 +874,7 @@ jobs: - name: Build vm image run: | ./vm-builder \ - -enable-file-cache \ - -cgroup-uid=postgres \ + -spec=vm-image-spec.yaml \ -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index d7f5295c5b..0d7db8dfbc 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -21,7 +21,10 @@ env: jobs: check-macos-build: - if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') + if: | + contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' timeout-minutes: 90 runs-on: macos-latest @@ -112,8 +115,182 @@ jobs: - name: Check that no warnings are produced run: ./run_clippy.sh + check-linux-arm-build: + timeout-minutes: 90 + runs-on: [ self-hosted, dev, arm64 ] + + env: + # Use release build only, to have less debug info around + # Hence keeping target/ (and general cache size) smaller + BUILD_TYPE: release + CARGO_FEATURES: --features testing + CARGO_FLAGS: --locked --release + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} + + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init + + steps: + - name: Fix git ownership + run: | + # Workaround for `fatal: detected dubious ownership in repository at ...` + # + # Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers + # Ref https://github.com/actions/checkout/issues/785 + # + git config --global --add safe.directory ${{ github.workspace }} + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 1 + + - name: Set pg 14 revision for caching + id: pg_v14_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT + + - name: Set pg 15 revision for caching + id: pg_v15_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT + + - name: Set pg 16 revision for caching + id: pg_v16_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT + + - name: Set env variables + run: | + echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo" >> $GITHUB_ENV + + - name: Cache postgres v14 build + id: cache_pg_14 + uses: actions/cache@v3 + with: + path: pg_install/v14 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Cache postgres v15 build + id: cache_pg_15 + uses: actions/cache@v3 + with: + path: pg_install/v15 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Cache postgres v16 build + id: cache_pg_16 + uses: actions/cache@v3 + with: + path: pg_install/v16 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Build postgres v14 + if: steps.cache_pg_14.outputs.cache-hit != 'true' + run: mold -run make postgres-v14 -j$(nproc) + + - name: Build postgres v15 + if: steps.cache_pg_15.outputs.cache-hit != 'true' + run: mold -run make postgres-v15 -j$(nproc) + + - name: Build postgres v16 + if: steps.cache_pg_16.outputs.cache-hit != 'true' + run: mold -run make postgres-v16 -j$(nproc) + + - name: Build neon extensions + run: mold -run make neon-pg-ext -j$(nproc) + + - name: Build walproposer-lib + run: mold -run make walproposer-lib -j$(nproc) + + - name: Run cargo build + run: | + mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests + + - name: Run cargo test + run: | + cargo test $CARGO_FLAGS $CARGO_FEATURES + + # Run separate tests for real S3 + export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty + export REMOTE_STORAGE_S3_BUCKET=neon-github-public-dev + export REMOTE_STORAGE_S3_REGION=eu-central-1 + # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now + cargo test $CARGO_FLAGS --package remote_storage --test test_real_s3 + + # Run separate tests for real Azure Blob Storage + # XXX: replace region with `eu-central-1`-like region + export ENABLE_REAL_AZURE_REMOTE_STORAGE=y + export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}" + export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}" + export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}" + export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}" + # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now + cargo test $CARGO_FLAGS --package remote_storage --test test_real_azure + + check-codestyle-rust-arm: + timeout-minutes: 90 + runs-on: [ self-hosted, dev, arm64 ] + + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 1 + + # Some of our rust modules use FFI and need those to be checked + - name: Get postgres headers + run: make postgres-headers -j$(nproc) + + # cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations. + # This will catch compiler & clippy warnings in all feature combinations. + # TODO: use cargo hack for build and test as well, but, that's quite expensive. + # NB: keep clippy args in sync with ./run_clippy.sh + - run: | + CLIPPY_COMMON_ARGS="$( source .neon_clippy_args; echo "$CLIPPY_COMMON_ARGS")" + if [ "$CLIPPY_COMMON_ARGS" = "" ]; then + echo "No clippy args found in .neon_clippy_args" + exit 1 + fi + echo "CLIPPY_COMMON_ARGS=${CLIPPY_COMMON_ARGS}" >> $GITHUB_ENV + - name: Run cargo clippy (debug) + run: cargo hack --feature-powerset clippy $CLIPPY_COMMON_ARGS + - name: Run cargo clippy (release) + run: cargo hack --feature-powerset clippy --release $CLIPPY_COMMON_ARGS + + - name: Check documentation generation + run: cargo doc --workspace --no-deps --document-private-items + env: + RUSTDOCFLAGS: "-Dwarnings -Arustdoc::private_intra_doc_links" + + # Use `${{ !cancelled() }}` to run quck tests after the longer clippy run + - name: Check formatting + if: ${{ !cancelled() }} + run: cargo fmt --all -- --check + + # https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci + - name: Check rust dependencies + if: ${{ !cancelled() }} + run: | + cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date + cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack + + # https://github.com/EmbarkStudios/cargo-deny + - name: Check rust licenses/bans/advisories/sources + if: ${{ !cancelled() }} + run: cargo deny check + gather-rust-build-stats: - if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') + if: | + contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned diff --git a/Cargo.lock b/Cargo.lock index 4d9b20dae4..f08ed2d744 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,69 +23,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aead" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" -dependencies = [ - "aes-soft", - "aesni", - "cipher", -] - -[[package]] -name = "aes-gcm" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - -[[package]] -name = "aes-soft" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" -dependencies = [ - "cipher", - "opaque-debug", -] - -[[package]] -name = "aesni" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" -dependencies = [ - "cipher", - "opaque-debug", -] - [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" dependencies = [ "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -170,6 +117,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "archery" version = "0.5.0" @@ -192,7 +145,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.21", + "time", ] [[package]] @@ -242,55 +195,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-executor" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1da3ae8dabd9c00f453a329dfe1fb28da3c0a72e2478cdcd93171740c20499" -dependencies = [ - "async-lock", - "async-task", - "concurrent-queue", - "fastrand 2.0.0", - "futures-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" -dependencies = [ - "async-channel", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite", - "log", - "parking", - "polling", - "rustix 0.37.25", - "slab", - "socket2 0.4.9", - "waker-fn", -] - [[package]] name = "async-lock" version = "2.8.0" @@ -300,32 +204,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-std" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" -dependencies = [ - "async-channel", - "async-global-executor", - "async-io", - "async-lock", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - [[package]] name = "async-stream" version = "0.3.5" @@ -348,12 +226,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "async-task" -version = "4.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9441c6b2fe128a7c2bf680a44c34d0df31ce09e5b7e401fcca3faa483dbc921" - [[package]] name = "async-trait" version = "0.1.68" @@ -374,12 +246,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.1.0" @@ -409,7 +275,7 @@ dependencies = [ "http", "hyper", "ring", - "time 0.3.21", + "time", "tokio", "tower", "tracing", @@ -562,13 +428,13 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac 0.12.1", + "hmac", "http", "once_cell", "percent-encoding", "regex", - "sha2 0.10.6", - "time 0.3.21", + "sha2", + "time", "tracing", ] @@ -600,8 +466,8 @@ dependencies = [ "http-body", "md-5", "pin-project-lite", - "sha1 0.10.5", - "sha2 0.10.6", + "sha1", + "sha2", "tracing", ] @@ -746,7 +612,7 @@ dependencies = [ "num-integer", "ryu", "serde", - "time 0.3.21", + "time", ] [[package]] @@ -770,7 +636,7 @@ dependencies = [ "aws-smithy-http", "aws-smithy-types", "http", - "rustc_version 0.4.0", + "rustc_version", "tracing", ] @@ -800,7 +666,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1 0.10.5", + "sha1", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -845,10 +711,10 @@ dependencies = [ "quick-xml", "rand 0.8.5", "reqwest", - "rustc_version 0.4.0", + "rustc_version", "serde", "serde_json", - "time 0.3.21", + "time", "url", "uuid", ] @@ -868,7 +734,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "time 0.3.21", + "time", "tz-rs", "url", "uuid", @@ -885,13 +751,13 @@ dependencies = [ "azure_core", "bytes", "futures", - "hmac 0.12.1", + "hmac", "log", "serde", "serde_derive", "serde_json", - "sha2 0.10.6", - "time 0.3.21", + "sha2", + "time", "url", "uuid", ] @@ -911,7 +777,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "time 0.3.21", + "time", "url", "uuid", ] @@ -931,12 +797,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - [[package]] name = "base64" version = "0.13.1" @@ -1009,15 +869,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1027,22 +878,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c36a4d0d48574b3dd360b4b7d95cc651d2b6557b6402848a27d4b228a473e2a" -dependencies = [ - "async-channel", - "async-lock", - "async-task", - "fastrand 2.0.0", - "futures-io", - "futures-lite", - "piper", - "tracing", -] - [[package]] name = "bstr" version = "1.5.0" @@ -1187,15 +1022,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" -dependencies = [ - "generic-array", -] - [[package]] name = "clang-sys" version = "1.6.1" @@ -1413,23 +1239,6 @@ dependencies = [ "workspace_hack", ] -[[package]] -name = "cookie" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" -dependencies = [ - "aes-gcm", - "base64 0.13.1", - "hkdf", - "hmac 0.10.1", - "percent-encoding", - "rand 0.8.5", - "sha2 0.9.9", - "time 0.2.27", - "version_check", -] - [[package]] name = "core-foundation" version = "0.9.3" @@ -1455,19 +1264,13 @@ dependencies = [ "libc", ] -[[package]] -name = "cpuid-bool" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" - [[package]] name = "crc32c" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e" dependencies = [ - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -1599,25 +1402,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ctr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" -dependencies = [ - "cipher", -] - [[package]] name = "darling" version = "0.20.1" @@ -1696,32 +1480,17 @@ dependencies = [ "rusticata-macros", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "crypto-common", "subtle", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "displaydoc" version = "0.2.4" @@ -2088,16 +1857,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "ghash" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" -dependencies = [ - "opaque-debug", - "polyval", -] - [[package]] name = "gimli" version = "0.27.2" @@ -2132,18 +1891,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "gloo-timers" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "h2" version = "0.3.19" @@ -2215,7 +1962,7 @@ source = "git+https://github.com/japaric/heapless.git?rev=644653bf3b831c6bb4963b dependencies = [ "atomic-polyfill", "hash32", - "rustc_version 0.4.0", + "rustc_version", "spin 0.9.8", "stable_deref_trait", ] @@ -2257,33 +2004,13 @@ dependencies = [ "thiserror", ] -[[package]] -name = "hkdf" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" -dependencies = [ - "digest 0.9.0", - "hmac 0.10.1", -] - -[[package]] -name = "hmac" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" -dependencies = [ - "crypto-mac", - "digest 0.9.0", -] - [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -2327,9 +2054,7 @@ checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" dependencies = [ "anyhow", "async-channel", - "async-std", "base64 0.13.1", - "cookie", "futures-lite", "infer", "pin-project-lite", @@ -2643,15 +2368,6 @@ dependencies = [ "libc", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2707,9 +2423,6 @@ name = "log" version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -dependencies = [ - "value-bag", -] [[package]] name = "match_cfg" @@ -2738,7 +2451,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -2984,7 +2697,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "sha2 0.10.6", + "sha2", "thiserror", "url", ] @@ -3019,12 +2732,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.55" @@ -3287,10 +2994,12 @@ name = "pageserver_api" version = "0.1.0" dependencies = [ "anyhow", + "bincode", "byteorder", "bytes", "const_format", "enum-map", + "hex", "postgres_ffi", "serde", "serde_json", @@ -3378,10 +3087,10 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0ca0b5a68607598bf3bad68f32227a8164f6254833f84eafaac409cd6746c31" dependencies = [ - "digest 0.10.7", - "hmac 0.12.1", + "digest", + "hmac", "password-hash", - "sha2 0.10.6", + "sha2", ] [[package]] @@ -3475,17 +3184,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" -dependencies = [ - "atomic-waker", - "fastrand 2.0.0", - "futures-io", -] - [[package]] name = "pkg-config" version = "0.3.27" @@ -3520,37 +3218,10 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "polyval" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" -dependencies = [ - "cpuid-bool", - "opaque-debug", - "universal-hash", -] - [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "bytes", "fallible-iterator", @@ -3563,7 +3234,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "native-tls", "tokio", @@ -3574,25 +3245,25 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "base64 0.20.0", "byteorder", "bytes", "fallible-iterator", - "hmac 0.12.1", + "hmac", "lazy_static", "md-5", "memchr", "rand 0.8.5", - "sha2 0.10.6", + "sha2", "stringprep", ] [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "bytes", "fallible-iterator", @@ -3814,7 +3485,7 @@ dependencies = [ "hashbrown 0.13.2", "hashlink", "hex", - "hmac 0.12.1", + "hmac", "hostname", "humantime", "hyper", @@ -3847,9 +3518,10 @@ dependencies = [ "scopeguard", "serde", "serde_json", - "sha2 0.10.6", + "sha2", "socket2 0.5.3", "sync_wrapper", + "task-local-extensions", "thiserror", "tls-listener", "tokio", @@ -3989,7 +3661,7 @@ checksum = "4954fbc00dcd4d8282c987710e50ba513d351400dbdd00e803a05172a90d8976" dependencies = [ "pem 2.0.1", "ring", - "time 0.3.21", + "time", "yasna", ] @@ -4058,6 +3730,7 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-s3", + "aws-smithy-async", "aws-smithy-http", "aws-types", "azure_core", @@ -4245,7 +3918,7 @@ dependencies = [ "futures", "futures-timer", "rstest_macros", - "rustc_version 0.4.0", + "rustc_version", ] [[package]] @@ -4260,7 +3933,7 @@ dependencies = [ "quote", "regex", "relative-path", - "rustc_version 0.4.0", + "rustc_version", "syn 2.0.28", "unicode-ident", ] @@ -4277,22 +3950,13 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.17", + "semver", ] [[package]] @@ -4419,6 +4083,7 @@ dependencies = [ "itertools", "pageserver", "rand 0.8.5", + "remote_storage", "reqwest", "serde", "serde_json", @@ -4477,6 +4142,7 @@ dependencies = [ "tokio", "tokio-io-timeout", "tokio-postgres", + "tokio-stream", "toml_edit", "tracing", "url", @@ -4552,27 +4218,12 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "sentry" version = "0.31.6" @@ -4613,7 +4264,7 @@ dependencies = [ "hostname", "libc", "os_info", - "rustc_version 0.4.0", + "rustc_version", "sentry-core", "uname", ] @@ -4665,7 +4316,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "time 0.3.21", + "time", "url", "uuid", ] @@ -4679,6 +4330,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_assert" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda563240c1288b044209be1f0d38bb4d15044fb3e00dc354fbc922ab4733e80" +dependencies = [ + "hashbrown 0.13.2", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.183" @@ -4756,7 +4417,7 @@ dependencies = [ "serde", "serde_json", "serde_with_macros", - "time 0.3.21", + "time", ] [[package]] @@ -4771,15 +4432,6 @@ dependencies = [ "syn 2.0.28", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - [[package]] name = "sha1" version = "0.10.5" @@ -4788,26 +4440,7 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "digest", ] [[package]] @@ -4818,7 +4451,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -4875,7 +4508,7 @@ dependencies = [ "num-bigint", "num-traits", "thiserror", - "time 0.3.21", + "time", ] [[package]] @@ -4940,70 +4573,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1 0.6.1", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - [[package]] name = "storage_broker" version = "0.1.0" @@ -5237,21 +4812,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros 0.1.1", - "version_check", - "winapi", -] - [[package]] name = "time" version = "0.3.21" @@ -5264,7 +4824,7 @@ dependencies = [ "num_threads", "serde", "time-core", - "time-macros 0.2.9", + "time-macros", ] [[package]] @@ -5273,16 +4833,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - [[package]] name = "time-macros" version = "0.2.9" @@ -5292,19 +4842,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn 1.0.109", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -5396,7 +4933,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" dependencies = [ "async-trait", "byteorder", @@ -5666,7 +5203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" dependencies = [ "crossbeam-channel", - "time 0.3.21", + "time", "tracing-subscriber", ] @@ -5801,7 +5338,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha1 0.10.5", + "sha1", "thiserror", "url", "utf-8", @@ -5873,16 +5410,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" -[[package]] -name = "universal-hash" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "untrusted" version = "0.7.1" @@ -5939,6 +5466,7 @@ name = "utils" version = "0.1.0" dependencies = [ "anyhow", + "arc-swap", "async-trait", "bincode", "byteorder", @@ -5965,6 +5493,7 @@ dependencies = [ "routerify", "sentry", "serde", + "serde_assert", "serde_json", "serde_with", "signal-hook", @@ -5998,12 +5527,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "value-bag" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" - [[package]] name = "vcpkg" version = "0.2.15" @@ -6035,7 +5558,6 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", - "workspace_hack", ] [[package]] @@ -6463,6 +5985,7 @@ dependencies = [ "clap", "clap_builder", "crossbeam-utils", + "dashmap", "either", "fail", "futures", @@ -6493,11 +6016,10 @@ dependencies = [ "serde", "serde_json", "smallvec", - "standback", "syn 1.0.109", "syn 2.0.28", - "time 0.3.21", - "time-macros 0.2.9", + "time", + "time-macros", "tokio", "tokio-rustls", "tokio-util", @@ -6525,7 +6047,7 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.21", + "time", ] [[package]] @@ -6549,7 +6071,27 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.21", + "time", +] + +[[package]] +name = "zerocopy" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7af71d8643341260a65f89fa60c0eeaa907f34544d8f6d9b0df72f069b5e74" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9731702e2f0617ad526794ae28fbc6f6ca8849b5ba729666c2a5bc4b6ddee2cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6f538941f9..7d7d74993e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ license = "Apache-2.0" ## All dependency versions, used in the project [workspace.dependencies] anyhow = { version = "1.0", features = ["backtrace"] } +arc-swap = "1.6" async-compression = { version = "0.4.0", features = ["tokio", "gzip"] } azure_core = "0.16" azure_identity = "0.16" @@ -47,6 +48,7 @@ async-trait = "0.1" aws-config = { version = "0.56", default-features = false, features=["rustls"] } aws-sdk-s3 = "0.29" aws-smithy-http = "0.56" +aws-smithy-async = { version = "0.56", default-features = false, features=["rt-tokio"] } aws-credential-types = "0.56" aws-types = "0.56" axum = { version = "0.6.20", features = ["ws"] } @@ -65,7 +67,7 @@ comfy-table = "6.1" const_format = "0.2" crc32c = "0.6" crossbeam-utils = "0.8.5" -dashmap = "5.5.0" +dashmap = { version = "5.5.0", features = ["raw-api"] } either = "1.8" enum-map = "2.4.2" enumset = "1.0.12" @@ -81,7 +83,7 @@ hex = "0.4" hex-literal = "0.4" hmac = "0.12.1" hostname = "0.3.1" -http-types = "2" +http-types = { version = "2", default-features = false } humantime = "2.1" humantime-serde = "1.1.1" hyper = "0.14" @@ -124,6 +126,7 @@ sentry = { version = "0.31", default-features = false, features = ["backtrace", serde = { version = "1.0", features = ["derive"] } serde_json = "1" serde_with = "2.0" +serde_assert = "0.5.0" sha2 = "0.10.2" signal-hook = "0.3" smallvec = "1.11" @@ -133,6 +136,7 @@ strum_macros = "0.24" svg_fmt = "0.4.1" sync_wrapper = "0.1.2" tar = "0.4" +task-local-extensions = "0.1.4" test-context = "0.1" thiserror = "1.0" tls-listener = { version = "0.7", features = ["rustls", "hyper-h1"] } @@ -161,11 +165,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -202,7 +206,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } ################# Binary contents sections diff --git a/Dockerfile b/Dockerfile index eb4c4bba25..60de9cfa3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN set -e \ FROM $REPOSITORY/$IMAGE:$TAG AS build WORKDIR /home/nonroot ARG GIT_VERSION=local +ARG BUILD_TAG # Enable https://github.com/paritytech/cachepot to cache Rust crates' compilation results in Docker builds. # Set up cachepot to use an AWS S3 bucket for cache results, to reuse it between `docker build` invocations. @@ -78,9 +79,9 @@ COPY --from=build --chown=neon:neon /home/nonroot/target/release/pg_sni_router COPY --from=build --chown=neon:neon /home/nonroot/target/release/pageserver /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/pagectl /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/safekeeper /usr/local/bin -COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_broker /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_broker /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy /usr/local/bin -COPY --from=build --chown=neon:neon /home/nonroot/target/release/neon_local /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/neon_local /usr/local/bin COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/ COPY --from=pg-build /home/nonroot/pg_install/v15 /usr/local/v15/ diff --git a/Makefile b/Makefile index 64bbc1677c..89acbe564a 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,10 @@ neon: postgres-headers walproposer-lib # $(POSTGRES_INSTALL_DIR)/build/%/config.status: +@echo "Configuring Postgres $* build" + @test -s $(ROOT_PROJECT_DIR)/vendor/postgres-$*/configure || { \ + echo "\nPostgres submodule not found in $(ROOT_PROJECT_DIR)/vendor/postgres-$*/, execute "; \ + echo "'git submodule update --init --recursive --depth 2 --progress .' in project root.\n"; \ + exit 1; } mkdir -p $(POSTGRES_INSTALL_DIR)/build/$* (cd $(POSTGRES_INSTALL_DIR)/build/$* && \ env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-$*/configure \ diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 81d4320b14..7f22bda13e 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -479,13 +479,6 @@ fn cli() -> clap::Command { ) .value_name("FILECACHE_CONNSTR"), ) - .arg( - // DEPRECATED, NO LONGER DOES ANYTHING. - // See https://github.com/neondatabase/cloud/issues/7516 - Arg::new("file-cache-on-disk") - .long("file-cache-on-disk") - .action(clap::ArgAction::SetTrue), - ) } #[test] diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 9fd88e5818..373f05ab2f 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -710,8 +710,12 @@ impl ComputeNode { // `pg_ctl` for start / stop, so this just seems much easier to do as we already // have opened connection to Postgres and superuser access. #[instrument(skip_all)] - fn pg_reload_conf(&self, client: &mut Client) -> Result<()> { - client.simple_query("SELECT pg_reload_conf()")?; + fn pg_reload_conf(&self) -> Result<()> { + let pgctl_bin = Path::new(&self.pgbin).parent().unwrap().join("pg_ctl"); + Command::new(pgctl_bin) + .args(["reload", "-D", &self.pgdata]) + .output() + .expect("cannot run pg_ctl process"); Ok(()) } @@ -724,9 +728,9 @@ impl ComputeNode { // Write new config let pgdata_path = Path::new(&self.pgdata); config::write_postgres_conf(&pgdata_path.join("postgresql.conf"), &spec, None)?; + self.pg_reload_conf()?; let mut client = Client::connect(self.connstr.as_str(), NoTls)?; - self.pg_reload_conf(&mut client)?; // Proceed with post-startup configuration. Note, that order of operations is important. // Disable DDL forwarding because control plane already knows about these roles/databases. diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 3d7ed8c360..9732d8adea 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -78,7 +78,7 @@ use regex::Regex; use remote_storage::*; use serde_json; use std::io::Read; -use std::num::{NonZeroU32, NonZeroUsize}; +use std::num::NonZeroUsize; use std::path::Path; use std::str; use tar::Archive; @@ -133,45 +133,6 @@ fn parse_pg_version(human_version: &str) -> &str { panic!("Unsuported postgres version {human_version}"); } -#[cfg(test)] -mod tests { - use super::parse_pg_version; - - #[test] - fn test_parse_pg_version() { - assert_eq!(parse_pg_version("PostgreSQL 15.4"), "v15"); - assert_eq!(parse_pg_version("PostgreSQL 15.14"), "v15"); - assert_eq!( - parse_pg_version("PostgreSQL 15.4 (Ubuntu 15.4-0ubuntu0.23.04.1)"), - "v15" - ); - - assert_eq!(parse_pg_version("PostgreSQL 14.15"), "v14"); - assert_eq!(parse_pg_version("PostgreSQL 14.0"), "v14"); - assert_eq!( - parse_pg_version("PostgreSQL 14.9 (Debian 14.9-1.pgdg120+1"), - "v14" - ); - - assert_eq!(parse_pg_version("PostgreSQL 16devel"), "v16"); - assert_eq!(parse_pg_version("PostgreSQL 16beta1"), "v16"); - assert_eq!(parse_pg_version("PostgreSQL 16rc2"), "v16"); - assert_eq!(parse_pg_version("PostgreSQL 16extra"), "v16"); - } - - #[test] - #[should_panic] - fn test_parse_pg_unsupported_version() { - parse_pg_version("PostgreSQL 13.14"); - } - - #[test] - #[should_panic] - fn test_parse_pg_incorrect_version_format() { - parse_pg_version("PostgreSQL 14"); - } -} - // download the archive for a given extension, // unzip it, and place files in the appropriate locations (share/lib) pub async fn download_extension( @@ -281,9 +242,46 @@ pub fn init_remote_storage(remote_ext_config: &str) -> anyhow::Result Result> { - let cp_uri = format!("{base_uri}/management/api/v2/computes/{compute_id}/spec"); + let cp_uri = format!("{base_uri}/compute/api/v2/computes/{compute_id}/spec"); let jwt: String = match std::env::var("NEON_CONTROL_PLANE_TOKEN") { Ok(v) => v, Err(_) => "".to_string(), diff --git a/control_plane/src/attachment_service.rs b/control_plane/src/attachment_service.rs index ca632c5eb6..822ac7d8a6 100644 --- a/control_plane/src/attachment_service.rs +++ b/control_plane/src/attachment_service.rs @@ -2,7 +2,6 @@ use crate::{background_process, local_env::LocalEnv}; use anyhow::anyhow; use camino::Utf8PathBuf; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::{path::PathBuf, process::Child}; use utils::id::{NodeId, TenantId}; @@ -10,14 +9,13 @@ pub struct AttachmentService { env: LocalEnv, listen: String, path: PathBuf, + client: reqwest::blocking::Client, } const COMMAND: &str = "attachment_service"; -#[serde_as] #[derive(Serialize, Deserialize)] pub struct AttachHookRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, pub node_id: Option, } @@ -27,6 +25,16 @@ pub struct AttachHookResponse { pub gen: Option, } +#[derive(Serialize, Deserialize)] +pub struct InspectRequest { + pub tenant_id: TenantId, +} + +#[derive(Serialize, Deserialize)] +pub struct InspectResponse { + pub attachment: Option<(u32, NodeId)>, +} + impl AttachmentService { pub fn from_env(env: &LocalEnv) -> Self { let path = env.base_data_dir.join("attachments.json"); @@ -45,6 +53,9 @@ impl AttachmentService { env: env.clone(), path, listen, + client: reqwest::blocking::ClientBuilder::new() + .build() + .expect("Failed to construct http client"), } } @@ -87,16 +98,13 @@ impl AttachmentService { .unwrap() .join("attach-hook") .unwrap(); - let client = reqwest::blocking::ClientBuilder::new() - .build() - .expect("Failed to construct http client"); let request = AttachHookRequest { tenant_id, node_id: Some(pageserver_id), }; - let response = client.post(url).json(&request).send()?; + let response = self.client.post(url).json(&request).send()?; if response.status() != StatusCode::OK { return Err(anyhow!("Unexpected status {}", response.status())); } @@ -104,4 +112,26 @@ impl AttachmentService { let response = response.json::()?; Ok(response.gen) } + + pub fn inspect(&self, tenant_id: TenantId) -> anyhow::Result> { + use hyper::StatusCode; + + let url = self + .env + .control_plane_api + .clone() + .unwrap() + .join("inspect") + .unwrap(); + + let request = InspectRequest { tenant_id }; + + let response = self.client.post(url).json(&request).send()?; + if response.status() != StatusCode::OK { + return Err(anyhow!("Unexpected status {}", response.status())); + } + + let response = response.json::()?; + Ok(response.attachment) + } } diff --git a/control_plane/src/background_process.rs b/control_plane/src/background_process.rs index c0016ece17..26fc08fc8f 100644 --- a/control_plane/src/background_process.rs +++ b/control_plane/src/background_process.rs @@ -262,7 +262,7 @@ where P: Into, { let path: Utf8PathBuf = path.into(); - // SAFETY + // SAFETY: // pre_exec is marked unsafe because it runs between fork and exec. // Why is that dangerous in various ways? // Long answer: https://github.com/rust-lang/rust/issues/39575 diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index 3205703c7d..ddd50ff51d 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -32,7 +32,9 @@ use pageserver_api::control_api::{ ValidateResponseTenant, }; -use control_plane::attachment_service::{AttachHookRequest, AttachHookResponse}; +use control_plane::attachment_service::{ + AttachHookRequest, AttachHookResponse, InspectRequest, InspectResponse, +}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -255,12 +257,28 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap ) } +async fn handle_inspect(mut req: Request) -> Result, ApiError> { + let inspect_req = json_request::(&mut req).await?; + + let state = get_state(&req).inner.clone(); + let locked = state.write().await; + let tenant_state = locked.tenants.get(&inspect_req.tenant_id); + + json_response( + StatusCode::OK, + InspectResponse { + attachment: tenant_state.and_then(|s| s.pageserver.map(|ps| (s.generation, ps))), + }, + ) +} + fn make_router(persistent_state: PersistentState) -> RouterBuilder { endpoint::make_router() .data(Arc::new(State::new(persistent_state))) .post("/re-attach", |r| request_span(r, handle_re_attach)) .post("/validate", |r| request_span(r, handle_validate)) .post("/attach-hook", |r| request_span(r, handle_attach_hook)) + .post("/inspect", |r| request_span(r, handle_inspect)) } #[tokio::main] diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index bc68eeaa8a..87ea519a9e 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -11,13 +11,14 @@ use compute_api::spec::ComputeMode; use control_plane::attachment_service::AttachmentService; use control_plane::endpoint::ComputeControlPlane; use control_plane::local_env::LocalEnv; -use control_plane::pageserver::PageServerNode; +use control_plane::pageserver::{PageServerNode, PAGESERVER_REMOTE_STORAGE_DIR}; use control_plane::safekeeper::SafekeeperNode; +use control_plane::tenant_migration::migrate_tenant; use control_plane::{broker, local_env}; use pageserver_api::models::TimelineInfo; use pageserver_api::{ - DEFAULT_HTTP_LISTEN_ADDR as DEFAULT_PAGESERVER_HTTP_ADDR, - DEFAULT_PG_LISTEN_ADDR as DEFAULT_PAGESERVER_PG_ADDR, + DEFAULT_HTTP_LISTEN_PORT as DEFAULT_PAGESERVER_HTTP_PORT, + DEFAULT_PG_LISTEN_PORT as DEFAULT_PAGESERVER_PG_PORT, }; use postgres_backend::AuthType; use safekeeper_api::{ @@ -46,8 +47,8 @@ const DEFAULT_PG_VERSION: &str = "15"; const DEFAULT_PAGESERVER_CONTROL_PLANE_API: &str = "http://127.0.0.1:1234/"; -fn default_conf() -> String { - format!( +fn default_conf(num_pageservers: u16) -> String { + let mut template = format!( r#" # Default built-in configuration, defined in main.rs control_plane_api = '{DEFAULT_PAGESERVER_CONTROL_PLANE_API}' @@ -55,21 +56,33 @@ control_plane_api = '{DEFAULT_PAGESERVER_CONTROL_PLANE_API}' [broker] listen_addr = '{DEFAULT_BROKER_ADDR}' -[[pageservers]] -id = {DEFAULT_PAGESERVER_ID} -listen_pg_addr = '{DEFAULT_PAGESERVER_PG_ADDR}' -listen_http_addr = '{DEFAULT_PAGESERVER_HTTP_ADDR}' -pg_auth_type = '{trust_auth}' -http_auth_type = '{trust_auth}' - [[safekeepers]] id = {DEFAULT_SAFEKEEPER_ID} pg_port = {DEFAULT_SAFEKEEPER_PG_PORT} http_port = {DEFAULT_SAFEKEEPER_HTTP_PORT} "#, - trust_auth = AuthType::Trust, - ) + ); + + for i in 0..num_pageservers { + let pageserver_id = NodeId(DEFAULT_PAGESERVER_ID.0 + i as u64); + let pg_port = DEFAULT_PAGESERVER_PG_PORT + i; + let http_port = DEFAULT_PAGESERVER_HTTP_PORT + i; + + template += &format!( + r#" +[[pageservers]] +id = {pageserver_id} +listen_pg_addr = '127.0.0.1:{pg_port}' +listen_http_addr = '127.0.0.1:{http_port}' +pg_auth_type = '{trust_auth}' +http_auth_type = '{trust_auth}' +"#, + trust_auth = AuthType::Trust, + ) + } + + template } /// @@ -295,6 +308,9 @@ fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result anyhow::Result { + let num_pageservers = init_match + .get_one::("num-pageservers") + .expect("num-pageservers arg has a default"); // Create config file let toml_file: String = if let Some(config_path) = init_match.get_one::("config") { // load and parse the file @@ -306,7 +322,7 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result { })? } else { // Built-in default config - default_conf() + default_conf(*num_pageservers) }; let pg_version = init_match @@ -320,6 +336,9 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result { env.init(pg_version, force) .context("Failed to initialize neon repository")?; + // Create remote storage location for default LocalFs remote storage + std::fs::create_dir_all(env.base_data_dir.join(PAGESERVER_REMOTE_STORAGE_DIR))?; + // Initialize pageserver, create initial tenant and timeline. for ps_conf in &env.pageservers { PageServerNode::from_env(&env, ps_conf) @@ -433,6 +452,15 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an .with_context(|| format!("Tenant config failed for tenant with id {tenant_id}"))?; println!("tenant {tenant_id} successfully configured on the pageserver"); } + Some(("migrate", matches)) => { + let tenant_id = get_tenant_id(matches, env)?; + let new_pageserver = get_pageserver(env, matches)?; + let new_pageserver_id = new_pageserver.conf.id; + + migrate_tenant(env, tenant_id, new_pageserver)?; + println!("tenant {tenant_id} migrated to {}", new_pageserver_id); + } + Some((sub_name, _)) => bail!("Unexpected tenant subcommand '{}'", sub_name), None => bail!("no tenant subcommand provided"), } @@ -867,20 +895,20 @@ fn handle_mappings(sub_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Res } } +fn get_pageserver(env: &local_env::LocalEnv, args: &ArgMatches) -> Result { + let node_id = if let Some(id_str) = args.get_one::("pageserver-id") { + NodeId(id_str.parse().context("while parsing pageserver id")?) + } else { + DEFAULT_PAGESERVER_ID + }; + + Ok(PageServerNode::from_env( + env, + env.get_pageserver_conf(node_id)?, + )) +} + fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { - fn get_pageserver(env: &local_env::LocalEnv, args: &ArgMatches) -> Result { - let node_id = if let Some(id_str) = args.get_one::("pageserver-id") { - NodeId(id_str.parse().context("while parsing pageserver id")?) - } else { - DEFAULT_PAGESERVER_ID - }; - - Ok(PageServerNode::from_env( - env, - env.get_pageserver_conf(node_id)?, - )) - } - match sub_match.subcommand() { Some(("start", subcommand_args)) => { if let Err(e) = get_pageserver(env, subcommand_args)? @@ -917,6 +945,20 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul } } + Some(("migrate", subcommand_args)) => { + let pageserver = get_pageserver(env, subcommand_args)?; + //TODO what shutdown strategy should we use here? + if let Err(e) = pageserver.stop(false) { + eprintln!("pageserver stop failed: {}", e); + exit(1); + } + + if let Err(e) = pageserver.start(&pageserver_config_overrides(subcommand_args)) { + eprintln!("pageserver start failed: {e}"); + exit(1); + } + } + Some(("status", subcommand_args)) => { match get_pageserver(env, subcommand_args)?.check_status() { Ok(_) => println!("Page server is up and running"), @@ -1224,6 +1266,13 @@ fn cli() -> Command { .help("Force initialization even if the repository is not empty") .required(false); + let num_pageservers_arg = Arg::new("num-pageservers") + .value_parser(value_parser!(u16)) + .long("num-pageservers") + .help("How many pageservers to create (default 1)") + .required(false) + .default_value("1"); + Command::new("Neon CLI") .arg_required_else_help(true) .version(GIT_VERSION) @@ -1231,6 +1280,7 @@ fn cli() -> Command { Command::new("init") .about("Initialize a new Neon repository, preparing configs for services to start with") .arg(pageserver_config_args.clone()) + .arg(num_pageservers_arg.clone()) .arg( Arg::new("config") .long("config") @@ -1301,6 +1351,10 @@ fn cli() -> Command { .subcommand(Command::new("config") .arg(tenant_id_arg.clone()) .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))) + .subcommand(Command::new("migrate") + .about("Migrate a tenant from one pageserver to another") + .arg(tenant_id_arg.clone()) + .arg(pageserver_id_arg.clone())) ) .subcommand( Command::new("pageserver") diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index cb16f48829..4443fd8704 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -46,7 +46,6 @@ use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{NodeId, TenantId, TimelineId}; use crate::local_env::LocalEnv; @@ -57,13 +56,10 @@ use compute_api::responses::{ComputeState, ComputeStatus}; use compute_api::spec::{Cluster, ComputeMode, ComputeSpec}; // contents of a endpoint.json file -#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct EndpointConf { endpoint_id: String, - #[serde_as(as = "DisplayFromStr")] tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] timeline_id: TimelineId, mode: ComputeMode, pg_port: u16, diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 7592880402..52a0e20429 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -1,11 +1,10 @@ -// -// Local control plane. -// -// Can start, configure and stop postgres instances running as a local processes. -// -// Intended to be used in integration tests and in CLI tools for -// local installations. -// +//! Local control plane. +//! +//! Can start, configure and stop postgres instances running as a local processes. +//! +//! Intended to be used in integration tests and in CLI tools for +//! local installations. +#![deny(clippy::undocumented_unsafe_blocks)] pub mod attachment_service; mod background_process; @@ -15,3 +14,4 @@ pub mod local_env; pub mod pageserver; pub mod postgresql_conf; pub mod safekeeper; +pub mod tenant_migration; diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 45a7469787..b9c8aeddcb 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -8,7 +8,6 @@ use anyhow::{bail, ensure, Context}; use postgres_backend::AuthType; use reqwest::Url; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::collections::HashMap; use std::env; use std::fs; @@ -33,7 +32,6 @@ pub const DEFAULT_PG_VERSION: u32 = 15; // to 'neon_local init --config=' option. See control_plane/simple.conf for // an example. // -#[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct LocalEnv { // Base directory for all the nodes (the pageserver, safekeepers and @@ -59,7 +57,6 @@ pub struct LocalEnv { // Default tenant ID to use with the 'neon_local' command line utility, when // --tenant_id is not explicitly specified. #[serde(default)] - #[serde_as(as = "Option")] pub default_tenant_id: Option, // used to issue tokens during e.g pg start @@ -84,7 +81,6 @@ pub struct LocalEnv { // A `HashMap>` would be more appropriate here, // but deserialization into a generic toml object as `toml::Value::try_from` fails with an error. // https://toml.io/en/v1.0.0 does not contain a concept of "a table inside another table". - #[serde_as(as = "HashMap<_, Vec<(DisplayFromStr, DisplayFromStr)>>")] branch_name_mappings: HashMap>, } diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 0746dde4ef..237df48543 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -15,7 +15,10 @@ use std::{io, result}; use anyhow::{bail, Context}; use camino::Utf8PathBuf; -use pageserver_api::models::{self, TenantInfo, TimelineInfo}; +use pageserver_api::models::{ + self, LocationConfig, TenantInfo, TenantLocationConfigRequest, TimelineInfo, +}; +use pageserver_api::shard::TenantShardId; use postgres_backend::AuthType; use postgres_connection::{parse_host_port, PgConnectionConfig}; use reqwest::blocking::{Client, RequestBuilder, Response}; @@ -31,6 +34,9 @@ use utils::{ use crate::local_env::PageServerConf; use crate::{background_process, local_env::LocalEnv}; +/// Directory within .neon which will be used by default for LocalFs remote storage. +pub const PAGESERVER_REMOTE_STORAGE_DIR: &str = "local_fs_remote_storage/pageserver"; + #[derive(Error, Debug)] pub enum PageserverHttpError { #[error("Reqwest error: {0}")] @@ -98,8 +104,10 @@ impl PageServerNode { } } - // pageserver conf overrides defined by neon_local configuration. - fn neon_local_overrides(&self) -> Vec { + /// Merge overrides provided by the user on the command line with our default overides derived from neon_local configuration. + /// + /// These all end up on the command line of the `pageserver` binary. + fn neon_local_overrides(&self, cli_overrides: &[&str]) -> Vec { let id = format!("id={}", self.conf.id); // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc. let pg_distrib_dir_param = format!( @@ -132,12 +140,25 @@ impl PageServerNode { )); } + if !cli_overrides + .iter() + .any(|c| c.starts_with("remote_storage")) + { + overrides.push(format!( + "remote_storage={{local_path='../{PAGESERVER_REMOTE_STORAGE_DIR}'}}" + )); + } + if self.conf.http_auth_type != AuthType::Trust || self.conf.pg_auth_type != AuthType::Trust { // Keys are generated in the toplevel repo dir, pageservers' workdirs // are one level below that, so refer to keys with ../ overrides.push("auth_validation_public_key_path='../auth_public_key.pem'".to_owned()); } + + // Apply the user-provided overrides + overrides.extend(cli_overrides.iter().map(|&c| c.to_owned())); + overrides } @@ -203,9 +224,6 @@ impl PageServerNode { } fn start_node(&self, config_overrides: &[&str], update_config: bool) -> anyhow::Result { - let mut overrides = self.neon_local_overrides(); - overrides.extend(config_overrides.iter().map(|&c| c.to_owned())); - let datadir = self.repo_path(); print!( "Starting pageserver node {} at '{}' in {:?}", @@ -248,8 +266,7 @@ impl PageServerNode { ) -> Vec> { let mut args = vec![Cow::Borrowed("-D"), Cow::Borrowed(datadir_path_str)]; - let mut overrides = self.neon_local_overrides(); - overrides.extend(config_overrides.iter().map(|&c| c.to_owned())); + let overrides = self.neon_local_overrides(config_overrides); for config_override in overrides { args.push(Cow::Borrowed("-c")); args.push(Cow::Owned(config_override)); @@ -392,7 +409,7 @@ impl PageServerNode { }; let request = models::TenantCreateRequest { - new_tenant_id, + new_tenant_id: TenantShardId::unsharded(new_tenant_id), generation, config, }; @@ -501,6 +518,27 @@ impl PageServerNode { Ok(()) } + pub fn location_config( + &self, + tenant_id: TenantId, + config: LocationConfig, + ) -> anyhow::Result<()> { + let req_body = TenantLocationConfigRequest { tenant_id, config }; + + self.http_request( + Method::PUT, + format!( + "{}/tenant/{}/location_config", + self.http_base_url, tenant_id + ), + )? + .json(&req_body) + .send()? + .error_from_body()?; + + Ok(()) + } + pub fn timeline_list(&self, tenant_id: &TenantId) -> anyhow::Result> { let timeline_infos: Vec = self .http_request( diff --git a/control_plane/src/tenant_migration.rs b/control_plane/src/tenant_migration.rs new file mode 100644 index 0000000000..d28d1f9fe8 --- /dev/null +++ b/control_plane/src/tenant_migration.rs @@ -0,0 +1,202 @@ +//! +//! Functionality for migrating tenants across pageservers: unlike most of neon_local, this code +//! isn't scoped to a particular physical service, as it needs to update compute endpoints to +//! point to the new pageserver. +//! +use crate::local_env::LocalEnv; +use crate::{ + attachment_service::AttachmentService, endpoint::ComputeControlPlane, + pageserver::PageServerNode, +}; +use pageserver_api::models::{ + LocationConfig, LocationConfigMode, LocationConfigSecondary, TenantConfig, +}; +use std::collections::HashMap; +use std::time::Duration; +use utils::{ + generation::Generation, + id::{TenantId, TimelineId}, + lsn::Lsn, +}; + +/// Given an attached pageserver, retrieve the LSN for all timelines +fn get_lsns( + tenant_id: TenantId, + pageserver: &PageServerNode, +) -> anyhow::Result> { + let timelines = pageserver.timeline_list(&tenant_id)?; + Ok(timelines + .into_iter() + .map(|t| (t.timeline_id, t.last_record_lsn)) + .collect()) +} + +/// Wait for the timeline LSNs on `pageserver` to catch up with or overtake +/// `baseline`. +fn await_lsn( + tenant_id: TenantId, + pageserver: &PageServerNode, + baseline: HashMap, +) -> anyhow::Result<()> { + loop { + let latest = match get_lsns(tenant_id, pageserver) { + Ok(l) => l, + Err(e) => { + println!( + "🕑 Can't get LSNs on pageserver {} yet, waiting ({e})", + pageserver.conf.id + ); + std::thread::sleep(Duration::from_millis(500)); + continue; + } + }; + + let mut any_behind: bool = false; + for (timeline_id, baseline_lsn) in &baseline { + match latest.get(timeline_id) { + Some(latest_lsn) => { + println!("🕑 LSN origin {baseline_lsn} vs destination {latest_lsn}"); + if latest_lsn < baseline_lsn { + any_behind = true; + } + } + None => { + // Expected timeline isn't yet visible on migration destination. + // (IRL we would have to account for timeline deletion, but this + // is just test helper) + any_behind = true; + } + } + } + + if !any_behind { + println!("✅ LSN caught up. Proceeding..."); + break; + } else { + std::thread::sleep(Duration::from_millis(500)); + } + } + + Ok(()) +} + +/// This function spans multiple services, to demonstrate live migration of a tenant +/// between pageservers: +/// - Coordinate attach/secondary/detach on pageservers +/// - call into attachment_service for generations +/// - reconfigure compute endpoints to point to new attached pageserver +pub fn migrate_tenant( + env: &LocalEnv, + tenant_id: TenantId, + dest_ps: PageServerNode, +) -> anyhow::Result<()> { + // Get a new generation + let attachment_service = AttachmentService::from_env(env); + + let previous = attachment_service.inspect(tenant_id)?; + let mut baseline_lsns = None; + if let Some((generation, origin_ps_id)) = &previous { + let origin_ps = PageServerNode::from_env(env, env.get_pageserver_conf(*origin_ps_id)?); + + if origin_ps_id == &dest_ps.conf.id { + println!("🔁 Already attached to {origin_ps_id}, freshening..."); + let gen = attachment_service.attach_hook(tenant_id, dest_ps.conf.id)?; + let dest_conf = LocationConfig { + mode: LocationConfigMode::AttachedSingle, + generation: gen.map(Generation::new), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + dest_ps.location_config(tenant_id, dest_conf)?; + println!("✅ Migration complete"); + return Ok(()); + } + + println!("🔁 Switching origin pageserver {origin_ps_id} to stale mode"); + + let stale_conf = LocationConfig { + mode: LocationConfigMode::AttachedStale, + generation: Some(Generation::new(*generation)), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + origin_ps.location_config(tenant_id, stale_conf)?; + + baseline_lsns = Some(get_lsns(tenant_id, &origin_ps)?); + } + + let gen = attachment_service.attach_hook(tenant_id, dest_ps.conf.id)?; + let dest_conf = LocationConfig { + mode: LocationConfigMode::AttachedMulti, + generation: gen.map(Generation::new), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + + println!("🔁 Attaching to pageserver {}", dest_ps.conf.id); + dest_ps.location_config(tenant_id, dest_conf)?; + + if let Some(baseline) = baseline_lsns { + println!("🕑 Waiting for LSN to catch up..."); + await_lsn(tenant_id, &dest_ps, baseline)?; + } + + let cplane = ComputeControlPlane::load(env.clone())?; + for (endpoint_name, endpoint) in &cplane.endpoints { + if endpoint.tenant_id == tenant_id { + println!( + "🔁 Reconfiguring endpoint {} to use pageserver {}", + endpoint_name, dest_ps.conf.id + ); + endpoint.reconfigure(Some(dest_ps.conf.id))?; + } + } + + for other_ps_conf in &env.pageservers { + if other_ps_conf.id == dest_ps.conf.id { + continue; + } + + let other_ps = PageServerNode::from_env(env, other_ps_conf); + let other_ps_tenants = other_ps.tenant_list()?; + + // Check if this tenant is attached + let found = other_ps_tenants + .into_iter() + .map(|t| t.id) + .any(|i| i == tenant_id); + if !found { + continue; + } + + // Downgrade to a secondary location + let secondary_conf = LocationConfig { + mode: LocationConfigMode::Secondary, + generation: None, + secondary_conf: Some(LocationConfigSecondary { warm: true }), + tenant_conf: TenantConfig::default(), + }; + + println!( + "💤 Switching to secondary mode on pageserver {}", + other_ps.conf.id + ); + other_ps.location_config(tenant_id, secondary_conf)?; + } + + println!( + "🔁 Switching to AttachedSingle mode on pageserver {}", + dest_ps.conf.id + ); + let dest_conf = LocationConfig { + mode: LocationConfigMode::AttachedSingle, + generation: gen.map(Generation::new), + secondary_conf: None, + tenant_conf: TenantConfig::default(), + }; + dest_ps.location_config(tenant_id, dest_conf)?; + + println!("✅ Migration complete"); + + Ok(()) +} diff --git a/deny.toml b/deny.toml index f4ea0d4dac..079dcac679 100644 --- a/deny.toml +++ b/deny.toml @@ -74,10 +74,30 @@ highlight = "all" workspace-default-features = "allow" external-default-features = "allow" allow = [] -deny = [] + skip = [] skip-tree = [] +[[bans.deny]] +# we use tokio, the same rationale applies for async-{io,waker,global-executor,executor,channel,lock}, smol +# if you find yourself here while adding a dependency, try "default-features = false", ask around on #rust +name = "async-std" + +[[bans.deny]] +name = "async-io" + +[[bans.deny]] +name = "async-waker" + +[[bans.deny]] +name = "async-global-executor" + +[[bans.deny]] +name = "async-executor" + +[[bans.deny]] +name = "smol" + # This section is considered when running `cargo deny check sources`. # More documentation about the 'sources' section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html diff --git a/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md b/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md index 9f22fc1ee4..836c91fb25 100644 --- a/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md +++ b/docs/rfcs/023-the-state-of-pageserver-tenant-relocation.md @@ -177,7 +177,7 @@ I e during migration create_branch can be called on old pageserver and newly cre The difference of simplistic approach from one described above is that it calls ignore on source tenant first and then calls attach on target pageserver. Approach above does it in opposite order thus opening a possibility for race conditions we strive to avoid. -The approach largely follows this guide: +The approach largely follows this guide: The happy path sequence: diff --git a/libs/compute_api/src/lib.rs b/libs/compute_api/src/lib.rs index b660799ec0..210a52d089 100644 --- a/libs/compute_api/src/lib.rs +++ b/libs/compute_api/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] pub mod requests; pub mod responses; pub mod spec; diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index c16deceebb..175b4461ac 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{TenantId, TimelineId}; use utils::lsn::Lsn; @@ -19,7 +18,6 @@ pub type PgIdent = String; /// Cluster spec or configuration represented as an optional number of /// delta operations + final cluster state description. -#[serde_as] #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ComputeSpec { pub format_version: f32, @@ -50,12 +48,12 @@ pub struct ComputeSpec { // these, and instead set the "neon.tenant_id", "neon.timeline_id", // etc. GUCs in cluster.settings. TODO: Once the control plane has been // updated to fill these fields, we can make these non optional. - #[serde_as(as = "Option")] pub tenant_id: Option, - #[serde_as(as = "Option")] + pub timeline_id: Option, - #[serde_as(as = "Option")] + pub pageserver_connstring: Option, + #[serde(default)] pub safekeeper_connstrings: Vec, @@ -140,14 +138,13 @@ impl RemoteExtSpec { } } -#[serde_as] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] pub enum ComputeMode { /// A read-write node #[default] Primary, /// A read-only node, pinned at a particular LSN - Static(#[serde_as(as = "DisplayFromStr")] Lsn), + Static(Lsn), /// A read-only node that follows the tip of the branch in hot standby mode /// /// Future versions may want to distinguish between replicas with hot standby diff --git a/libs/consumption_metrics/src/lib.rs b/libs/consumption_metrics/src/lib.rs index 9e89327e84..810196aff6 100644 --- a/libs/consumption_metrics/src/lib.rs +++ b/libs/consumption_metrics/src/lib.rs @@ -1,6 +1,6 @@ -//! //! Shared code for consumption metics collection -//! +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use chrono::{DateTime, Utc}; use rand::Rng; use serde::{Deserialize, Serialize}; diff --git a/libs/metrics/src/lib.rs b/libs/metrics/src/lib.rs index 0b3b24c18e..ed375a152f 100644 --- a/libs/metrics/src/lib.rs +++ b/libs/metrics/src/lib.rs @@ -2,6 +2,7 @@ //! make sure that we use the same dep version everywhere. //! Otherwise, we might not see all metrics registered via //! a default registry. +#![deny(clippy::undocumented_unsafe_blocks)] use once_cell::sync::Lazy; use prometheus::core::{AtomicU64, Collector, GenericGauge, GenericGaugeVec}; pub use prometheus::opts; diff --git a/libs/pageserver_api/Cargo.toml b/libs/pageserver_api/Cargo.toml index f97ec54e91..df9796b039 100644 --- a/libs/pageserver_api/Cargo.toml +++ b/libs/pageserver_api/Cargo.toml @@ -17,5 +17,9 @@ postgres_ffi.workspace = true enum-map.workspace = true strum.workspace = true strum_macros.workspace = true +hex.workspace = true workspace_hack.workspace = true + +[dev-dependencies] +bincode.workspace = true diff --git a/libs/pageserver_api/src/control_api.rs b/libs/pageserver_api/src/control_api.rs index 3819111a5b..8232e81b98 100644 --- a/libs/pageserver_api/src/control_api.rs +++ b/libs/pageserver_api/src/control_api.rs @@ -4,7 +4,6 @@ //! See docs/rfcs/025-generation-numbers.md use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::{NodeId, TenantId}; #[derive(Serialize, Deserialize)] @@ -12,10 +11,8 @@ pub struct ReAttachRequest { pub node_id: NodeId, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ReAttachResponseTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub gen: u32, } @@ -25,10 +22,8 @@ pub struct ReAttachResponse { pub tenants: Vec, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ValidateRequestTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub gen: u32, } @@ -43,10 +38,8 @@ pub struct ValidateResponse { pub tenants: Vec, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct ValidateResponseTenant { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, pub valid: bool, } diff --git a/libs/pageserver_api/src/key.rs b/libs/pageserver_api/src/key.rs new file mode 100644 index 0000000000..b5350d6384 --- /dev/null +++ b/libs/pageserver_api/src/key.rs @@ -0,0 +1,142 @@ +use anyhow::{bail, Result}; +use byteorder::{ByteOrder, BE}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// Key used in the Repository kv-store. +/// +/// The Repository treats this as an opaque struct, but see the code in pgdatadir_mapping.rs +/// for what we actually store in these fields. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct Key { + pub field1: u8, + pub field2: u32, + pub field3: u32, + pub field4: u32, + pub field5: u8, + pub field6: u32, +} + +pub const KEY_SIZE: usize = 18; + +impl Key { + /// 'field2' is used to store tablespaceid for relations and small enum numbers for other relish. + /// As long as Neon does not support tablespace (because of lack of access to local file system), + /// we can assume that only some predefined namespace OIDs are used which can fit in u16 + pub fn to_i128(&self) -> i128 { + assert!(self.field2 < 0xFFFF || self.field2 == 0xFFFFFFFF || self.field2 == 0x22222222); + (((self.field1 & 0xf) as i128) << 120) + | (((self.field2 & 0xFFFF) as i128) << 104) + | ((self.field3 as i128) << 72) + | ((self.field4 as i128) << 40) + | ((self.field5 as i128) << 32) + | self.field6 as i128 + } + + pub const fn from_i128(x: i128) -> Self { + Key { + field1: ((x >> 120) & 0xf) as u8, + field2: ((x >> 104) & 0xFFFF) as u32, + field3: (x >> 72) as u32, + field4: (x >> 40) as u32, + field5: (x >> 32) as u8, + field6: x as u32, + } + } + + pub fn next(&self) -> Key { + self.add(1) + } + + pub fn add(&self, x: u32) -> Key { + let mut key = *self; + + let r = key.field6.overflowing_add(x); + key.field6 = r.0; + if r.1 { + let r = key.field5.overflowing_add(1); + key.field5 = r.0; + if r.1 { + let r = key.field4.overflowing_add(1); + key.field4 = r.0; + if r.1 { + let r = key.field3.overflowing_add(1); + key.field3 = r.0; + if r.1 { + let r = key.field2.overflowing_add(1); + key.field2 = r.0; + if r.1 { + let r = key.field1.overflowing_add(1); + key.field1 = r.0; + assert!(!r.1); + } + } + } + } + } + key + } + + pub fn from_slice(b: &[u8]) -> Self { + Key { + field1: b[0], + field2: u32::from_be_bytes(b[1..5].try_into().unwrap()), + field3: u32::from_be_bytes(b[5..9].try_into().unwrap()), + field4: u32::from_be_bytes(b[9..13].try_into().unwrap()), + field5: b[13], + field6: u32::from_be_bytes(b[14..18].try_into().unwrap()), + } + } + + pub fn write_to_byte_slice(&self, buf: &mut [u8]) { + buf[0] = self.field1; + BE::write_u32(&mut buf[1..5], self.field2); + BE::write_u32(&mut buf[5..9], self.field3); + BE::write_u32(&mut buf[9..13], self.field4); + buf[13] = self.field5; + BE::write_u32(&mut buf[14..18], self.field6); + } +} + +impl fmt::Display for Key { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:02X}{:08X}{:08X}{:08X}{:02X}{:08X}", + self.field1, self.field2, self.field3, self.field4, self.field5, self.field6 + ) + } +} + +impl Key { + pub const MIN: Key = Key { + field1: u8::MIN, + field2: u32::MIN, + field3: u32::MIN, + field4: u32::MIN, + field5: u8::MIN, + field6: u32::MIN, + }; + pub const MAX: Key = Key { + field1: u8::MAX, + field2: u32::MAX, + field3: u32::MAX, + field4: u32::MAX, + field5: u8::MAX, + field6: u32::MAX, + }; + + pub fn from_hex(s: &str) -> Result { + if s.len() != 36 { + bail!("parse error"); + } + Ok(Key { + field1: u8::from_str_radix(&s[0..2], 16)?, + field2: u32::from_str_radix(&s[2..10], 16)?, + field3: u32::from_str_radix(&s[10..18], 16)?, + field4: u32::from_str_radix(&s[18..26], 16)?, + field5: u8::from_str_radix(&s[26..28], 16)?, + field6: u32::from_str_radix(&s[28..36], 16)?, + }) + } +} diff --git a/libs/pageserver_api/src/lib.rs b/libs/pageserver_api/src/lib.rs index d844021785..511c5ed208 100644 --- a/libs/pageserver_api/src/lib.rs +++ b/libs/pageserver_api/src/lib.rs @@ -1,9 +1,13 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use const_format::formatcp; /// Public API types pub mod control_api; +pub mod key; pub mod models; pub mod reltag; +pub mod shard; pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000; pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}"); diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index e667645c0a..71e32e479f 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -6,7 +6,7 @@ use std::{ use byteorder::{BigEndian, ReadBytesExt}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; +use serde_with::serde_as; use strum_macros; use utils::{ completion, @@ -16,7 +16,7 @@ use utils::{ lsn::Lsn, }; -use crate::reltag::RelTag; +use crate::{reltag::RelTag, shard::TenantShardId}; use anyhow::bail; use bytes::{BufMut, Bytes, BytesMut}; @@ -174,26 +174,20 @@ pub enum TimelineState { Broken { reason: String, backtrace: String }, } -#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub new_timeline_id: TimelineId, #[serde(default)] - #[serde_as(as = "Option")] pub ancestor_timeline_id: Option, #[serde(default)] - #[serde_as(as = "Option")] pub ancestor_start_lsn: Option, pub pg_version: Option, } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantCreateRequest { - #[serde_as(as = "DisplayFromStr")] - pub new_tenant_id: TenantId, + pub new_tenant_id: TenantShardId, #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub generation: Option, @@ -201,7 +195,6 @@ pub struct TenantCreateRequest { pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it } -#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantLoadRequest { @@ -278,31 +271,26 @@ pub struct LocationConfig { pub tenant_conf: TenantConfig, } -#[serde_as] #[derive(Serialize, Deserialize)] #[serde(transparent)] -pub struct TenantCreateResponse(#[serde_as(as = "DisplayFromStr")] pub TenantId); +pub struct TenantCreateResponse(pub TenantId); #[derive(Serialize)] pub struct StatusResponse { pub id: NodeId, } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantLocationConfigRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, #[serde(flatten)] pub config: LocationConfig, // as we have a flattened field, we should reject all unknown fields in it } -#[serde_as] #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TenantConfigRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, #[serde(flatten)] pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it @@ -374,10 +362,8 @@ pub enum TenantAttachmentStatus { Failed { reason: String }, } -#[serde_as] #[derive(Serialize, Deserialize, Clone)] pub struct TenantInfo { - #[serde_as(as = "DisplayFromStr")] pub id: TenantId, // NB: intentionally not part of OpenAPI, we don't want to commit to a specific set of TenantState's pub state: TenantState, @@ -388,33 +374,22 @@ pub struct TenantInfo { } /// This represents the output of the "timeline_detail" and "timeline_list" API calls. -#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimelineInfo { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, - #[serde_as(as = "Option")] pub ancestor_timeline_id: Option, - #[serde_as(as = "Option")] pub ancestor_lsn: Option, - #[serde_as(as = "DisplayFromStr")] pub last_record_lsn: Lsn, - #[serde_as(as = "Option")] pub prev_record_lsn: Option, - #[serde_as(as = "DisplayFromStr")] pub latest_gc_cutoff_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, /// The LSN that we have succesfully uploaded to remote storage - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, /// The LSN that we are advertizing to safekeepers - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn_visible: Lsn, pub current_logical_size: Option, // is None when timeline is Unloaded @@ -426,7 +401,6 @@ pub struct TimelineInfo { pub timeline_dir_layer_file_size_sum: Option, pub wal_source_connstr: Option, - #[serde_as(as = "Option")] pub last_received_msg_lsn: Option, /// the timestamp (in microseconds) of the last received message pub last_received_msg_ts: Option, @@ -523,23 +497,13 @@ pub struct LayerAccessStats { pub residence_events_history: HistoryBufferWithDropCounter, } -#[serde_as] #[derive(Debug, Clone, Serialize)] #[serde(tag = "kind")] pub enum InMemoryLayerInfo { - Open { - #[serde_as(as = "DisplayFromStr")] - lsn_start: Lsn, - }, - Frozen { - #[serde_as(as = "DisplayFromStr")] - lsn_start: Lsn, - #[serde_as(as = "DisplayFromStr")] - lsn_end: Lsn, - }, + Open { lsn_start: Lsn }, + Frozen { lsn_start: Lsn, lsn_end: Lsn }, } -#[serde_as] #[derive(Debug, Clone, Serialize)] #[serde(tag = "kind")] pub enum HistoricLayerInfo { @@ -547,9 +511,7 @@ pub enum HistoricLayerInfo { layer_file_name: String, layer_file_size: u64, - #[serde_as(as = "DisplayFromStr")] lsn_start: Lsn, - #[serde_as(as = "DisplayFromStr")] lsn_end: Lsn, remote: bool, access_stats: LayerAccessStats, @@ -558,7 +520,6 @@ pub enum HistoricLayerInfo { layer_file_name: String, layer_file_size: u64, - #[serde_as(as = "DisplayFromStr")] lsn_start: Lsn, remote: bool, access_stats: LayerAccessStats, diff --git a/libs/pageserver_api/src/shard.rs b/libs/pageserver_api/src/shard.rs new file mode 100644 index 0000000000..32a834a26a --- /dev/null +++ b/libs/pageserver_api/src/shard.rs @@ -0,0 +1,321 @@ +use std::{ops::RangeInclusive, str::FromStr}; + +use hex::FromHex; +use serde::{Deserialize, Serialize}; +use utils::id::TenantId; + +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug)] +pub struct ShardNumber(pub u8); + +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug)] +pub struct ShardCount(pub u8); + +impl ShardCount { + pub const MAX: Self = Self(u8::MAX); +} + +impl ShardNumber { + pub const MAX: Self = Self(u8::MAX); +} + +/// TenantShardId identify the units of work for the Pageserver. +/// +/// These are written as `-`, for example: +/// +/// # The second shard in a two-shard tenant +/// 072f1291a5310026820b2fe4b2968934-0102 +/// +/// Historically, tenants could not have multiple shards, and were identified +/// by TenantId. To support this, TenantShardId has a special legacy +/// mode where `shard_count` is equal to zero: this represents a single-sharded +/// tenant which should be written as a TenantId with no suffix. +/// +/// The human-readable encoding of TenantShardId, such as used in API URLs, +/// is both forward and backward compatible: a legacy TenantId can be +/// decoded as a TenantShardId, and when re-encoded it will be parseable +/// as a TenantId. +/// +/// Note that the binary encoding is _not_ backward compatible, because +/// at the time sharding is introduced, there are no existing binary structures +/// containing TenantId that we need to handle. +#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy)] +pub struct TenantShardId { + pub tenant_id: TenantId, + pub shard_number: ShardNumber, + pub shard_count: ShardCount, +} + +impl TenantShardId { + pub fn unsharded(tenant_id: TenantId) -> Self { + Self { + tenant_id, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + } + } + + /// The range of all TenantShardId that belong to a particular TenantId. This is useful when + /// you have a BTreeMap of TenantShardId, and are querying by TenantId. + pub fn tenant_range(tenant_id: TenantId) -> RangeInclusive { + RangeInclusive::new( + Self { + tenant_id, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + }, + Self { + tenant_id, + shard_number: ShardNumber::MAX, + shard_count: ShardCount::MAX, + }, + ) + } + + pub fn shard_slug(&self) -> String { + format!("{:02x}{:02x}", self.shard_number.0, self.shard_count.0) + } +} + +impl std::fmt::Display for TenantShardId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.shard_count != ShardCount(0) { + write!( + f, + "{}-{:02x}{:02x}", + self.tenant_id, self.shard_number.0, self.shard_count.0 + ) + } else { + // Legacy case (shard_count == 0) -- format as just the tenant id. Note that this + // is distinct from the normal single shard case (shard count == 1). + self.tenant_id.fmt(f) + } + } +} + +impl std::fmt::Debug for TenantShardId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Debug is the same as Display: the compact hex representation + write!(f, "{}", self) + } +} + +impl std::str::FromStr for TenantShardId { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + // Expect format: 16 byte TenantId, '-', 1 byte shard number, 1 byte shard count + if s.len() == 32 { + // Legacy case: no shard specified + Ok(Self { + tenant_id: TenantId::from_str(s)?, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + }) + } else if s.len() == 37 { + let bytes = s.as_bytes(); + let tenant_id = TenantId::from_hex(&bytes[0..32])?; + let mut shard_parts: [u8; 2] = [0u8; 2]; + hex::decode_to_slice(&bytes[33..37], &mut shard_parts)?; + Ok(Self { + tenant_id, + shard_number: ShardNumber(shard_parts[0]), + shard_count: ShardCount(shard_parts[1]), + }) + } else { + Err(hex::FromHexError::InvalidStringLength) + } + } +} + +impl From<[u8; 18]> for TenantShardId { + fn from(b: [u8; 18]) -> Self { + let tenant_id_bytes: [u8; 16] = b[0..16].try_into().unwrap(); + + Self { + tenant_id: TenantId::from(tenant_id_bytes), + shard_number: ShardNumber(b[16]), + shard_count: ShardCount(b[17]), + } + } +} + +impl Serialize for TenantShardId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + let mut packed: [u8; 18] = [0; 18]; + packed[0..16].clone_from_slice(&self.tenant_id.as_arr()); + packed[16] = self.shard_number.0; + packed[17] = self.shard_count.0; + + packed.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for TenantShardId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> serde::de::Visitor<'de> for IdVisitor { + type Value = TenantShardId; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 18])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 18] = Deserialize::deserialize(s)?; + Ok(TenantShardId::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + TenantShardId::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 18, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use bincode; + use utils::{id::TenantId, Hex}; + + use super::*; + + const EXAMPLE_TENANT_ID: &str = "1f359dd625e519a1a4e8d7509690f6fc"; + + #[test] + fn tenant_shard_id_string() -> Result<(), hex::FromHexError> { + let example = TenantShardId { + tenant_id: TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(), + shard_count: ShardCount(10), + shard_number: ShardNumber(7), + }; + + let encoded = format!("{example}"); + + let expected = format!("{EXAMPLE_TENANT_ID}-070a"); + assert_eq!(&encoded, &expected); + + let decoded = TenantShardId::from_str(&encoded)?; + + assert_eq!(example, decoded); + + Ok(()) + } + + #[test] + fn tenant_shard_id_binary() -> Result<(), hex::FromHexError> { + let example = TenantShardId { + tenant_id: TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(), + shard_count: ShardCount(10), + shard_number: ShardNumber(7), + }; + + let encoded = bincode::serialize(&example).unwrap(); + let expected: [u8; 18] = [ + 0x1f, 0x35, 0x9d, 0xd6, 0x25, 0xe5, 0x19, 0xa1, 0xa4, 0xe8, 0xd7, 0x50, 0x96, 0x90, + 0xf6, 0xfc, 0x07, 0x0a, + ]; + assert_eq!(Hex(&encoded), Hex(&expected)); + + let decoded = bincode::deserialize(&encoded).unwrap(); + + assert_eq!(example, decoded); + + Ok(()) + } + + #[test] + fn tenant_shard_id_backward_compat() -> Result<(), hex::FromHexError> { + // Test that TenantShardId can decode a TenantId in human + // readable form + let example = TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(); + let encoded = format!("{example}"); + + assert_eq!(&encoded, EXAMPLE_TENANT_ID); + + let decoded = TenantShardId::from_str(&encoded)?; + + assert_eq!(example, decoded.tenant_id); + assert_eq!(decoded.shard_count, ShardCount(0)); + assert_eq!(decoded.shard_number, ShardNumber(0)); + + Ok(()) + } + + #[test] + fn tenant_shard_id_forward_compat() -> Result<(), hex::FromHexError> { + // Test that a legacy TenantShardId encodes into a form that + // can be decoded as TenantId + let example_tenant_id = TenantId::from_str(EXAMPLE_TENANT_ID).unwrap(); + let example = TenantShardId::unsharded(example_tenant_id); + let encoded = format!("{example}"); + + assert_eq!(&encoded, EXAMPLE_TENANT_ID); + + let decoded = TenantId::from_str(&encoded)?; + + assert_eq!(example_tenant_id, decoded); + + Ok(()) + } + + #[test] + fn tenant_shard_id_legacy_binary() -> Result<(), hex::FromHexError> { + // Unlike in human readable encoding, binary encoding does not + // do any special handling of legacy unsharded TenantIds: this test + // is equivalent to the main test for binary encoding, just verifying + // that the same behavior applies when we have used `unsharded()` to + // construct a TenantShardId. + let example = TenantShardId::unsharded(TenantId::from_str(EXAMPLE_TENANT_ID).unwrap()); + let encoded = bincode::serialize(&example).unwrap(); + + let expected: [u8; 18] = [ + 0x1f, 0x35, 0x9d, 0xd6, 0x25, 0xe5, 0x19, 0xa1, 0xa4, 0xe8, 0xd7, 0x50, 0x96, 0x90, + 0xf6, 0xfc, 0x00, 0x00, + ]; + assert_eq!(Hex(&encoded), Hex(&expected)); + + let decoded = bincode::deserialize::(&encoded).unwrap(); + assert_eq!(example, decoded); + + Ok(()) + } +} diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index ad3af4e794..1dae008a4f 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -2,6 +2,8 @@ //! To use, create PostgresBackend and run() it, passing the Handler //! implementation determining how to process the queries. Currently its API //! is rather narrow, but we can extend it once required. +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use anyhow::Context; use bytes::Bytes; use futures::pin_mut; @@ -15,7 +17,7 @@ use std::{fmt, io}; use std::{future::Future, str::FromStr}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_rustls::TlsAcceptor; -use tracing::{debug, error, info, trace}; +use tracing::{debug, error, info, trace, warn}; use pq_proto::framed::{ConnectionError, Framed, FramedReader, FramedWriter}; use pq_proto::{ @@ -33,6 +35,11 @@ pub enum QueryError { /// We were instructed to shutdown while processing the query #[error("Shutting down")] Shutdown, + /// Authentication failure + #[error("Unauthorized: {0}")] + Unauthorized(std::borrow::Cow<'static, str>), + #[error("Simulated Connection Error")] + SimulatedConnectionError, /// Some other error #[error(transparent)] Other(#[from] anyhow::Error), @@ -47,8 +54,9 @@ impl From for QueryError { impl QueryError { pub fn pg_error_code(&self) -> &'static [u8; 5] { match self { - Self::Disconnected(_) => b"08006", // connection failure + Self::Disconnected(_) | Self::SimulatedConnectionError => b"08006", // connection failure Self::Shutdown => SQLSTATE_ADMIN_SHUTDOWN, + Self::Unauthorized(_) => SQLSTATE_INTERNAL_ERROR, Self::Other(_) => SQLSTATE_INTERNAL_ERROR, // internal error } } @@ -608,7 +616,7 @@ impl PostgresBackend { if let Err(e) = handler.check_auth_jwt(self, jwt_response) { self.write_message_noflush(&BeMessage::ErrorResponse( - &e.to_string(), + &short_error(&e), Some(e.pg_error_code()), ))?; return Err(e); @@ -728,12 +736,20 @@ impl PostgresBackend { trace!("got query {query_string:?}"); if let Err(e) = handler.process_query(self, query_string).await { - log_query_error(query_string, &e); - let short_error = short_error(&e); - self.write_message_noflush(&BeMessage::ErrorResponse( - &short_error, - Some(e.pg_error_code()), - ))?; + match e { + QueryError::Shutdown => return Ok(ProcessMsgResult::Break), + QueryError::SimulatedConnectionError => { + return Err(QueryError::SimulatedConnectionError) + } + e => { + log_query_error(query_string, &e); + let short_error = short_error(&e); + self.write_message_noflush(&BeMessage::ErrorResponse( + &short_error, + Some(e.pg_error_code()), + ))?; + } + } } self.write_message_noflush(&BeMessage::ReadyForQuery)?; } @@ -959,6 +975,8 @@ pub fn short_error(e: &QueryError) -> String { match e { QueryError::Disconnected(connection_error) => connection_error.to_string(), QueryError::Shutdown => "shutdown".to_string(), + QueryError::Unauthorized(_e) => "JWT authentication error".to_string(), + QueryError::SimulatedConnectionError => "simulated connection error".to_string(), QueryError::Other(e) => format!("{e:#}"), } } @@ -975,9 +993,15 @@ fn log_query_error(query: &str, e: &QueryError) { QueryError::Disconnected(other_connection_error) => { error!("query handler for '{query}' failed with connection error: {other_connection_error:?}") } + QueryError::SimulatedConnectionError => { + error!("query handler for query '{query}' failed due to a simulated connection error") + } QueryError::Shutdown => { info!("query handler for '{query}' cancelled during tenant shutdown") } + QueryError::Unauthorized(e) => { + warn!("query handler for '{query}' failed with authentication error: {e}"); + } QueryError::Other(e) => { error!("query handler for '{query}' failed: {e:?}"); } diff --git a/libs/postgres_connection/src/lib.rs b/libs/postgres_connection/src/lib.rs index 35344a9168..35cb1a2691 100644 --- a/libs/postgres_connection/src/lib.rs +++ b/libs/postgres_connection/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use anyhow::{bail, Context}; use itertools::Itertools; use std::borrow::Cow; diff --git a/libs/postgres_ffi/src/lib.rs b/libs/postgres_ffi/src/lib.rs index d0009d0dc9..d10ebfe277 100644 --- a/libs/postgres_ffi/src/lib.rs +++ b/libs/postgres_ffi/src/lib.rs @@ -8,6 +8,7 @@ // modules included with the postgres_ffi macro depend on the types of the specific version's // types, and trigger a too eager lint. #![allow(clippy::duplicate_mod)] +#![deny(clippy::undocumented_unsafe_blocks)] use bytes::Bytes; use utils::bin_ser::SerializeError; @@ -20,6 +21,7 @@ macro_rules! postgres_ffi { pub mod bindings { // bindgen generates bindings for a lot of stuff we don't need #![allow(dead_code)] + #![allow(clippy::undocumented_unsafe_blocks)] use serde::{Deserialize, Serialize}; include!(concat!( diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs index 94bc7f60f8..41fc206cd7 100644 --- a/libs/pq_proto/src/lib.rs +++ b/libs/pq_proto/src/lib.rs @@ -1,6 +1,7 @@ //! Postgres protocol messages serialization-deserialization. See //! //! on message formats. +#![deny(clippy::undocumented_unsafe_blocks)] pub mod framed; diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index 65b5392389..d7bcce28cb 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true anyhow.workspace = true async-trait.workspace = true once_cell.workspace = true +aws-smithy-async.workspace = true aws-smithy-http.workspace = true aws-types.workspace = true aws-config.workspace = true diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index f0521d27c5..ae08e9b171 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -1,21 +1,18 @@ //! Azure Blob Storage wrapper +use std::collections::HashMap; use std::env; use std::num::NonZeroU32; use std::sync::Arc; -use std::{borrow::Cow, collections::HashMap, io::Cursor}; +use std::{borrow::Cow, io::Cursor}; use super::REMOTE_STORAGE_PREFIX_SEPARATOR; use anyhow::Result; use azure_core::request_options::{MaxResults, Metadata, Range}; -use azure_core::Header; use azure_identity::DefaultAzureCredential; use azure_storage::StorageCredentials; use azure_storage_blobs::prelude::ClientBuilder; -use azure_storage_blobs::{ - blob::operations::GetBlobBuilder, - prelude::{BlobClient, ContainerClient}, -}; +use azure_storage_blobs::{blob::operations::GetBlobBuilder, prelude::ContainerClient}; use futures_util::StreamExt; use http_types::StatusCode; use tokio::io::AsyncRead; @@ -112,16 +109,19 @@ impl AzureBlobStorage { async fn download_for_builder( &self, - metadata: StorageMetadata, builder: GetBlobBuilder, ) -> Result { let mut response = builder.into_stream(); + let mut metadata = HashMap::new(); // TODO give proper streaming response instead of buffering into RAM // https://github.com/neondatabase/neon/issues/5563 let mut buf = Vec::new(); while let Some(part) = response.next().await { let part = part.map_err(to_download_error)?; + if let Some(blob_meta) = part.blob.metadata { + metadata.extend(blob_meta.iter().map(|(k, v)| (k.to_owned(), v.to_owned()))); + } let data = part .data .collect() @@ -131,28 +131,9 @@ impl AzureBlobStorage { } Ok(Download { download_stream: Box::pin(Cursor::new(buf)), - metadata: Some(metadata), + metadata: Some(StorageMetadata(metadata)), }) } - // TODO get rid of this function once we have metadata included in the response - // https://github.com/Azure/azure-sdk-for-rust/issues/1439 - async fn get_metadata( - &self, - blob_client: &BlobClient, - ) -> Result { - let builder = blob_client.get_metadata(); - - let response = builder.into_future().await.map_err(to_download_error)?; - let mut map = HashMap::new(); - - for md in response.metadata.iter() { - map.insert( - md.name().as_str().to_string(), - md.value().as_str().to_string(), - ); - } - Ok(StorageMetadata(map)) - } async fn permit(&self, kind: RequestKind) -> tokio::sync::SemaphorePermit<'_> { self.concurrency_limiter @@ -269,11 +250,9 @@ impl RemoteStorage for AzureBlobStorage { let _permit = self.permit(RequestKind::Get).await; let blob_client = self.client.blob_client(self.relative_path_to_name(from)); - let metadata = self.get_metadata(&blob_client).await?; - let builder = blob_client.get(); - self.download_for_builder(metadata, builder).await + self.download_for_builder(builder).await } async fn download_byte_range( @@ -285,8 +264,6 @@ impl RemoteStorage for AzureBlobStorage { let _permit = self.permit(RequestKind::Get).await; let blob_client = self.client.blob_client(self.relative_path_to_name(from)); - let metadata = self.get_metadata(&blob_client).await?; - let mut builder = blob_client.get(); if let Some(end_exclusive) = end_exclusive { @@ -301,7 +278,7 @@ impl RemoteStorage for AzureBlobStorage { builder = builder.range(Range::new(start_inclusive, end_exclusive)); } - self.download_for_builder(metadata, builder).await + self.download_for_builder(builder).await } async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> { diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 9b5b340db0..e6d306ff66 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -6,19 +6,15 @@ //! * [`s3_bucket`] uses AWS S3 bucket as an external storage //! * [`azure_blob`] allows to use Azure Blob storage as an external storage //! +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] mod azure_blob; mod local_fs; mod s3_bucket; mod simulate_failures; -use std::{ - collections::HashMap, - fmt::Debug, - num::{NonZeroU32, NonZeroUsize}, - pin::Pin, - sync::Arc, -}; +use std::{collections::HashMap, fmt::Debug, num::NonZeroUsize, pin::Pin, sync::Arc}; use anyhow::{bail, Context}; use camino::{Utf8Path, Utf8PathBuf}; @@ -34,12 +30,6 @@ pub use self::{ }; use s3_bucket::RequestKind; -/// How many different timelines can be processed simultaneously when synchronizing layers with the remote storage. -/// During regular work, pageserver produces one layer file per timeline checkpoint, with bursts of concurrency -/// during start (where local and remote timelines are compared and initial sync tasks are scheduled) and timeline attach. -/// Both cases may trigger timeline download, that might download a lot of layers. This concurrency is limited by the clients internally, if needed. -pub const DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS: usize = 50; -pub const DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS: u32 = 10; /// Currently, sync happens with AWS S3, that has two limits on requests per second: /// ~200 RPS for IAM services /// @@ -441,10 +431,6 @@ pub struct StorageMetadata(HashMap); /// External backup storage configuration, enough for creating a client for that storage. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RemoteStorageConfig { - /// Max allowed number of concurrent sync operations between the API user and the remote storage. - pub max_concurrent_syncs: NonZeroUsize, - /// Max allowed errors before the sync task is considered failed and evicted. - pub max_sync_errors: NonZeroU32, /// The storage connection configuration. pub storage: RemoteStorageKind, } @@ -540,18 +526,6 @@ impl RemoteStorageConfig { let use_azure = container_name.is_some() && container_region.is_some(); - let max_concurrent_syncs = NonZeroUsize::new( - parse_optional_integer("max_concurrent_syncs", toml)? - .unwrap_or(DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS), - ) - .context("Failed to parse 'max_concurrent_syncs' as a positive integer")?; - - let max_sync_errors = NonZeroU32::new( - parse_optional_integer("max_sync_errors", toml)? - .unwrap_or(DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS), - ) - .context("Failed to parse 'max_sync_errors' as a positive integer")?; - let default_concurrency_limit = if use_azure { DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT } else { @@ -633,11 +607,7 @@ impl RemoteStorageConfig { } }; - Ok(Some(RemoteStorageConfig { - max_concurrent_syncs, - max_sync_errors, - storage, - })) + Ok(Some(RemoteStorageConfig { storage })) } } diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 560a2c14e9..ab3fd3fe62 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -4,23 +4,27 @@ //! allowing multiple api users to independently work with the same S3 bucket, if //! their bucket prefixes are both specified and different. -use std::borrow::Cow; +use std::{borrow::Cow, sync::Arc}; use anyhow::Context; use aws_config::{ environment::credentials::EnvironmentVariableCredentialsProvider, - imds::credentials::ImdsCredentialsProvider, meta::credentials::CredentialsProviderChain, - provider_config::ProviderConfig, web_identity_token::WebIdentityTokenCredentialsProvider, + imds::credentials::ImdsCredentialsProvider, + meta::credentials::CredentialsProviderChain, + provider_config::ProviderConfig, + retry::{RetryConfigBuilder, RetryMode}, + web_identity_token::WebIdentityTokenCredentialsProvider, }; use aws_credential_types::cache::CredentialsCache; use aws_sdk_s3::{ - config::{Config, Region}, + config::{AsyncSleep, Config, Region, SharedAsyncSleep}, error::SdkError, operation::get_object::GetObjectError, primitives::ByteStream, types::{Delete, ObjectIdentifier}, Client, }; +use aws_smithy_async::rt::sleep::TokioSleep; use aws_smithy_http::body::SdkBody; use hyper::Body; use scopeguard::ScopeGuard; @@ -83,10 +87,23 @@ impl S3Bucket { .or_else("imds", ImdsCredentialsProvider::builder().build()) }; + // AWS SDK requires us to specify how the RetryConfig should sleep when it wants to back off + let sleep_impl: Arc = Arc::new(TokioSleep::new()); + + // We do our own retries (see [`backoff::retry`]). However, for the AWS SDK to enable rate limiting in response to throttling + // responses (e.g. 429 on too many ListObjectsv2 requests), we must provide a retry config. We set it to use at most one + // attempt, and enable 'Adaptive' mode, which causes rate limiting to be enabled. + let mut retry_config = RetryConfigBuilder::new(); + retry_config + .set_max_attempts(Some(1)) + .set_mode(Some(RetryMode::Adaptive)); + let mut config_builder = Config::builder() .region(region) .credentials_cache(CredentialsCache::lazy()) - .credentials_provider(credentials_provider); + .credentials_provider(credentials_provider) + .sleep_impl(SharedAsyncSleep::from(sleep_impl)) + .retry_config(retry_config.build()); if let Some(custom_endpoint) = aws_config.endpoint.clone() { config_builder = config_builder diff --git a/libs/remote_storage/tests/test_real_azure.rs b/libs/remote_storage/tests/test_real_azure.rs index 0338270aaf..59b92b48fb 100644 --- a/libs/remote_storage/tests/test_real_azure.rs +++ b/libs/remote_storage/tests/test_real_azure.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; use std::env; -use std::num::{NonZeroU32, NonZeroUsize}; +use std::num::NonZeroUsize; use std::ops::ControlFlow; use std::path::PathBuf; use std::sync::Arc; @@ -469,8 +469,6 @@ fn create_azure_client( let random = rand::thread_rng().gen::(); let remote_storage_config = RemoteStorageConfig { - max_concurrent_syncs: NonZeroUsize::new(100).unwrap(), - max_sync_errors: NonZeroU32::new(5).unwrap(), storage: RemoteStorageKind::AzureContainer(AzureConfig { container_name: remote_storage_azure_container, container_region: remote_storage_azure_region, diff --git a/libs/remote_storage/tests/test_real_s3.rs b/libs/remote_storage/tests/test_real_s3.rs index 7e2aa9f6d7..e2360b7533 100644 --- a/libs/remote_storage/tests/test_real_s3.rs +++ b/libs/remote_storage/tests/test_real_s3.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; use std::env; -use std::num::{NonZeroU32, NonZeroUsize}; +use std::num::NonZeroUsize; use std::ops::ControlFlow; use std::path::PathBuf; use std::sync::Arc; @@ -396,8 +396,6 @@ fn create_s3_client( let random = rand::thread_rng().gen::(); let remote_storage_config = RemoteStorageConfig { - max_concurrent_syncs: NonZeroUsize::new(100).unwrap(), - max_sync_errors: NonZeroU32::new(5).unwrap(), storage: RemoteStorageKind::AwsS3(S3Config { bucket_name: remote_storage_s3_bucket, bucket_region: remote_storage_s3_region, diff --git a/libs/safekeeper_api/src/lib.rs b/libs/safekeeper_api/src/lib.rs index 0a391478da..63c2c51188 100644 --- a/libs/safekeeper_api/src/lib.rs +++ b/libs/safekeeper_api/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use const_format::formatcp; /// Public API types diff --git a/libs/safekeeper_api/src/models.rs b/libs/safekeeper_api/src/models.rs index eaea266f32..786712deb1 100644 --- a/libs/safekeeper_api/src/models.rs +++ b/libs/safekeeper_api/src/models.rs @@ -1,23 +1,18 @@ use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::{ id::{NodeId, TenantId, TimelineId}, lsn::Lsn, }; -#[serde_as] #[derive(Serialize, Deserialize)] pub struct TimelineCreateRequest { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub peer_ids: Option>, pub pg_version: u32, pub system_id: Option, pub wal_seg_size: Option, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, // If not passed, it is assigned to the beginning of commit_lsn segment. pub local_start_lsn: Option, @@ -28,7 +23,6 @@ fn lsn_invalid() -> Lsn { } /// Data about safekeeper's timeline, mirrors broker.proto. -#[serde_as] #[derive(Debug, Clone, Deserialize, Serialize)] pub struct SkTimelineInfo { /// Term. @@ -36,25 +30,19 @@ pub struct SkTimelineInfo { /// Term of the last entry. pub last_log_term: Option, /// LSN of the last record. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub flush_lsn: Lsn, /// Up to which LSN safekeeper regards its WAL as committed. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub commit_lsn: Lsn, /// LSN up to which safekeeper has backed WAL. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub backup_lsn: Lsn, /// LSN of last checkpoint uploaded by pageserver. - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub remote_consistent_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub peer_horizon_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] #[serde(default = "lsn_invalid")] pub local_start_lsn: Lsn, /// A connection string to use for WAL receiving. diff --git a/libs/tenant_size_model/src/lib.rs b/libs/tenant_size_model/src/lib.rs index c151e3b42c..a3e12cf0e3 100644 --- a/libs/tenant_size_model/src/lib.rs +++ b/libs/tenant_size_model/src/lib.rs @@ -1,4 +1,6 @@ //! Synthetic size calculation +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] mod calculation; pub mod svg; diff --git a/libs/tracing-utils/src/lib.rs b/libs/tracing-utils/src/lib.rs index de0e2ad799..9cf2495771 100644 --- a/libs/tracing-utils/src/lib.rs +++ b/libs/tracing-utils/src/lib.rs @@ -32,6 +32,8 @@ //! .init(); //! } //! ``` +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] use opentelemetry::sdk::Resource; use opentelemetry::KeyValue; diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 27df0265b4..ccf6f4f2d7 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true [dependencies] +arc-swap.workspace = true sentry.workspace = true async-trait.workspace = true anyhow.workspace = true @@ -55,6 +56,7 @@ bytes.workspace = true criterion.workspace = true hex-literal.workspace = true camino-tempfile.workspace = true +serde_assert.workspace = true [[bench]] name = "benchmarks" diff --git a/libs/utils/src/auth.rs b/libs/utils/src/auth.rs index 54b90fa070..66b1f6e866 100644 --- a/libs/utils/src/auth.rs +++ b/libs/utils/src/auth.rs @@ -1,7 +1,8 @@ // For details about authentication see docs/authentication.md +use arc_swap::ArcSwap; use serde; -use std::fs; +use std::{borrow::Cow, fmt::Display, fs, sync::Arc}; use anyhow::Result; use camino::Utf8Path; @@ -9,9 +10,8 @@ use jsonwebtoken::{ decode, encode, Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, }; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; -use crate::id::TenantId; +use crate::{http::error::ApiError, id::TenantId}; /// Algorithm to use. We require EdDSA. const STORAGE_TOKEN_ALGORITHM: Algorithm = Algorithm::EdDSA; @@ -32,11 +32,9 @@ pub enum Scope { } /// JWT payload. See docs/authentication.md for the format -#[serde_as] #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct Claims { #[serde(default)] - #[serde_as(as = "Option")] pub tenant_id: Option, pub scope: Scope, } @@ -47,31 +45,106 @@ impl Claims { } } +pub struct SwappableJwtAuth(ArcSwap); + +impl SwappableJwtAuth { + pub fn new(jwt_auth: JwtAuth) -> Self { + SwappableJwtAuth(ArcSwap::new(Arc::new(jwt_auth))) + } + pub fn swap(&self, jwt_auth: JwtAuth) { + self.0.swap(Arc::new(jwt_auth)); + } + pub fn decode(&self, token: &str) -> std::result::Result, AuthError> { + self.0.load().decode(token) + } +} + +impl std::fmt::Debug for SwappableJwtAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Swappable({:?})", self.0.load()) + } +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct AuthError(pub Cow<'static, str>); + +impl Display for AuthError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for ApiError { + fn from(_value: AuthError) -> Self { + // Don't pass on the value of the AuthError as a precautionary measure. + // Being intentionally vague in public error communication hurts debugability + // but it is more secure. + ApiError::Forbidden("JWT authentication error".to_string()) + } +} + pub struct JwtAuth { - decoding_key: DecodingKey, + decoding_keys: Vec, validation: Validation, } impl JwtAuth { - pub fn new(decoding_key: DecodingKey) -> Self { + pub fn new(decoding_keys: Vec) -> Self { let mut validation = Validation::default(); validation.algorithms = vec![STORAGE_TOKEN_ALGORITHM]; // The default 'required_spec_claims' is 'exp'. But we don't want to require // expiration. validation.required_spec_claims = [].into(); Self { - decoding_key, + decoding_keys, validation, } } pub fn from_key_path(key_path: &Utf8Path) -> Result { - let public_key = fs::read(key_path)?; - Ok(Self::new(DecodingKey::from_ed_pem(&public_key)?)) + let metadata = key_path.metadata()?; + let decoding_keys = if metadata.is_dir() { + let mut keys = Vec::new(); + for entry in fs::read_dir(key_path)? { + let path = entry?.path(); + if !path.is_file() { + // Ignore directories (don't recurse) + continue; + } + let public_key = fs::read(path)?; + keys.push(DecodingKey::from_ed_pem(&public_key)?); + } + keys + } else if metadata.is_file() { + let public_key = fs::read(key_path)?; + vec![DecodingKey::from_ed_pem(&public_key)?] + } else { + anyhow::bail!("path is neither a directory or a file") + }; + if decoding_keys.is_empty() { + anyhow::bail!("Configured for JWT auth with zero decoding keys. All JWT gated requests would be rejected."); + } + Ok(Self::new(decoding_keys)) } - pub fn decode(&self, token: &str) -> Result> { - Ok(decode(token, &self.decoding_key, &self.validation)?) + /// Attempt to decode the token with the internal decoding keys. + /// + /// The function tries the stored decoding keys in succession, + /// and returns the first yielding a successful result. + /// If there is no working decoding key, it returns the last error. + pub fn decode(&self, token: &str) -> std::result::Result, AuthError> { + let mut res = None; + for decoding_key in &self.decoding_keys { + res = Some(decode(token, decoding_key, &self.validation)); + if let Some(Ok(res)) = res { + return Ok(res); + } + } + if let Some(res) = res { + res.map_err(|e| AuthError(Cow::Owned(e.to_string()))) + } else { + Err(AuthError(Cow::Borrowed("no JWT decoding keys configured"))) + } } } @@ -111,9 +184,9 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH "#; #[test] - fn test_decode() -> Result<(), anyhow::Error> { + fn test_decode() { let expected_claims = Claims { - tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081")?), + tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081").unwrap()), scope: Scope::Tenant, }; @@ -132,28 +205,24 @@ MC4CAQAwBQYDK2VwBCIEID/Drmc1AA6U/znNRWpF3zEGegOATQxfkdWxitcOMsIH let encoded_eddsa = "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJzY29wZSI6InRlbmFudCIsInRlbmFudF9pZCI6IjNkMWY3NTk1YjQ2ODIzMDMwNGUwYjczY2VjYmNiMDgxIiwiaXNzIjoibmVvbi5jb250cm9scGxhbmUiLCJleHAiOjE3MDkyMDA4NzksImlhdCI6MTY3ODQ0MjQ3OX0.U3eA8j-uU-JnhzeO3EDHRuXLwkAUFCPxtGHEgw6p7Ccc3YRbFs2tmCdbD9PZEXP-XsxSeBQi1FY0YPcT3NXADw"; // Check it can be validated with the public key - let auth = JwtAuth::new(DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?); - let claims_from_token = auth.decode(encoded_eddsa)?.claims; + let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519).unwrap()]); + let claims_from_token = auth.decode(encoded_eddsa).unwrap().claims; assert_eq!(claims_from_token, expected_claims); - - Ok(()) } #[test] - fn test_encode() -> Result<(), anyhow::Error> { + fn test_encode() { let claims = Claims { - tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081")?), + tenant_id: Some(TenantId::from_str("3d1f7595b468230304e0b73cecbcb081").unwrap()), scope: Scope::Tenant, }; - let encoded = encode_from_key_file(&claims, TEST_PRIV_KEY_ED25519)?; + let encoded = encode_from_key_file(&claims, TEST_PRIV_KEY_ED25519).unwrap(); // decode it back - let auth = JwtAuth::new(DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519)?); - let decoded = auth.decode(&encoded)?; + let auth = JwtAuth::new(vec![DecodingKey::from_ed_pem(TEST_PUB_KEY_ED25519).unwrap()]); + let decoded = auth.decode(&encoded).unwrap(); assert_eq!(decoded.claims, claims); - - Ok(()) } } diff --git a/libs/utils/src/generation.rs b/libs/utils/src/generation.rs index 88d50905c6..49e290dab8 100644 --- a/libs/utils/src/generation.rs +++ b/libs/utils/src/generation.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; /// /// See docs/rfcs/025-generation-numbers.md for detail on how generation /// numbers are used. -#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)] pub enum Generation { // Generations with this magic value will not add a suffix to S3 keys, and will not // be included in persisted index_part.json. This value is only to be used diff --git a/libs/utils/src/hex.rs b/libs/utils/src/hex.rs new file mode 100644 index 0000000000..fc0bb7e4a2 --- /dev/null +++ b/libs/utils/src/hex.rs @@ -0,0 +1,41 @@ +/// Useful type for asserting that expected bytes match reporting the bytes more readable +/// array-syntax compatible hex bytes. +/// +/// # Usage +/// +/// ``` +/// use utils::Hex; +/// +/// let actual = serialize_something(); +/// let expected = [0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64]; +/// +/// // the type implements PartialEq and on mismatch, both sides are printed in 16 wide multiline +/// // output suffixed with an array style length for easier comparisons. +/// assert_eq!(Hex(&actual), Hex(&expected)); +/// +/// // with `let expected = [0x68];` the error would had been: +/// // assertion `left == right` failed +/// // left: [0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64; 11] +/// // right: [0x68; 1] +/// # fn serialize_something() -> Vec { "hello world".as_bytes().to_vec() } +/// ``` +#[derive(PartialEq)] +pub struct Hex<'a>(pub &'a [u8]); + +impl std::fmt::Debug for Hex<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for (i, c) in self.0.chunks(16).enumerate() { + if i > 0 && !c.is_empty() { + writeln!(f, ", ")?; + } + for (j, b) in c.iter().enumerate() { + if j > 0 { + write!(f, ", ")?; + } + write!(f, "0x{b:02x}")?; + } + } + write!(f, "; {}]", self.0.len()) + } +} diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index f3f5e95d0b..550ab10700 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -1,4 +1,4 @@ -use crate::auth::{Claims, JwtAuth}; +use crate::auth::{AuthError, Claims, SwappableJwtAuth}; use crate::http::error::{api_error_handler, route_error_handler, ApiError}; use anyhow::Context; use hyper::header::{HeaderName, AUTHORIZATION}; @@ -14,6 +14,11 @@ use tracing::{self, debug, info, info_span, warn, Instrument}; use std::future::Future; use std::str::FromStr; +use bytes::{Bytes, BytesMut}; +use std::io::Write as _; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + static SERVE_METRICS_COUNT: Lazy = Lazy::new(|| { register_int_counter!( "libmetrics_metric_handler_requests_total", @@ -146,94 +151,89 @@ impl Drop for RequestCancelled { } } +/// An [`std::io::Write`] implementation on top of a channel sending [`bytes::Bytes`] chunks. +pub struct ChannelWriter { + buffer: BytesMut, + pub tx: mpsc::Sender>, + written: usize, +} + +impl ChannelWriter { + pub fn new(buf_len: usize, tx: mpsc::Sender>) -> Self { + assert_ne!(buf_len, 0); + ChannelWriter { + // split about half off the buffer from the start, because we flush depending on + // capacity. first flush will come sooner than without this, but now resizes will + // have better chance of picking up the "other" half. not guaranteed of course. + buffer: BytesMut::with_capacity(buf_len).split_off(buf_len / 2), + tx, + written: 0, + } + } + + pub fn flush0(&mut self) -> std::io::Result { + let n = self.buffer.len(); + if n == 0 { + return Ok(0); + } + + tracing::trace!(n, "flushing"); + let ready = self.buffer.split().freeze(); + + // not ideal to call from blocking code to block_on, but we are sure that this + // operation does not spawn_blocking other tasks + let res: Result<(), ()> = tokio::runtime::Handle::current().block_on(async { + self.tx.send(Ok(ready)).await.map_err(|_| ())?; + + // throttle sending to allow reuse of our buffer in `write`. + self.tx.reserve().await.map_err(|_| ())?; + + // now the response task has picked up the buffer and hopefully started + // sending it to the client. + Ok(()) + }); + if res.is_err() { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + self.written += n; + Ok(n) + } + + pub fn flushed_bytes(&self) -> usize { + self.written + } +} + +impl std::io::Write for ChannelWriter { + fn write(&mut self, mut buf: &[u8]) -> std::io::Result { + let remaining = self.buffer.capacity() - self.buffer.len(); + + let out_of_space = remaining < buf.len(); + + let original_len = buf.len(); + + if out_of_space { + let can_still_fit = buf.len() - remaining; + self.buffer.extend_from_slice(&buf[..can_still_fit]); + buf = &buf[can_still_fit..]; + self.flush0()?; + } + + // assume that this will often under normal operation just move the pointer back to the + // beginning of allocation, because previous split off parts are already sent and + // dropped. + self.buffer.extend_from_slice(buf); + Ok(original_len) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.flush0().map(|_| ()) + } +} + async fn prometheus_metrics_handler(_req: Request) -> Result, ApiError> { - use bytes::{Bytes, BytesMut}; - use std::io::Write as _; - use tokio::sync::mpsc; - use tokio_stream::wrappers::ReceiverStream; - SERVE_METRICS_COUNT.inc(); - /// An [`std::io::Write`] implementation on top of a channel sending [`bytes::Bytes`] chunks. - struct ChannelWriter { - buffer: BytesMut, - tx: mpsc::Sender>, - written: usize, - } - - impl ChannelWriter { - fn new(buf_len: usize, tx: mpsc::Sender>) -> Self { - assert_ne!(buf_len, 0); - ChannelWriter { - // split about half off the buffer from the start, because we flush depending on - // capacity. first flush will come sooner than without this, but now resizes will - // have better chance of picking up the "other" half. not guaranteed of course. - buffer: BytesMut::with_capacity(buf_len).split_off(buf_len / 2), - tx, - written: 0, - } - } - - fn flush0(&mut self) -> std::io::Result { - let n = self.buffer.len(); - if n == 0 { - return Ok(0); - } - - tracing::trace!(n, "flushing"); - let ready = self.buffer.split().freeze(); - - // not ideal to call from blocking code to block_on, but we are sure that this - // operation does not spawn_blocking other tasks - let res: Result<(), ()> = tokio::runtime::Handle::current().block_on(async { - self.tx.send(Ok(ready)).await.map_err(|_| ())?; - - // throttle sending to allow reuse of our buffer in `write`. - self.tx.reserve().await.map_err(|_| ())?; - - // now the response task has picked up the buffer and hopefully started - // sending it to the client. - Ok(()) - }); - if res.is_err() { - return Err(std::io::ErrorKind::BrokenPipe.into()); - } - self.written += n; - Ok(n) - } - - fn flushed_bytes(&self) -> usize { - self.written - } - } - - impl std::io::Write for ChannelWriter { - fn write(&mut self, mut buf: &[u8]) -> std::io::Result { - let remaining = self.buffer.capacity() - self.buffer.len(); - - let out_of_space = remaining < buf.len(); - - let original_len = buf.len(); - - if out_of_space { - let can_still_fit = buf.len() - remaining; - self.buffer.extend_from_slice(&buf[..can_still_fit]); - buf = &buf[can_still_fit..]; - self.flush0()?; - } - - // assume that this will often under normal operation just move the pointer back to the - // beginning of allocation, because previous split off parts are already sent and - // dropped. - self.buffer.extend_from_slice(buf); - Ok(original_len) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.flush0().map(|_| ()) - } - } - let started_at = std::time::Instant::now(); let (tx, rx) = mpsc::channel(1); @@ -389,7 +389,7 @@ fn parse_token(header_value: &str) -> Result<&str, ApiError> { } pub fn auth_middleware( - provide_auth: fn(&Request) -> Option<&JwtAuth>, + provide_auth: fn(&Request) -> Option<&SwappableJwtAuth>, ) -> Middleware { Middleware::pre(move |req| async move { if let Some(auth) = provide_auth(&req) { @@ -400,9 +400,11 @@ pub fn auth_middleware( })?; let token = parse_token(header_value)?; - let data = auth - .decode(token) - .map_err(|_| ApiError::Unauthorized("malformed jwt token".to_string()))?; + let data = auth.decode(token).map_err(|err| { + warn!("Authentication error: {err}"); + // Rely on From for ApiError impl + err + })?; req.set_context(data.claims); } None => { @@ -450,12 +452,11 @@ where pub fn check_permission_with( req: &Request, - check_permission: impl Fn(&Claims) -> Result<(), anyhow::Error>, + check_permission: impl Fn(&Claims) -> Result<(), AuthError>, ) -> Result<(), ApiError> { match req.context::() { - Some(claims) => { - Ok(check_permission(&claims).map_err(|err| ApiError::Forbidden(err.to_string()))?) - } + Some(claims) => Ok(check_permission(&claims) + .map_err(|_err| ApiError::Forbidden("JWT authentication error".to_string()))?), None => Ok(()), // claims is None because auth is disabled } } diff --git a/libs/utils/src/http/error.rs b/libs/utils/src/http/error.rs index 7233d3a662..ac68b04888 100644 --- a/libs/utils/src/http/error.rs +++ b/libs/utils/src/http/error.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::error::Error as StdError; use thiserror::Error; -use tracing::{error, info}; +use tracing::{error, info, warn}; #[derive(Debug, Error)] pub enum ApiError { @@ -118,6 +118,9 @@ pub fn api_error_handler(api_error: ApiError) -> Response { // Print a stack trace for Internal Server errors match api_error { + ApiError::Forbidden(_) | ApiError::Unauthorized(_) => { + warn!("Error processing HTTP request: {api_error:#}") + } ApiError::ResourceUnavailable(_) => info!("Error processing HTTP request: {api_error:#}"), ApiError::NotFound(_) => info!("Error processing HTTP request: {api_error:#}"), ApiError::InternalServerError(_) => error!("Error processing HTTP request: {api_error:?}"), diff --git a/libs/utils/src/id.rs b/libs/utils/src/id.rs index ec13c2f96f..57dcc27719 100644 --- a/libs/utils/src/id.rs +++ b/libs/utils/src/id.rs @@ -3,6 +3,7 @@ use std::{fmt, str::FromStr}; use anyhow::Context; use hex::FromHex; use rand::Rng; +use serde::de::Visitor; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -17,12 +18,74 @@ pub enum IdError { /// /// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look /// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// -/// Use `#[serde_as(as = "DisplayFromStr")]` to (de)serialize it as hex string instead: `ad50847381e248feaac9876cc71ae418`. -/// Check the `serde_with::serde_as` documentation for options for more complex types. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] struct Id([u8; 16]); +impl Serialize for Id { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + self.0.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for Id { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> Visitor<'de> for IdVisitor { + type Value = Id; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 16])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 16] = Deserialize::deserialize(s)?; + Ok(Id::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Id::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 16, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} + impl Id { pub fn get_from_buf(buf: &mut impl bytes::Buf) -> Id { let mut arr = [0u8; 16]; @@ -57,6 +120,8 @@ impl Id { chunk[0] = HEX[((b >> 4) & 0xf) as usize]; chunk[1] = HEX[(b & 0xf) as usize]; } + + // SAFETY: vec constructed out of `HEX`, it can only be ascii unsafe { String::from_utf8_unchecked(buf) } } } @@ -308,3 +373,112 @@ impl fmt::Display for NodeId { write!(f, "{}", self.0) } } + +#[cfg(test)] +mod tests { + use serde_assert::{Deserializer, Serializer, Token, Tokens}; + + use crate::bin_ser::BeSer; + + use super::*; + + #[test] + fn test_id_serde_non_human_readable() { + let original_id = Id([ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]); + let expected_tokens = Tokens(vec![ + Token::Tuple { len: 16 }, + Token::U8(173), + Token::U8(80), + Token::U8(132), + Token::U8(115), + Token::U8(129), + Token::U8(226), + Token::U8(72), + Token::U8(254), + Token::U8(170), + Token::U8(201), + Token::U8(135), + Token::U8(108), + Token::U8(199), + Token::U8(26), + Token::U8(228), + Token::U8(24), + Token::TupleEnd, + ]); + + let serializer = Serializer::builder().is_human_readable(false).build(); + let serialized_tokens = original_id.serialize(&serializer).unwrap(); + assert_eq!(serialized_tokens, expected_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(serialized_tokens) + .build(); + let deserialized_id = Id::deserialize(&mut deserializer).unwrap(); + assert_eq!(deserialized_id, original_id); + } + + #[test] + fn test_id_serde_human_readable() { + let original_id = Id([ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]); + let expected_tokens = Tokens(vec![Token::Str(String::from( + "ad50847381e248feaac9876cc71ae418", + ))]); + + let serializer = Serializer::builder().is_human_readable(true).build(); + let serialized_tokens = original_id.serialize(&serializer).unwrap(); + assert_eq!(serialized_tokens, expected_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(Tokens(vec![Token::Str(String::from( + "ad50847381e248feaac9876cc71ae418", + ))])) + .build(); + assert_eq!(Id::deserialize(&mut deserializer).unwrap(), original_id); + } + + macro_rules! roundtrip_type { + ($type:ty, $expected_bytes:expr) => {{ + let expected_bytes: [u8; 16] = $expected_bytes; + let original_id = <$type>::from(expected_bytes); + + let ser_bytes = original_id.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_id = <$type>::des(&ser_bytes).unwrap(); + assert_eq!(des_id, original_id); + }}; + } + + #[test] + fn test_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(Id, expected_bytes); + } + + #[test] + fn test_tenant_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(TenantId, expected_bytes); + } + + #[test] + fn test_timeline_id_bincode_serde() { + let expected_bytes = [ + 173, 80, 132, 115, 129, 226, 72, 254, 170, 201, 135, 108, 199, 26, 228, 24, + ]; + + roundtrip_type!(TimelineId, expected_bytes); + } +} diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 88cefd516d..bb6c848bf4 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -1,5 +1,6 @@ //! `utils` is intended to be a place to put code that is shared //! between other crates in this repository. +#![deny(clippy::undocumented_unsafe_blocks)] pub mod backoff; @@ -24,6 +25,10 @@ pub mod auth; // utility functions and helper traits for unified unique id generation/serialization etc. pub mod id; + +mod hex; +pub use hex::Hex; + // http endpoint utils pub mod http; @@ -73,6 +78,9 @@ pub mod completion; /// Reporting utilities pub mod error; +/// async timeout helper +pub mod timeout; + pub mod sync; /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages diff --git a/libs/utils/src/lsn.rs b/libs/utils/src/lsn.rs index 7d9baf7d49..262dcb8a8a 100644 --- a/libs/utils/src/lsn.rs +++ b/libs/utils/src/lsn.rs @@ -1,7 +1,7 @@ #![warn(missing_docs)] use camino::Utf8Path; -use serde::{Deserialize, Serialize}; +use serde::{de::Visitor, Deserialize, Serialize}; use std::fmt; use std::ops::{Add, AddAssign}; use std::str::FromStr; @@ -13,10 +13,114 @@ use crate::seqwait::MonotonicCounter; pub const XLOG_BLCKSZ: u32 = 8192; /// A Postgres LSN (Log Sequence Number), also known as an XLogRecPtr -#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash, Serialize, Deserialize)] -#[serde(transparent)] +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash)] pub struct Lsn(pub u64); +impl Serialize for Lsn { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + self.0.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for Lsn { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct LsnVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> Visitor<'de> for LsnVisitor { + type Value = Lsn; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str( + "value in form of hex string({upper_u32_hex}/{lower_u32_hex}) representing u64 integer", + ) + } else { + formatter.write_str("value in form of integer(u64)") + } + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + Ok(Lsn(v)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Lsn::from_str(v).map_err(|e| E::custom(e)) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(LsnVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_u64(LsnVisitor { + is_human_readable_deserializer: false, + }) + } + } +} + +/// Allows (de)serialization of an `Lsn` always as `u64`. +/// +/// ### Example +/// +/// ```rust +/// # use serde::{Serialize, Deserialize}; +/// use utils::lsn::Lsn; +/// +/// #[derive(PartialEq, Serialize, Deserialize, Debug)] +/// struct Foo { +/// #[serde(with = "utils::lsn::serde_as_u64")] +/// always_u64: Lsn, +/// } +/// +/// let orig = Foo { always_u64: Lsn(1234) }; +/// +/// let res = serde_json::to_string(&orig).unwrap(); +/// assert_eq!(res, r#"{"always_u64":1234}"#); +/// +/// let foo = serde_json::from_str::(&res).unwrap(); +/// assert_eq!(foo, orig); +/// ``` +/// +pub mod serde_as_u64 { + use super::Lsn; + + /// Serializes the Lsn as u64 disregarding the human readability of the format. + /// + /// Meant to be used via `#[serde(with = "...")]` or `#[serde(serialize_with = "...")]`. + pub fn serialize(lsn: &Lsn, serializer: S) -> Result { + use serde::Serialize; + lsn.0.serialize(serializer) + } + + /// Deserializes the Lsn as u64 disregarding the human readability of the format. + /// + /// Meant to be used via `#[serde(with = "...")]` or `#[serde(deserialize_with = "...")]`. + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + use serde::Deserialize; + u64::deserialize(deserializer).map(Lsn) + } +} + /// We tried to parse an LSN from a string, but failed #[derive(Debug, PartialEq, Eq, thiserror::Error)] #[error("LsnParseError")] @@ -264,8 +368,13 @@ impl MonotonicCounter for RecordLsn { #[cfg(test)] mod tests { + use crate::bin_ser::BeSer; + use super::*; + use serde::ser::Serialize; + use serde_assert::{Deserializer, Serializer, Token, Tokens}; + #[test] fn test_lsn_strings() { assert_eq!("12345678/AAAA5555".parse(), Ok(Lsn(0x12345678AAAA5555))); @@ -341,4 +450,95 @@ mod tests { assert_eq!(lsn.fetch_max(Lsn(6000)), Lsn(5678)); assert_eq!(lsn.fetch_max(Lsn(5000)), Lsn(6000)); } + + #[test] + fn test_lsn_serde() { + let original_lsn = Lsn(0x0123456789abcdef); + let expected_readable_tokens = Tokens(vec![Token::U64(0x0123456789abcdef)]); + let expected_non_readable_tokens = + Tokens(vec![Token::Str(String::from("1234567/89ABCDEF"))]); + + // Testing human_readable ser/de + let serializer = Serializer::builder().is_human_readable(false).build(); + let readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + assert_eq!(readable_ser_tokens, expected_readable_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(readable_ser_tokens) + .build(); + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + + // Testing NON human_readable ser/de + let serializer = Serializer::builder().is_human_readable(true).build(); + let non_readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + assert_eq!(non_readable_ser_tokens, expected_non_readable_tokens); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(non_readable_ser_tokens) + .build(); + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + + // Testing mismatching ser/de + let serializer = Serializer::builder().is_human_readable(false).build(); + let non_readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(true) + .tokens(non_readable_ser_tokens) + .build(); + Lsn::deserialize(&mut deserializer).unwrap_err(); + + let serializer = Serializer::builder().is_human_readable(true).build(); + let readable_ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(readable_ser_tokens) + .build(); + Lsn::deserialize(&mut deserializer).unwrap_err(); + } + + #[test] + fn test_lsn_ensure_roundtrip() { + let original_lsn = Lsn(0xaaaabbbb); + + let serializer = Serializer::builder().is_human_readable(false).build(); + let ser_tokens = original_lsn.serialize(&serializer).unwrap(); + + let mut deserializer = Deserializer::builder() + .is_human_readable(false) + .tokens(ser_tokens) + .build(); + + let des_lsn = Lsn::deserialize(&mut deserializer).unwrap(); + assert_eq!(des_lsn, original_lsn); + } + + #[test] + fn test_lsn_bincode_serde() { + let lsn = Lsn(0x0123456789abcdef); + let expected_bytes = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + + let ser_bytes = lsn.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_lsn = Lsn::des(&ser_bytes).unwrap(); + assert_eq!(des_lsn, lsn); + } + + #[test] + fn test_lsn_bincode_ensure_roundtrip() { + let original_lsn = Lsn(0x01_02_03_04_05_06_07_08); + let expected_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + + let ser_bytes = original_lsn.ser().unwrap(); + assert_eq!(ser_bytes, expected_bytes); + + let des_lsn = Lsn::des(&ser_bytes).unwrap(); + assert_eq!(des_lsn, original_lsn); + } } diff --git a/libs/utils/src/pageserver_feedback.rs b/libs/utils/src/pageserver_feedback.rs index a3b53201d3..c9fbdde928 100644 --- a/libs/utils/src/pageserver_feedback.rs +++ b/libs/utils/src/pageserver_feedback.rs @@ -3,7 +3,6 @@ use std::time::{Duration, SystemTime}; use bytes::{Buf, BufMut, Bytes, BytesMut}; use pq_proto::{read_cstr, PG_EPOCH}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use tracing::{trace, warn}; use crate::lsn::Lsn; @@ -15,21 +14,17 @@ use crate::lsn::Lsn; /// /// serde Serialize is used only for human readable dump to json (e.g. in /// safekeepers debug_dump). -#[serde_as] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct PageserverFeedback { /// Last known size of the timeline. Used to enforce timeline size limit. pub current_timeline_size: u64, /// LSN last received and ingested by the pageserver. Controls backpressure. - #[serde_as(as = "DisplayFromStr")] pub last_received_lsn: Lsn, /// LSN up to which data is persisted by the pageserver to its local disc. /// Controls backpressure. - #[serde_as(as = "DisplayFromStr")] pub disk_consistent_lsn: Lsn, /// LSN up to which data is persisted by the pageserver on s3; safekeepers /// consider WAL before it can be removed. - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, // Serialize with RFC3339 format. #[serde(with = "serde_systemtime")] diff --git a/libs/utils/src/seqwait.rs b/libs/utils/src/seqwait.rs index 5bc7ca91d6..effc9c67b5 100644 --- a/libs/utils/src/seqwait.rs +++ b/libs/utils/src/seqwait.rs @@ -125,6 +125,9 @@ where // Wake everyone with an error. let mut internal = self.internal.lock().unwrap(); + // Block any future waiters from starting + internal.shutdown = true; + // This will steal the entire waiters map. // When we drop it all waiters will be woken. mem::take(&mut internal.waiters) diff --git a/libs/utils/src/shutdown.rs b/libs/utils/src/shutdown.rs index 7eba905997..cb5a44d664 100644 --- a/libs/utils/src/shutdown.rs +++ b/libs/utils/src/shutdown.rs @@ -1,6 +1,7 @@ /// Immediately terminate the calling process without calling /// atexit callbacks, C runtime destructors etc. We mainly use /// this to protect coverage data from concurrent writes. -pub fn exit_now(code: u8) { +pub fn exit_now(code: u8) -> ! { + // SAFETY: exiting is safe, the ffi is not safe unsafe { nix::libc::_exit(code as _) }; } diff --git a/libs/utils/src/sync.rs b/libs/utils/src/sync.rs index 125eeca129..2ee8f35449 100644 --- a/libs/utils/src/sync.rs +++ b/libs/utils/src/sync.rs @@ -1 +1,3 @@ pub mod heavier_once_cell; + +pub mod gate; diff --git a/libs/utils/src/sync/gate.rs b/libs/utils/src/sync/gate.rs new file mode 100644 index 0000000000..9aad0af22d --- /dev/null +++ b/libs/utils/src/sync/gate.rs @@ -0,0 +1,158 @@ +use std::{sync::Arc, time::Duration}; + +/// Gates are a concurrency helper, primarily used for implementing safe shutdown. +/// +/// Users of a resource call `enter()` to acquire a GateGuard, and the owner of +/// the resource calls `close()` when they want to ensure that all holders of guards +/// have released them, and that no future guards will be issued. +pub struct Gate { + /// Each caller of enter() takes one unit from the semaphore. In close(), we + /// take all the units to ensure all GateGuards are destroyed. + sem: Arc, + + /// For observability only: a name that will be used to log warnings if a particular + /// gate is holding up shutdown + name: String, +} + +/// RAII guard for a [`Gate`]: as long as this exists, calls to [`Gate::close`] will +/// not complete. +#[derive(Debug)] +pub struct GateGuard(tokio::sync::OwnedSemaphorePermit); + +/// Observability helper: every `warn_period`, emit a log warning that we're still waiting on this gate +async fn warn_if_stuck( + fut: Fut, + name: &str, + warn_period: std::time::Duration, +) -> ::Output { + let started = std::time::Instant::now(); + + let mut fut = std::pin::pin!(fut); + + loop { + match tokio::time::timeout(warn_period, &mut fut).await { + Ok(ret) => return ret, + Err(_) => { + tracing::warn!( + gate = name, + elapsed_ms = started.elapsed().as_millis(), + "still waiting, taking longer than expected..." + ); + } + } + } +} + +#[derive(Debug)] +pub enum GateError { + GateClosed, +} + +impl Gate { + const MAX_UNITS: u32 = u32::MAX; + + pub fn new(name: String) -> Self { + Self { + sem: Arc::new(tokio::sync::Semaphore::new(Self::MAX_UNITS as usize)), + name, + } + } + + /// Acquire a guard that will prevent close() calls from completing. If close() + /// was already called, this will return an error which should be interpreted + /// as "shutting down". + /// + /// This function would typically be used from e.g. request handlers. While holding + /// the guard returned from this function, it is important to respect a CancellationToken + /// to avoid blocking close() indefinitely: typically types that contain a Gate will + /// also contain a CancellationToken. + pub fn enter(&self) -> Result { + self.sem + .clone() + .try_acquire_owned() + .map(GateGuard) + .map_err(|_| GateError::GateClosed) + } + + /// Types with a shutdown() method and a gate should call this method at the + /// end of shutdown, to ensure that all GateGuard holders are done. + /// + /// This will wait for all guards to be destroyed. For this to complete promptly, it is + /// important that the holders of such guards are respecting a CancellationToken which has + /// been cancelled before entering this function. + pub async fn close(&self) { + warn_if_stuck(self.do_close(), &self.name, Duration::from_millis(1000)).await + } + + /// Check if [`Self::close()`] has finished waiting for all [`Self::enter()`] users to finish. This + /// is usually analoguous for "Did shutdown finish?" for types that include a Gate, whereas checking + /// the CancellationToken on such types is analogous to "Did shutdown start?" + pub fn close_complete(&self) -> bool { + self.sem.is_closed() + } + + async fn do_close(&self) { + tracing::debug!(gate = self.name, "Closing Gate..."); + match self.sem.acquire_many(Self::MAX_UNITS).await { + Ok(_units) => { + // While holding all units, close the semaphore. All subsequent calls to enter() will fail. + self.sem.close(); + } + Err(_) => { + // Semaphore closed: we are the only function that can do this, so it indicates a double-call. + // This is legal. Timeline::shutdown for example is not protected from being called more than + // once. + tracing::debug!(gate = self.name, "Double close") + } + } + tracing::debug!(gate = self.name, "Closed Gate.") + } +} + +#[cfg(test)] +mod tests { + use futures::FutureExt; + + use super::*; + + #[tokio::test] + async fn test_idle_gate() { + // Having taken no gates, we should not be blocked in close + let gate = Gate::new("test".to_string()); + gate.close().await; + + // If a guard is dropped before entering, close should not be blocked + let gate = Gate::new("test".to_string()); + let guard = gate.enter().unwrap(); + drop(guard); + gate.close().await; + + // Entering a closed guard fails + gate.enter().expect_err("enter should fail after close"); + } + + #[tokio::test] + async fn test_busy_gate() { + let gate = Gate::new("test".to_string()); + + let guard = gate.enter().unwrap(); + + let mut close_fut = std::pin::pin!(gate.close()); + + // Close should be blocked + assert!(close_fut.as_mut().now_or_never().is_none()); + + // Attempting to enter() should fail, even though close isn't done yet. + gate.enter() + .expect_err("enter should fail after entering close"); + + drop(guard); + + // Guard is gone, close should finish + assert!(close_fut.as_mut().now_or_never().is_some()); + + // Attempting to enter() is still forbidden + gate.enter().expect_err("enter should fail finishing close"); + } +} diff --git a/libs/utils/src/timeout.rs b/libs/utils/src/timeout.rs new file mode 100644 index 0000000000..11fa417242 --- /dev/null +++ b/libs/utils/src/timeout.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use tokio_util::sync::CancellationToken; + +pub enum TimeoutCancellableError { + Timeout, + Cancelled, +} + +/// Wrap [`tokio::time::timeout`] with a CancellationToken. +/// +/// This wrapper is appropriate for any long running operation in a task +/// that ought to respect a CancellationToken (which means most tasks). +/// +/// The only time you should use a bare tokio::timeout is when the future `F` +/// itself respects a CancellationToken: otherwise, always use this wrapper +/// with your CancellationToken to ensure that your task does not hold up +/// graceful shutdown. +pub async fn timeout_cancellable( + duration: Duration, + cancel: &CancellationToken, + future: F, +) -> Result +where + F: std::future::Future, +{ + tokio::select!( + r = tokio::time::timeout(duration, future) => { + r.map_err(|_| TimeoutCancellableError::Timeout) + + }, + _ = cancel.cancelled() => { + Err(TimeoutCancellableError::Cancelled) + + } + ) +} diff --git a/libs/vm_monitor/Cargo.toml b/libs/vm_monitor/Cargo.toml index 26b976830a..46e9f880a1 100644 --- a/libs/vm_monitor/Cargo.toml +++ b/libs/vm_monitor/Cargo.toml @@ -19,13 +19,12 @@ inotify.workspace = true serde.workspace = true serde_json.workspace = true sysinfo.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread"] } tokio-postgres.workspace = true tokio-stream.workspace = true tokio-util.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -workspace_hack = { version = "0.1", path = "../../workspace_hack" } [target.'cfg(target_os = "linux")'.dependencies] cgroups-rs = "0.3.3" diff --git a/libs/vm_monitor/src/lib.rs b/libs/vm_monitor/src/lib.rs index a844f78bd6..89ca91fdd7 100644 --- a/libs/vm_monitor/src/lib.rs +++ b/libs/vm_monitor/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] #![cfg(target_os = "linux")] use anyhow::Context; diff --git a/libs/walproposer/src/api_bindings.rs b/libs/walproposer/src/api_bindings.rs index b0ac2a879e..7f1bbc3b80 100644 --- a/libs/walproposer/src/api_bindings.rs +++ b/libs/walproposer/src/api_bindings.rs @@ -188,6 +188,7 @@ extern "C" fn recovery_download( } } +#[allow(clippy::unnecessary_cast)] extern "C" fn wal_read( sk: *mut Safekeeper, buf: *mut ::std::os::raw::c_char, @@ -421,6 +422,7 @@ impl std::fmt::Display for Level { } /// Take ownership of `Vec` from StringInfoData. +#[allow(clippy::unnecessary_cast)] pub(crate) fn take_vec_u8(pg: &mut StringInfoData) -> Option> { if pg.data.is_null() { return None; diff --git a/libs/walproposer/src/walproposer.rs b/libs/walproposer/src/walproposer.rs index 4be15344c5..0661d3a969 100644 --- a/libs/walproposer/src/walproposer.rs +++ b/libs/walproposer/src/walproposer.rs @@ -186,7 +186,7 @@ impl Wrapper { .unwrap() .into_bytes_with_nul(); assert!(safekeepers_list_vec.len() == safekeepers_list_vec.capacity()); - let safekeepers_list = safekeepers_list_vec.as_mut_ptr() as *mut i8; + let safekeepers_list = safekeepers_list_vec.as_mut_ptr() as *mut std::ffi::c_char; let callback_data = Box::into_raw(Box::new(api)) as *mut ::std::os::raw::c_void; diff --git a/pageserver/src/auth.rs b/pageserver/src/auth.rs index 268117cae2..2cb661863d 100644 --- a/pageserver/src/auth.rs +++ b/pageserver/src/auth.rs @@ -1,22 +1,21 @@ -use anyhow::{bail, Result}; -use utils::auth::{Claims, Scope}; +use utils::auth::{AuthError, Claims, Scope}; use utils::id::TenantId; -pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<()> { +pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<(), AuthError> { match (&claims.scope, tenant_id) { - (Scope::Tenant, None) => { - bail!("Attempt to access management api with tenant scope. Permission denied") - } + (Scope::Tenant, None) => Err(AuthError( + "Attempt to access management api with tenant scope. Permission denied".into(), + )), (Scope::Tenant, Some(tenant_id)) => { if claims.tenant_id.unwrap() != tenant_id { - bail!("Tenant id mismatch. Permission denied") + return Err(AuthError("Tenant id mismatch. Permission denied".into())); } Ok(()) } (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope - (Scope::SafekeeperData, _) => { - bail!("SafekeeperData scope makes no sense for Pageserver") - } + (Scope::SafekeeperData, _) => Err(AuthError( + "SafekeeperData scope makes no sense for Pageserver".into(), + )), } } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index df2db25ec6..5b0c140d00 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -34,8 +34,11 @@ use postgres_backend::AuthType; use utils::logging::TracingErrorLayerEnablement; use utils::signals::ShutdownSignals; use utils::{ - auth::JwtAuth, logging, project_build_tag, project_git_version, sentry_init::init_sentry, - signals::Signal, tcp_listener, + auth::{JwtAuth, SwappableJwtAuth}, + logging, project_build_tag, project_git_version, + sentry_init::init_sentry, + signals::Signal, + tcp_listener, }; project_git_version!(GIT_VERSION); @@ -321,13 +324,12 @@ fn start_pageserver( let http_auth; let pg_auth; if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT { - // unwrap is ok because check is performed when creating config, so path is set and file exists + // unwrap is ok because check is performed when creating config, so path is set and exists let key_path = conf.auth_validation_public_key_path.as_ref().unwrap(); - info!( - "Loading public key for verifying JWT tokens from {:#?}", - key_path - ); - let auth: Arc = Arc::new(JwtAuth::from_key_path(key_path)?); + info!("Loading public key(s) for verifying JWT tokens from {key_path:?}"); + + let jwt_auth = JwtAuth::from_key_path(key_path)?; + let auth: Arc = Arc::new(SwappableJwtAuth::new(jwt_auth)); http_auth = match &conf.http_auth_type { AuthType::Trust => None, @@ -410,7 +412,7 @@ fn start_pageserver( // Scan the local 'tenants/' directory and start loading the tenants let deletion_queue_client = deletion_queue.new_client(); - BACKGROUND_RUNTIME.block_on(mgr::init_tenant_mgr( + let tenant_manager = BACKGROUND_RUNTIME.block_on(mgr::init_tenant_mgr( conf, TenantSharedResources { broker_client: broker_client.clone(), @@ -420,6 +422,7 @@ fn start_pageserver( order, shutdown_pageserver.clone(), ))?; + let tenant_manager = Arc::new(tenant_manager); BACKGROUND_RUNTIME.spawn({ let init_done_rx = init_done_rx; @@ -548,6 +551,7 @@ fn start_pageserver( let router_state = Arc::new( http::routes::State::new( conf, + tenant_manager, http_auth.clone(), remote_storage.clone(), broker_client.clone(), diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index a58c08e29b..87d9cc522e 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -161,7 +161,7 @@ pub struct PageServerConf { pub http_auth_type: AuthType, /// authentication method for libpq connections from compute pub pg_auth_type: AuthType, - /// Path to a file containing public key for verifying JWT tokens. + /// Path to a file or directory containing public key(s) for verifying JWT tokens. /// Used for both mgmt and compute auth, if enabled. pub auth_validation_public_key_path: Option, @@ -1314,12 +1314,6 @@ broker_endpoint = '{broker_endpoint}' assert_eq!( parsed_remote_storage_config, RemoteStorageConfig { - max_concurrent_syncs: NonZeroUsize::new( - remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS - ) - .unwrap(), - max_sync_errors: NonZeroU32::new(remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS) - .unwrap(), storage: RemoteStorageKind::LocalFs(local_storage_path.clone()), }, "Remote storage config should correctly parse the local FS config and fill other storage defaults" @@ -1380,8 +1374,6 @@ broker_endpoint = '{broker_endpoint}' assert_eq!( parsed_remote_storage_config, RemoteStorageConfig { - max_concurrent_syncs, - max_sync_errors, storage: RemoteStorageKind::AwsS3(S3Config { bucket_name: bucket_name.clone(), bucket_region: bucket_region.clone(), diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 061045eb76..9e8377c1f1 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -266,7 +266,7 @@ async fn calculate_synthetic_size_worker( continue; } - if let Ok(tenant) = mgr::get_tenant(tenant_id, true).await { + if let Ok(tenant) = mgr::get_tenant(tenant_id, true) { // TODO should we use concurrent_background_tasks_rate_limit() here, like the other background tasks? // We can put in some prioritization for consumption metrics. // Same for the loop that fetches computed metrics. diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index 652dd98683..2989e15e8e 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -3,7 +3,6 @@ use anyhow::Context; use chrono::{DateTime, Utc}; use consumption_metrics::EventType; use futures::stream::StreamExt; -use serde_with::serde_as; use std::{sync::Arc, time::SystemTime}; use utils::{ id::{TenantId, TimelineId}, @@ -42,13 +41,10 @@ pub(super) enum Name { /// /// This is a denormalization done at the MetricsKey const methods; these should not be constructed /// elsewhere. -#[serde_with::serde_as] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub(crate) struct MetricsKey { - #[serde_as(as = "serde_with::DisplayFromStr")] pub(super) tenant_id: TenantId, - #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] pub(super) timeline_id: Option, @@ -206,7 +202,6 @@ pub(super) async fn collect_all_metrics( None } else { crate::tenant::mgr::get_tenant(id, true) - .await .ok() .map(|tenant| (id, tenant)) } diff --git a/pageserver/src/consumption_metrics/upload.rs b/pageserver/src/consumption_metrics/upload.rs index d69d43a2a8..322ed95cc8 100644 --- a/pageserver/src/consumption_metrics/upload.rs +++ b/pageserver/src/consumption_metrics/upload.rs @@ -1,5 +1,4 @@ use consumption_metrics::{Event, EventChunk, IdempotencyKey, CHUNK_SIZE}; -use serde_with::serde_as; use tokio_util::sync::CancellationToken; use tracing::Instrument; @@ -7,12 +6,9 @@ use super::{metrics::Name, Cache, MetricsKey, RawMetric}; use utils::id::{TenantId, TimelineId}; /// How the metrics from pageserver are identified. -#[serde_with::serde_as] #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq)] struct Ids { - #[serde_as(as = "serde_with::DisplayFromStr")] pub(super) tenant_id: TenantId, - #[serde_as(as = "Option")] #[serde(skip_serializing_if = "Option::is_none")] pub(super) timeline_id: Option, } diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 7b2db929fa..6a732d1029 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -18,7 +18,6 @@ use hex::FromHex; use remote_storage::{GenericRemoteStorage, RemotePath}; use serde::Deserialize; use serde::Serialize; -use serde_with::serde_as; use thiserror::Error; use tokio; use tokio_util::sync::CancellationToken; @@ -215,7 +214,6 @@ where /// during recovery as startup. const TEMP_SUFFIX: &str = "tmp"; -#[serde_as] #[derive(Debug, Serialize, Deserialize)] struct DeletionList { /// Serialization version, for future use @@ -244,7 +242,6 @@ struct DeletionList { validated: bool, } -#[serde_as] #[derive(Debug, Serialize, Deserialize)] struct DeletionHeader { /// Serialization version, for future use @@ -896,14 +893,6 @@ mod test { std::fs::create_dir_all(remote_fs_dir)?; let remote_fs_dir = harness.conf.workdir.join("remote_fs").canonicalize_utf8()?; let storage_config = RemoteStorageConfig { - max_concurrent_syncs: std::num::NonZeroUsize::new( - remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS, - ) - .unwrap(), - max_sync_errors: std::num::NonZeroU32::new( - remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS, - ) - .unwrap(), storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), }; let storage = GenericRemoteStorage::from_config(&storage_config).unwrap(); diff --git a/pageserver/src/deletion_queue/deleter.rs b/pageserver/src/deletion_queue/deleter.rs index 81cb016d49..57421b1547 100644 --- a/pageserver/src/deletion_queue/deleter.rs +++ b/pageserver/src/deletion_queue/deleter.rs @@ -55,21 +55,24 @@ impl Deleter { /// Wrap the remote `delete_objects` with a failpoint async fn remote_delete(&self) -> Result<(), anyhow::Error> { - fail::fail_point!("deletion-queue-before-execute", |_| { - info!("Skipping execution, failpoint set"); - metrics::DELETION_QUEUE - .remote_errors - .with_label_values(&["failpoint"]) - .inc(); - Err(anyhow::anyhow!("failpoint hit")) - }); - // A backoff::retry is used here for two reasons: // - To provide a backoff rather than busy-polling the API on errors // - To absorb transient 429/503 conditions without hitting our error // logging path for issues deleting objects. backoff::retry( - || async { self.remote_storage.delete_objects(&self.accumulator).await }, + || async { + fail::fail_point!("deletion-queue-before-execute", |_| { + info!("Skipping execution, failpoint set"); + + metrics::DELETION_QUEUE + .remote_errors + .with_label_values(&["failpoint"]) + .inc(); + Err(anyhow::anyhow!("failpoint: deletion-queue-before-execute")) + }); + + self.remote_storage.delete_objects(&self.accumulator).await + }, |_| false, 3, 10, diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index 413c941bc4..642cafad28 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -403,7 +403,7 @@ pub async fn disk_usage_eviction_task_iteration_impl( return (evicted_bytes, evictions_failed); }; - let results = timeline.evict_layers(&batch, &cancel).await; + let results = timeline.evict_layers(&batch).await; match results { Ok(results) => { @@ -545,7 +545,7 @@ async fn collect_eviction_candidates( if cancel.is_cancelled() { return Ok(EvictionCandidates::Cancelled); } - let tenant = match tenant::mgr::get_tenant(*tenant_id, true).await { + let tenant = match tenant::mgr::get_tenant(*tenant_id, true) { Ok(tenant) => tenant, Err(e) => { // this can happen if tenant has lifecycle transition after we fetched it @@ -554,6 +554,11 @@ async fn collect_eviction_candidates( } }; + if tenant.cancel.is_cancelled() { + info!(%tenant_id, "Skipping tenant for eviction, it is shutting down"); + continue; + } + // collect layers from all timelines in this tenant // // If one of the timelines becomes `!is_active()` during the iteration, diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index d31e4a1554..4d455243f0 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -52,6 +52,31 @@ paths: schema: type: object + /v1/reload_auth_validation_keys: + post: + description: Reloads the JWT public keys from their pre-configured location on disk. + responses: + "200": + description: The reload completed successfully. + "401": + description: Unauthorized Error + content: + application/json: + schema: + $ref: "#/components/schemas/UnauthorizedError" + "403": + description: Forbidden Error + content: + application/json: + schema: + $ref: "#/components/schemas/ForbiddenError" + "500": + description: Generic operation error (also hits if no keys were found) + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /v1/tenant/{tenant_id}: parameters: - name: tenant_id @@ -327,7 +352,8 @@ paths: in: query required: true schema: - type: integer + type: string + format: hex description: A LSN to get the timestamp responses: "200": diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 2c46d733d6..aa2b017471 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -16,11 +16,12 @@ use pageserver_api::models::{ DownloadRemoteLayersTaskSpawnRequest, LocationConfigMode, TenantAttachRequest, TenantLoadRequest, TenantLocationConfigRequest, }; +use pageserver_api::shard::TenantShardId; use remote_storage::GenericRemoteStorage; -use serde_with::{serde_as, DisplayFromStr}; use tenant_size_model::{SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; use tracing::*; +use utils::auth::JwtAuth; use utils::http::endpoint::request_span; use utils::http::json::json_request_or_empty_body; use utils::http::request::{get_request_param, must_get_query_param, parse_query_param}; @@ -36,7 +37,8 @@ use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; use crate::tenant::config::{LocationConf, TenantConfOpt}; use crate::tenant::mgr::{ - GetTenantError, SetNewTenantConfigError, TenantMapInsertError, TenantStateError, + GetTenantError, SetNewTenantConfigError, TenantManager, TenantMapError, TenantMapInsertError, + TenantSlotError, TenantSlotUpsertError, TenantStateError, }; use crate::tenant::size::ModelInputs; use crate::tenant::storage_layer::LayerAccessStatsReset; @@ -45,7 +47,7 @@ use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError, TenantSha use crate::{config::PageServerConf, tenant::mgr}; use crate::{disk_usage_eviction_task, tenant}; use utils::{ - auth::JwtAuth, + auth::SwappableJwtAuth, generation::Generation, http::{ endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with}, @@ -63,7 +65,8 @@ use super::models::ConfigureFailpointsRequest; pub struct State { conf: &'static PageServerConf, - auth: Option>, + tenant_manager: Arc, + auth: Option>, allowlist_routes: Vec, remote_storage: Option, broker_client: storage_broker::BrokerClientChannel, @@ -74,7 +77,8 @@ pub struct State { impl State { pub fn new( conf: &'static PageServerConf, - auth: Option>, + tenant_manager: Arc, + auth: Option>, remote_storage: Option, broker_client: storage_broker::BrokerClientChannel, disk_usage_eviction_state: Arc, @@ -86,6 +90,7 @@ impl State { .collect::>(); Ok(Self { conf, + tenant_manager, auth, allowlist_routes, remote_storage, @@ -147,28 +152,59 @@ impl From for ApiError { impl From for ApiError { fn from(tmie: TenantMapInsertError) -> ApiError { match tmie { - TenantMapInsertError::StillInitializing | TenantMapInsertError::ShuttingDown => { - ApiError::ResourceUnavailable(format!("{tmie}").into()) - } - TenantMapInsertError::TenantAlreadyExists(id, state) => { - ApiError::Conflict(format!("tenant {id} already exists, state: {state:?}")) - } - TenantMapInsertError::TenantExistsSecondary(id) => { - ApiError::Conflict(format!("tenant {id} already exists as secondary")) - } + TenantMapInsertError::SlotError(e) => e.into(), + TenantMapInsertError::SlotUpsertError(e) => e.into(), TenantMapInsertError::Other(e) => ApiError::InternalServerError(e), } } } +impl From for ApiError { + fn from(e: TenantSlotError) -> ApiError { + use TenantSlotError::*; + match e { + NotFound(tenant_id) => { + ApiError::NotFound(anyhow::anyhow!("NotFound: tenant {tenant_id}").into()) + } + e @ (AlreadyExists(_, _) | Conflict(_)) => ApiError::Conflict(format!("{e}")), + InProgress => { + ApiError::ResourceUnavailable("Tenant is being modified concurrently".into()) + } + MapState(e) => e.into(), + } + } +} + +impl From for ApiError { + fn from(e: TenantSlotUpsertError) -> ApiError { + use TenantSlotUpsertError::*; + match e { + InternalError(e) => ApiError::InternalServerError(anyhow::anyhow!("{e}")), + MapState(e) => e.into(), + } + } +} + +impl From for ApiError { + fn from(e: TenantMapError) -> ApiError { + use TenantMapError::*; + match e { + StillInitializing | ShuttingDown => { + ApiError::ResourceUnavailable(format!("{e}").into()) + } + } + } +} + impl From for ApiError { fn from(tse: TenantStateError) -> ApiError { match tse { - TenantStateError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid).into()), TenantStateError::IsStopping(_) => { ApiError::ResourceUnavailable("Tenant is stopping".into()) } - _ => ApiError::InternalServerError(anyhow::Error::new(tse)), + TenantStateError::SlotError(e) => e.into(), + TenantStateError::SlotUpsertError(e) => e.into(), + TenantStateError::Other(e) => ApiError::InternalServerError(anyhow!(e)), } } } @@ -189,6 +225,7 @@ impl From for ApiError { // (We can produce this variant only in `mgr::get_tenant(..., active=true)` calls). ApiError::ResourceUnavailable("Tenant not yet active".into()) } + GetTenantError::MapState(e) => ApiError::ResourceUnavailable(format!("{e}").into()), } } } @@ -243,6 +280,9 @@ impl From for ApiError { Get(g) => ApiError::from(g), e @ AlreadyInProgress => ApiError::Conflict(e.to_string()), Timeline(t) => ApiError::from(t), + NotAttached => ApiError::NotFound(anyhow::anyhow!("Tenant is not attached").into()), + SlotError(e) => e.into(), + SlotUpsertError(e) => e.into(), Other(o) => ApiError::InternalServerError(o), e @ InvalidState(_) => ApiError::PreconditionFailed(e.to_string().into_boxed_str()), } @@ -264,11 +304,7 @@ async fn build_timeline_info( // we're executing this function, we will outlive the timeline on-disk state. info.current_logical_size_non_incremental = Some( timeline - .get_current_logical_size_non_incremental( - info.last_record_lsn, - CancellationToken::new(), - ctx, - ) + .get_current_logical_size_non_incremental(info.last_record_lsn, ctx) .await?, ); } @@ -354,13 +390,39 @@ async fn status_handler( json_response(StatusCode::OK, StatusResponse { id: config.id }) } +async fn reload_auth_validation_keys_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + check_permission(&request, None)?; + let config = get_config(&request); + let state = get_state(&request); + let Some(shared_auth) = &state.auth else { + return json_response(StatusCode::BAD_REQUEST, ()); + }; + // unwrap is ok because check is performed when creating config, so path is set and exists + let key_path = config.auth_validation_public_key_path.as_ref().unwrap(); + info!("Reloading public key(s) for verifying JWT tokens from {key_path:?}"); + + match JwtAuth::from_key_path(key_path) { + Ok(new_auth) => { + shared_auth.swap(new_auth); + json_response(StatusCode::OK, ()) + } + Err(e) => { + warn!("Error reloading public keys from {key_path:?}: {e:}"); + json_response(StatusCode::INTERNAL_SERVER_ERROR, ()) + } + } +} + async fn timeline_create_handler( mut request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; let request_data: TimelineCreateRequest = json_request(&mut request).await?; - check_permission(&request, Some(tenant_id))?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let new_timeline_id = request_data.new_timeline_id; @@ -369,7 +431,7 @@ async fn timeline_create_handler( let state = get_state(&request); async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = state.tenant_manager.get_attached_tenant_shard(tenant_shard_id, true)?; match tenant.create_timeline( new_timeline_id, request_data.ancestor_timeline_id.map(TimelineId::from), @@ -397,10 +459,16 @@ async fn timeline_create_handler( Err(e @ tenant::CreateTimelineError::AncestorNotActive) => { json_response(StatusCode::SERVICE_UNAVAILABLE, HttpErrorBody::from_msg(e.to_string())) } + Err(tenant::CreateTimelineError::ShuttingDown) => { + json_response(StatusCode::SERVICE_UNAVAILABLE,HttpErrorBody::from_msg("tenant shutting down".to_string())) + } Err(tenant::CreateTimelineError::Other(err)) => Err(ApiError::InternalServerError(err)), } } - .instrument(info_span!("timeline_create", %tenant_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version)) + .instrument(info_span!("timeline_create", + tenant_id = %tenant_shard_id.tenant_id, + shard = %tenant_shard_id.shard_slug(), + timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version)) .await } @@ -416,7 +484,7 @@ async fn timeline_list_handler( let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let response_data = async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; let timelines = tenant.list_timelines(); let mut response_data = Vec::with_capacity(timelines.len()); @@ -455,7 +523,7 @@ async fn timeline_detail_handler( let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let timeline_info = async { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; let timeline = tenant .get_timeline(timeline_id, false) @@ -499,10 +567,8 @@ async fn get_lsn_by_timestamp_handler( let result = timeline.find_lsn_for_timestamp(timestamp_pg, &ctx).await?; if version.unwrap_or(0) > 1 { - #[serde_as] #[derive(serde::Serialize)] struct Result { - #[serde_as(as = "DisplayFromStr")] lsn: Lsn, kind: &'static str, } @@ -598,14 +664,15 @@ async fn timeline_delete_handler( request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; - check_permission(&request, Some(tenant_id))?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); + let state = get_state(&request); - mgr::delete_timeline(tenant_id, timeline_id, &ctx) - .instrument(info_span!("timeline_delete", %tenant_id, %timeline_id)) + state.tenant_manager.delete_timeline(tenant_shard_id, timeline_id, &ctx) + .instrument(info_span!("timeline_delete", tenant_id=%tenant_shard_id.tenant_id, shard=%tenant_shard_id.shard_slug(), %timeline_id)) .await?; json_response(StatusCode::ACCEPTED, ()) @@ -619,11 +686,14 @@ async fn tenant_detach_handler( check_permission(&request, Some(tenant_id))?; let detach_ignored: Option = parse_query_param(&request, "detach_ignored")?; + // This is a legacy API (`/location_conf` is the replacement). It only supports unsharded tenants + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + let state = get_state(&request); let conf = state.conf; mgr::detach_tenant( conf, - tenant_id, + tenant_shard_id, detach_ignored.unwrap_or(false), &state.deletion_queue_client, ) @@ -713,7 +783,7 @@ async fn tenant_status( check_permission(&request, Some(tenant_id))?; let tenant_info = async { - let tenant = mgr::get_tenant(tenant_id, false).await?; + let tenant = mgr::get_tenant(tenant_id, false)?; // Calculate total physical size of all timelines let mut current_physical_size = 0; @@ -740,13 +810,16 @@ async fn tenant_delete_handler( _cancel: CancellationToken, ) -> Result, ApiError> { // TODO openapi spec - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - check_permission(&request, Some(tenant_id))?; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let state = get_state(&request); - mgr::delete_tenant(state.conf, state.remote_storage.clone(), tenant_id) - .instrument(info_span!("tenant_delete_handler", %tenant_id)) + mgr::delete_tenant(state.conf, state.remote_storage.clone(), tenant_shard_id) + .instrument(info_span!("tenant_delete_handler", + tenant_id = %tenant_shard_id.tenant_id, + shard = tenant_shard_id.shard_slug() + )) .await?; json_response(StatusCode::ACCEPTED, ()) @@ -776,7 +849,7 @@ async fn tenant_size_handler( let headers = request.headers(); let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; // this can be long operation let inputs = tenant @@ -811,10 +884,8 @@ async fn tenant_size_handler( } /// The type resides in the pageserver not to expose `ModelInputs`. - #[serde_with::serde_as] #[derive(serde::Serialize)] struct TenantHistorySize { - #[serde_as(as = "serde_with::DisplayFromStr")] id: TenantId, /// Size is a mixture of WAL and logical size, so the unit is bytes. /// @@ -1035,7 +1106,7 @@ async fn get_tenant_config_handler( let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; - let tenant = mgr::get_tenant(tenant_id, false).await?; + let tenant = mgr::get_tenant(tenant_id, false)?; let response = HashMap::from([ ( @@ -1078,9 +1149,10 @@ async fn put_tenant_location_config_handler( mut request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; + let request_data: TenantLocationConfigRequest = json_request(&mut request).await?; - let tenant_id = request_data.tenant_id; - check_permission(&request, Some(tenant_id))?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); let state = get_state(&request); @@ -1089,12 +1161,16 @@ async fn put_tenant_location_config_handler( // The `Detached` state is special, it doesn't upsert a tenant, it removes // its local disk content and drops it from memory. if let LocationConfigMode::Detached = request_data.config.mode { - if let Err(e) = mgr::detach_tenant(conf, tenant_id, true, &state.deletion_queue_client) - .instrument(info_span!("tenant_detach", %tenant_id)) - .await + if let Err(e) = + mgr::detach_tenant(conf, tenant_shard_id, true, &state.deletion_queue_client) + .instrument(info_span!("tenant_detach", + tenant_id = %tenant_shard_id.tenant_id, + shard = tenant_shard_id.shard_slug() + )) + .await { match e { - TenantStateError::NotFound(_) => { + TenantStateError::SlotError(TenantSlotError::NotFound(_)) => { // This API is idempotent: a NotFound on a detach is fine. } _ => return Err(e.into()), @@ -1106,20 +1182,14 @@ async fn put_tenant_location_config_handler( let location_conf = LocationConf::try_from(&request_data.config).map_err(ApiError::BadRequest)?; - mgr::upsert_location( - state.conf, - tenant_id, - location_conf, - state.broker_client.clone(), - state.remote_storage.clone(), - state.deletion_queue_client.clone(), - &ctx, - ) - .await - // TODO: badrequest assumes the caller was asking for something unreasonable, but in - // principle we might have hit something like concurrent API calls to the same tenant, - // which is not a 400 but a 409. - .map_err(ApiError::BadRequest)?; + state + .tenant_manager + .upsert_location(tenant_shard_id, location_conf, &ctx) + .await + // TODO: badrequest assumes the caller was asking for something unreasonable, but in + // principle we might have hit something like concurrent API calls to the same tenant, + // which is not a 400 but a 409. + .map_err(ApiError::BadRequest)?; json_response(StatusCode::OK, ()) } @@ -1132,7 +1202,6 @@ async fn handle_tenant_break( let tenant_id: TenantId = parse_request_param(&r, "tenant_id")?; let tenant = crate::tenant::mgr::get_tenant(tenant_id, true) - .await .map_err(|_| ApiError::Conflict(String::from("no active tenant found")))?; tenant.set_broken("broken from test".to_owned()).await; @@ -1425,7 +1494,7 @@ async fn timeline_collect_keyspace( let keys = timeline .collect_keyspace(at_lsn, &ctx) .await - .map_err(ApiError::InternalServerError)?; + .map_err(|e| ApiError::InternalServerError(e.into()))?; json_response(StatusCode::OK, Partitioning { keys, at_lsn }) } @@ -1437,7 +1506,7 @@ async fn active_timeline_of_active_tenant( tenant_id: TenantId, timeline_id: TimelineId, ) -> Result, ApiError> { - let tenant = mgr::get_tenant(tenant_id, true).await?; + let tenant = mgr::get_tenant(tenant_id, true)?; tenant .get_timeline(timeline_id, true) .map_err(|e| ApiError::NotFound(e.into())) @@ -1614,6 +1683,8 @@ where ); match handle.await { + // TODO: never actually return Err from here, always Ok(...) so that we can log + // spanned errors. Call api_error_handler instead and return appropriate Body. Ok(result) => result, Err(e) => { // The handler task panicked. We have a global panic handler that logs the @@ -1662,7 +1733,7 @@ where pub fn make_router( state: Arc, launch_ts: &'static LaunchTimestamp, - auth: Option>, + auth: Option>, ) -> anyhow::Result> { let spec = include_bytes!("openapi_spec.yml"); let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc"); @@ -1691,10 +1762,13 @@ pub fn make_router( .put("/v1/failpoints", |r| { testing_api_handler("manage failpoints", r, failpoints_handler) }) + .post("/v1/reload_auth_validation_keys", |r| { + api_handler(r, reload_auth_validation_keys_handler) + }) .get("/v1/tenant", |r| api_handler(r, tenant_list_handler)) .post("/v1/tenant", |r| api_handler(r, tenant_create_handler)) .get("/v1/tenant/:tenant_id", |r| api_handler(r, tenant_status)) - .delete("/v1/tenant/:tenant_id", |r| { + .delete("/v1/tenant/:tenant_shard_id", |r| { api_handler(r, tenant_delete_handler) }) .get("/v1/tenant/:tenant_id/synthetic_size", |r| { @@ -1706,13 +1780,13 @@ pub fn make_router( .get("/v1/tenant/:tenant_id/config", |r| { api_handler(r, get_tenant_config_handler) }) - .put("/v1/tenant/:tenant_id/location_config", |r| { + .put("/v1/tenant/:tenant_shard_id/location_config", |r| { api_handler(r, put_tenant_location_config_handler) }) .get("/v1/tenant/:tenant_id/timeline", |r| { api_handler(r, timeline_list_handler) }) - .post("/v1/tenant/:tenant_id/timeline", |r| { + .post("/v1/tenant/:tenant_shard_id/timeline", |r| { api_handler(r, timeline_create_handler) }) .post("/v1/tenant/:tenant_id/attach", |r| { @@ -1756,7 +1830,7 @@ pub fn make_router( "/v1/tenant/:tenant_id/timeline/:timeline_id/download_remote_layers", |r| api_handler(r, timeline_download_remote_layers_handler_get), ) - .delete("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| { + .delete("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| { api_handler(r, timeline_delete_handler) }) .get("/v1/tenant/:tenant_id/timeline/:timeline_id/layer", |r| { diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index e71130e8af..3f74694ef2 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(clippy::undocumented_unsafe_blocks)] + mod auth; pub mod basebackup; pub mod config; @@ -61,14 +63,6 @@ pub async fn shutdown_pageserver(deletion_queue: Option, exit_cod ) .await; - // Shut down any page service tasks. - timed( - task_mgr::shutdown_tasks(Some(TaskKind::PageRequestHandler), None, None), - "shutdown PageRequestHandlers", - Duration::from_secs(1), - ) - .await; - // Shut down all the tenants. This flushes everything to disk and kills // the checkpoint and GC tasks. timed( @@ -78,6 +72,15 @@ pub async fn shutdown_pageserver(deletion_queue: Option, exit_cod ) .await; + // Shut down any page service tasks: any in-progress work for particular timelines or tenants + // should already have been canclled via mgr::shutdown_all_tenants + timed( + task_mgr::shutdown_tasks(Some(TaskKind::PageRequestHandler), None, None), + "shutdown PageRequestHandlers", + Duration::from_secs(1), + ) + .await; + // Best effort to persist any outstanding deletions, to avoid leaking objects if let Some(mut deletion_queue) = deletion_queue { deletion_queue.shutdown(Duration::from_secs(5)).await; diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 3e15be67dc..d5915f4c98 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -638,7 +638,7 @@ const STORAGE_IO_TIME_BUCKETS: &[f64] = &[ /// /// Operations: /// - open ([`std::fs::OpenOptions::open`]) -/// - close (dropping [`std::fs::File`]) +/// - close (dropping [`crate::virtual_file::VirtualFile`]) /// - close-by-replace (close by replacement algorithm) /// - read (`read_at`) /// - write (`write_at`) @@ -962,6 +962,32 @@ static REMOTE_TIMELINE_CLIENT_BYTES_FINISHED_COUNTER: Lazy = Lazy .expect("failed to define a metric") }); +pub(crate) struct TenantManagerMetrics { + pub(crate) tenant_slots: UIntGauge, + pub(crate) tenant_slot_writes: IntCounter, + pub(crate) unexpected_errors: IntCounter, +} + +pub(crate) static TENANT_MANAGER: Lazy = Lazy::new(|| { + TenantManagerMetrics { + tenant_slots: register_uint_gauge!( + "pageserver_tenant_manager_slots", + "How many slots currently exist, including all attached, secondary and in-progress operations", + ) + .expect("failed to define a metric"), + tenant_slot_writes: register_int_counter!( + "pageserver_tenant_manager_slot_writes", + "Writes to a tenant slot, including all of create/attach/detach/delete" + ) + .expect("failed to define a metric"), + unexpected_errors: register_int_counter!( + "pageserver_tenant_manager_unexpected_errors_total", + "Number of unexpected conditions encountered: nonzero value indicates a non-fatal bug." + ) + .expect("failed to define a metric"), +} +}); + pub(crate) struct DeletionQueueMetrics { pub(crate) keys_submitted: IntCounter, pub(crate) keys_dropped: IntCounter, @@ -1199,15 +1225,6 @@ pub(crate) static WAL_REDO_TIME: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); -pub(crate) static WAL_REDO_WAIT_TIME: Lazy = Lazy::new(|| { - register_histogram!( - "pageserver_wal_redo_wait_seconds", - "Time spent waiting for access to the Postgres WAL redo process", - redo_histogram_time_buckets!(), - ) - .expect("failed to define a metric") -}); - pub(crate) static WAL_REDO_RECORDS_HISTOGRAM: Lazy = Lazy::new(|| { register_histogram!( "pageserver_wal_redo_records_histogram", @@ -1235,6 +1252,46 @@ pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy = Lazy::new(|| { .unwrap() }); +pub(crate) struct WalRedoProcessCounters { + pub(crate) started: IntCounter, + pub(crate) killed_by_cause: enum_map::EnumMap, +} + +#[derive(Debug, enum_map::Enum, strum_macros::IntoStaticStr)] +pub(crate) enum WalRedoKillCause { + WalRedoProcessDrop, + NoLeakChildDrop, + Startup, +} + +impl Default for WalRedoProcessCounters { + fn default() -> Self { + let started = register_int_counter!( + "pageserver_wal_redo_process_started_total", + "Number of WAL redo processes started", + ) + .unwrap(); + + let killed = register_int_counter_vec!( + "pageserver_wal_redo_process_stopped_total", + "Number of WAL redo processes stopped", + &["cause"], + ) + .unwrap(); + Self { + started, + killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| { + let cause = ::from_usize(i); + let cause_str: &'static str = cause.into(); + killed.with_label_values(&[cause_str]) + })), + } + } +} + +pub(crate) static WAL_REDO_PROCESS_COUNTERS: Lazy = + Lazy::new(WalRedoProcessCounters::default); + /// Similar to `prometheus::HistogramTimer` but does not record on drop. pub struct StorageTimeMetricsTimer { metrics: StorageTimeMetrics, @@ -1884,6 +1941,9 @@ pub fn preinitialize_metrics() { // Deletion queue stats Lazy::force(&DELETION_QUEUE); + // Tenant manager stats + Lazy::force(&TENANT_MANAGER); + // countervecs [&BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT] .into_iter() @@ -1899,7 +1959,6 @@ pub fn preinitialize_metrics() { &READ_NUM_FS_LAYERS, &WAIT_LSN_TIME, &WAL_REDO_TIME, - &WAL_REDO_WAIT_TIME, &WAL_REDO_RECORDS_HISTOGRAM, &WAL_REDO_BYTES_HISTOGRAM, ] diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 536334d051..ee5f1732e4 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -40,7 +40,7 @@ use tracing::field; use tracing::*; use utils::id::ConnectionId; use utils::{ - auth::{Claims, JwtAuth, Scope}, + auth::{Claims, Scope, SwappableJwtAuth}, id::{TenantId, TimelineId}, lsn::Lsn, simple_rcu::RcuReadGuard, @@ -55,16 +55,20 @@ use crate::metrics; use crate::metrics::LIVE_CONNECTIONS_COUNT; use crate::task_mgr; use crate::task_mgr::TaskKind; -use crate::tenant; use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::mgr; -use crate::tenant::mgr::GetTenantError; -use crate::tenant::{Tenant, Timeline}; +use crate::tenant::mgr::get_active_tenant_with_timeout; +use crate::tenant::mgr::GetActiveTenantError; +use crate::tenant::Timeline; use crate::trace::Tracer; use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID; use postgres_ffi::BLCKSZ; +// How long we may block waiting for a [`TenantSlot::InProgress`]` and/or a [`Tenant`] which +// is not yet in state [`TenantState::Active`]. +const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(5000); + /// Read the end of a tar archive. /// /// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each. @@ -118,7 +122,7 @@ async fn read_tar_eof(mut reader: (impl AsyncRead + Unpin)) -> anyhow::Result<() pub async fn libpq_listener_main( conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, listener: TcpListener, auth_type: AuthType, listener_ctx: RequestContext, @@ -186,7 +190,7 @@ pub async fn libpq_listener_main( async fn page_service_conn_main( conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, socket: tokio::net::TcpStream, auth_type: AuthType, connection_ctx: RequestContext, @@ -214,22 +218,34 @@ async fn page_service_conn_main( // no write timeout is used, because the kernel is assumed to error writes after some time. let mut socket = tokio_io_timeout::TimeoutReader::new(socket); - // timeout should be lower, but trying out multiple days for - // - socket.set_timeout(Some(std::time::Duration::from_secs(60 * 60 * 24 * 3))); + let default_timeout_ms = 10 * 60 * 1000; // 10 minutes by default + let socket_timeout_ms = (|| { + fail::fail_point!("simulated-bad-compute-connection", |avg_timeout_ms| { + // Exponential distribution for simulating + // poor network conditions, expect about avg_timeout_ms to be around 15 + // in tests + if let Some(avg_timeout_ms) = avg_timeout_ms { + let avg = avg_timeout_ms.parse::().unwrap() as f32; + let u = rand::random::(); + ((1.0 - u).ln() / (-avg)) as u64 + } else { + default_timeout_ms + } + }); + default_timeout_ms + })(); + + // A timeout here does not mean the client died, it can happen if it's just idle for + // a while: we will tear down this PageServerHandler and instantiate a new one if/when + // they reconnect. + socket.set_timeout(Some(std::time::Duration::from_millis(socket_timeout_ms))); let socket = std::pin::pin!(socket); // XXX: pgbackend.run() should take the connection_ctx, // and create a child per-query context when it invokes process_query. // But it's in a shared crate, so, we store connection_ctx inside PageServerHandler // and create the per-query context in process_query ourselves. - let mut conn_handler = PageServerHandler::new( - conf, - broker_client, - auth, - connection_ctx, - task_mgr::shutdown_token(), - ); + let mut conn_handler = PageServerHandler::new(conf, broker_client, auth, connection_ctx); let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; match pgbackend @@ -255,7 +271,7 @@ async fn page_service_conn_main( struct PageServerHandler { _conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, claims: Option, /// The context created for the lifetime of the connection @@ -263,19 +279,14 @@ struct PageServerHandler { /// For each query received over the connection, /// `process_query` creates a child context from this one. connection_ctx: RequestContext, - - /// A token that should fire when the tenant transitions from - /// attached state, or when the pageserver is shutting down. - cancel: CancellationToken, } impl PageServerHandler { pub fn new( conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, - auth: Option>, + auth: Option>, connection_ctx: RequestContext, - cancel: CancellationToken, ) -> Self { PageServerHandler { _conf: conf, @@ -283,7 +294,6 @@ impl PageServerHandler { auth, claims: None, connection_ctx, - cancel, } } @@ -291,7 +301,11 @@ impl PageServerHandler { /// this rather than naked flush() in order to shut down promptly. Without this, we would /// block shutdown of a tenant if a postgres client was failing to consume bytes we send /// in the flush. - async fn flush_cancellable(&self, pgb: &mut PostgresBackend) -> Result<(), QueryError> + async fn flush_cancellable( + &self, + pgb: &mut PostgresBackend, + cancel: &CancellationToken, + ) -> Result<(), QueryError> where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { @@ -299,7 +313,7 @@ impl PageServerHandler { flush_r = pgb.flush() => { Ok(flush_r?) }, - _ = self.cancel.cancelled() => { + _ = cancel.cancelled() => { Err(QueryError::Shutdown) } ) @@ -308,6 +322,7 @@ impl PageServerHandler { fn copyin_stream<'a, IO>( &'a self, pgb: &'a mut PostgresBackend, + cancel: &'a CancellationToken, ) -> impl Stream> + 'a where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, @@ -317,7 +332,7 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = self.cancel.cancelled() => { + _ = cancel.cancelled() => { // We were requested to shut down. let msg = "pageserver is shutting down"; let _ = pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, None)); @@ -357,7 +372,7 @@ impl PageServerHandler { let query_error = QueryError::Disconnected(ConnectionError::Io(io::Error::new(io::ErrorKind::ConnectionReset, msg))); // error can't happen here, ErrorResponse serialization should be always ok pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, Some(query_error.pg_error_code()))).map_err(|e| e.into_io_error())?; - self.flush_cancellable(pgb).await.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + self.flush_cancellable(pgb, cancel).await.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?; } Err(QueryError::Disconnected(ConnectionError::Io(io_error))) => { @@ -384,12 +399,13 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - // NOTE: pagerequests handler exits when connection is closed, - // so there is no need to reset the association - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); - // Make request tracer if needed - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = mgr::get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; let mut tracer = if tenant.get_trace_read_requests() { let connection_id = ConnectionId::generate(); let path = tenant @@ -405,9 +421,14 @@ impl PageServerHandler { .get_timeline(timeline_id, true) .map_err(|e| anyhow::anyhow!(e))?; + // Avoid starting new requests if the timeline has already started shutting down, + // and block timeline shutdown until this request is complete, or drops out due + // to cancellation. + let _timeline_guard = timeline.gate.enter().map_err(|_| QueryError::Shutdown)?; + // switch client to COPYBOTH pgb.write_message_noflush(&BeMessage::CopyBothResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; let metrics = metrics::SmgrQueryTimePerTimeline::new(&tenant_id, &timeline_id); @@ -415,7 +436,7 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = self.cancel.cancelled() => { + _ = timeline.cancel.cancelled() => { // We were requested to shut down. info!("shutdown request received in page handler"); return Err(QueryError::Shutdown) @@ -490,9 +511,24 @@ impl PageServerHandler { } }; + if let Err(e) = &response { + // Requests may fail as soon as we are Stopping, even if the Timeline's cancellation token wasn't fired yet, + // because wait_lsn etc will drop out + // is_stopping(): [`Timeline::flush_and_shutdown`] has entered + // is_canceled(): [`Timeline::shutdown`]` has entered + if timeline.cancel.is_cancelled() || timeline.is_stopping() { + // If we fail to fulfil a request during shutdown, which may be _because_ of + // shutdown, then do not send the error to the client. Instead just drop the + // connection. + span.in_scope(|| info!("dropped response during shutdown: {e:#}")); + return Err(QueryError::Shutdown); + } + } + let response = response.unwrap_or_else(|e| { // print the all details to the log with {:#}, but for the client the - // error message is enough + // error message is enough. Do not log if shutting down, as the anyhow::Error + // here includes cancellation which is not an error. span.in_scope(|| error!("error reading relation or page version: {:#}", e)); PagestreamBeMessage::Error(PagestreamErrorResponse { message: e.to_string(), @@ -500,7 +536,7 @@ impl PageServerHandler { }); pgb.write_message_noflush(&BeMessage::CopyData(&response.serialize()))?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; } Ok(()) } @@ -522,10 +558,14 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); // Create empty timeline info!("creating new timeline"); - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; let timeline = tenant .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx) .await?; @@ -543,9 +583,9 @@ impl PageServerHandler { // Import basebackup provided via CopyData info!("importing basebackup"); pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &tenant.cancel).await?; - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb))); + let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &tenant.cancel))); timeline .import_basebackup_from_tar( &mut copyin_reader, @@ -582,9 +622,10 @@ impl PageServerHandler { IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { debug_assert_current_span_has_tenant_and_timeline_id(); - task_mgr::associate_with(Some(tenant_id), Some(timeline_id)); - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let last_record_lsn = timeline.get_last_record_lsn(); if last_record_lsn != start_lsn { return Err(QueryError::Other( @@ -598,8 +639,8 @@ impl PageServerHandler { // Import wal provided via CopyData info!("importing wal"); pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb).await?; - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb))); + self.flush_cancellable(pgb, &timeline.cancel).await?; + let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &timeline.cancel))); import_wal_from_tar(&timeline, &mut copyin_reader, start_lsn, end_lsn, &ctx).await?; info!("wal import complete"); @@ -792,7 +833,9 @@ impl PageServerHandler { let started = std::time::Instant::now(); // check that the timeline exists - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); if let Some(lsn) = lsn { // Backup was requested at a particular LSN. Wait for it to arrive. @@ -807,7 +850,7 @@ impl PageServerHandler { // switch client to COPYOUT pgb.write_message_noflush(&BeMessage::CopyOutResponse)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; // Send a tarball of the latest layer on the timeline. Compress if not // fullbackup. TODO Compress in that case too (tests need to be updated) @@ -859,7 +902,7 @@ impl PageServerHandler { } pgb.write_message_noflush(&BeMessage::CopyDone)?; - self.flush_cancellable(pgb).await?; + self.flush_cancellable(pgb, &timeline.cancel).await?; let basebackup_after = started .elapsed() @@ -877,7 +920,7 @@ impl PageServerHandler { // when accessing management api supply None as an argument // when using to authorize tenant pass corresponding tenant id - fn check_permission(&self, tenant_id: Option) -> anyhow::Result<()> { + fn check_permission(&self, tenant_id: Option) -> Result<(), QueryError> { if self.auth.is_none() { // auth is set to Trust, nothing to check so just return ok return Ok(()); @@ -889,7 +932,26 @@ impl PageServerHandler { .claims .as_ref() .expect("claims presence already checked"); - check_permission(claims, tenant_id) + check_permission(claims, tenant_id).map_err(|e| QueryError::Unauthorized(e.0)) + } + + /// Shorthand for getting a reference to a Timeline of an Active tenant. + async fn get_active_tenant_timeline( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> Result, GetActiveTimelineError> { + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await + .map_err(GetActiveTimelineError::Tenant)?; + let timeline = tenant + .get_timeline(timeline_id, true) + .map_err(|e| GetActiveTimelineError::Timeline(anyhow::anyhow!(e)))?; + Ok(timeline) } } @@ -909,16 +971,17 @@ where .auth .as_ref() .unwrap() - .decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?)?; + .decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?) + .map_err(|e| QueryError::Unauthorized(e.0))?; if matches!(data.claims.scope, Scope::Tenant) && data.claims.tenant_id.is_none() { - return Err(QueryError::Other(anyhow::anyhow!( - "jwt token scope is Tenant, but tenant id is missing" - ))); + return Err(QueryError::Unauthorized( + "jwt token scope is Tenant, but tenant id is missing".into(), + )); } - info!( - "jwt auth succeeded for scope: {:#?} by tenant id: {:?}", + debug!( + "jwt scope check succeeded for scope: {:#?} by tenant id: {:?}", data.claims.scope, data.claims.tenant_id, ); @@ -940,9 +1003,13 @@ where pgb: &mut PostgresBackend, query_string: &str, ) -> Result<(), QueryError> { + fail::fail_point!("simulated-bad-compute-connection", |_| { + info!("Hit failpoint for bad connection"); + Err(QueryError::SimulatedConnectionError) + }); + let ctx = self.connection_ctx.attached_child(); debug!("process query {query_string:?}"); - if query_string.starts_with("pagestream ") { let (_, params_raw) = query_string.split_at("pagestream ".len()); let params = params_raw.split(' ').collect::>(); @@ -1048,7 +1115,9 @@ where .record("timeline_id", field::display(timeline_id)); self.check_permission(Some(tenant_id))?; - let timeline = get_active_tenant_timeline(tenant_id, timeline_id, &ctx).await?; + let timeline = self + .get_active_tenant_timeline(tenant_id, timeline_id) + .await?; let end_of_timeline = timeline.get_last_record_rlsn(); @@ -1232,7 +1301,12 @@ where self.check_permission(Some(tenant_id))?; - let tenant = get_active_tenant_with_timeout(tenant_id, &ctx).await?; + let tenant = get_active_tenant_with_timeout( + tenant_id, + ACTIVE_TENANT_TIMEOUT, + &task_mgr::shutdown_token(), + ) + .await?; pgb.write_message_noflush(&BeMessage::RowDescription(&[ RowDescriptor::int8_col(b"checkpoint_distance"), RowDescriptor::int8_col(b"checkpoint_timeout"), @@ -1278,67 +1352,16 @@ where } } -#[derive(thiserror::Error, Debug)] -enum GetActiveTenantError { - #[error( - "Timed out waiting {wait_time:?} for tenant active state. Latest state: {latest_state:?}" - )] - WaitForActiveTimeout { - latest_state: TenantState, - wait_time: Duration, - }, - #[error(transparent)] - NotFound(GetTenantError), - #[error(transparent)] - WaitTenantActive(tenant::WaitToBecomeActiveError), -} - impl From for QueryError { fn from(e: GetActiveTenantError) -> Self { match e { GetActiveTenantError::WaitForActiveTimeout { .. } => QueryError::Disconnected( ConnectionError::Io(io::Error::new(io::ErrorKind::TimedOut, e.to_string())), ), - GetActiveTenantError::WaitTenantActive(e) => QueryError::Other(anyhow::Error::new(e)), - GetActiveTenantError::NotFound(e) => QueryError::Other(anyhow::Error::new(e)), - } - } -} - -/// Get active tenant. -/// -/// If the tenant is Loading, waits for it to become Active, for up to 30 s. That -/// ensures that queries don't fail immediately after pageserver startup, because -/// all tenants are still loading. -async fn get_active_tenant_with_timeout( - tenant_id: TenantId, - _ctx: &RequestContext, /* require get a context to support cancellation in the future */ -) -> Result, GetActiveTenantError> { - let tenant = match mgr::get_tenant(tenant_id, false).await { - Ok(tenant) => tenant, - Err(e @ GetTenantError::NotFound(_)) => return Err(GetActiveTenantError::NotFound(e)), - Err(GetTenantError::NotActive(_)) => { - unreachable!("we're calling get_tenant with active_only=false") - } - Err(GetTenantError::Broken(_)) => { - unreachable!("we're calling get_tenant with active_only=false") - } - }; - let wait_time = Duration::from_secs(30); - match tokio::time::timeout(wait_time, tenant.wait_to_become_active()).await { - Ok(Ok(())) => Ok(tenant), - // no .context(), the error message is good enough and some tests depend on it - Ok(Err(e)) => Err(GetActiveTenantError::WaitTenantActive(e)), - Err(_) => { - let latest_state = tenant.current_state(); - if latest_state == TenantState::Active { - Ok(tenant) - } else { - Err(GetActiveTenantError::WaitForActiveTimeout { - latest_state, - wait_time, - }) + GetActiveTenantError::WillNotBecomeActive(TenantState::Stopping { .. }) => { + QueryError::Shutdown } + e => QueryError::Other(anyhow::anyhow!(e)), } } } @@ -1359,18 +1382,3 @@ impl From for QueryError { } } } - -/// Shorthand for getting a reference to a Timeline of an Active tenant. -async fn get_active_tenant_timeline( - tenant_id: TenantId, - timeline_id: TimelineId, - ctx: &RequestContext, -) -> Result, GetActiveTimelineError> { - let tenant = get_active_tenant_with_timeout(tenant_id, ctx) - .await - .map_err(GetActiveTimelineError::Tenant)?; - let timeline = tenant - .get_timeline(timeline_id, true) - .map_err(|e| GetActiveTimelineError::Timeline(anyhow::anyhow!(e)))?; - Ok(timeline) -} diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index daadf6abd4..827278af72 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -21,8 +21,8 @@ use serde::{Deserialize, Serialize}; use std::collections::{hash_map, HashMap, HashSet}; use std::ops::ControlFlow; use std::ops::Range; -use tokio_util::sync::CancellationToken; use tracing::{debug, trace, warn}; +use utils::bin_ser::DeserializeError; use utils::{bin_ser::BeSer, lsn::Lsn}; /// Block number within a relation or SLRU. This matches PostgreSQL's BlockNumber type. @@ -30,9 +30,33 @@ pub type BlockNumber = u32; #[derive(Debug)] pub enum LsnForTimestamp { + /// Found commits both before and after the given timestamp Present(Lsn), + + /// Found no commits after the given timestamp, this means + /// that the newest data in the branch is older than the given + /// timestamp. + /// + /// All commits <= LSN happened before the given timestamp Future(Lsn), + + /// The queried timestamp is past our horizon we look back at (PITR) + /// + /// All commits > LSN happened after the given timestamp, + /// but any commits < LSN might have happened before or after + /// the given timestamp. We don't know because no data before + /// the given lsn is available. Past(Lsn), + + /// We have found no commit with a timestamp, + /// so we can't return anything meaningful. + /// + /// The associated LSN is the lower bound value we can safely + /// create branches on, but no statement is made if it is + /// older or newer than the timestamp. + /// + /// This variant can e.g. be returned right after a + /// cluster import. NoData(Lsn), } @@ -44,6 +68,36 @@ pub enum CalculateLogicalSizeError { Other(#[from] anyhow::Error), } +#[derive(Debug, thiserror::Error)] +pub(crate) enum CollectKeySpaceError { + #[error(transparent)] + Decode(#[from] DeserializeError), + #[error(transparent)] + PageRead(PageReconstructError), + #[error("cancelled")] + Cancelled, +} + +impl From for CollectKeySpaceError { + fn from(err: PageReconstructError) -> Self { + match err { + PageReconstructError::Cancelled => Self::Cancelled, + err => Self::PageRead(err), + } + } +} + +impl From for CalculateLogicalSizeError { + fn from(pre: PageReconstructError) -> Self { + match pre { + PageReconstructError::AncestorStopping(_) | PageReconstructError::Cancelled => { + Self::Cancelled + } + _ => Self::Other(pre.into()), + } + } +} + #[derive(Debug, thiserror::Error)] pub enum RelationError { #[error("Relation Already Exists")] @@ -314,7 +368,11 @@ impl Timeline { ctx: &RequestContext, ) -> Result { let gc_cutoff_lsn_guard = self.get_latest_gc_cutoff_lsn(); - let min_lsn = *gc_cutoff_lsn_guard; + // We use this method to figure out the branching LSN for the new branch, but the + // GC cutoff could be before the branching point and we cannot create a new branch + // with LSN < `ancestor_lsn`. Thus, pick the maximum of these two to be + // on the safe side. + let min_lsn = std::cmp::max(*gc_cutoff_lsn_guard, self.get_ancestor_lsn()); let max_lsn = self.get_last_record_lsn(); // LSNs are always 8-byte aligned. low/mid/high represent the @@ -344,30 +402,33 @@ impl Timeline { low = mid + 1; } } + // If `found_smaller == true`, `low = t + 1` where `t` is the target LSN, + // so the LSN of the last commit record before or at `search_timestamp`. + // Remove one from `low` to get `t`. + // + // FIXME: it would be better to get the LSN of the previous commit. + // Otherwise, if you restore to the returned LSN, the database will + // include physical changes from later commits that will be marked + // as aborted, and will need to be vacuumed away. + let commit_lsn = Lsn((low - 1) * 8); match (found_smaller, found_larger) { (false, false) => { // This can happen if no commit records have been processed yet, e.g. // just after importing a cluster. - Ok(LsnForTimestamp::NoData(max_lsn)) - } - (true, false) => { - // Didn't find any commit timestamps larger than the request - Ok(LsnForTimestamp::Future(max_lsn)) + Ok(LsnForTimestamp::NoData(min_lsn)) } (false, true) => { // Didn't find any commit timestamps smaller than the request - Ok(LsnForTimestamp::Past(max_lsn)) + Ok(LsnForTimestamp::Past(min_lsn)) } - (true, true) => { - // low is the LSN of the first commit record *after* the search_timestamp, - // Back off by one to get to the point just before the commit. - // - // FIXME: it would be better to get the LSN of the previous commit. - // Otherwise, if you restore to the returned LSN, the database will - // include physical changes from later commits that will be marked - // as aborted, and will need to be vacuumed away. - Ok(LsnForTimestamp::Present(Lsn((low - 1) * 8))) + (true, false) => { + // Only found commits with timestamps smaller than the request. + // It's still a valid case for branch creation, return it. + // And `update_gc_info()` ignores LSN for a `LsnForTimestamp::Future` + // case, anyway. + Ok(LsnForTimestamp::Future(commit_lsn)) } + (true, true) => Ok(LsnForTimestamp::Present(commit_lsn)), } } @@ -567,30 +628,22 @@ impl Timeline { pub async fn get_current_logical_size_non_incremental( &self, lsn: Lsn, - cancel: CancellationToken, ctx: &RequestContext, ) -> Result { crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id(); // Fetch list of database dirs and iterate them - let buf = self.get(DBDIR_KEY, lsn, ctx).await.context("read dbdir")?; + let buf = self.get(DBDIR_KEY, lsn, ctx).await?; let dbdir = DbDirectory::des(&buf).context("deserialize db directory")?; let mut total_size: u64 = 0; for (spcnode, dbnode) in dbdir.dbdirs.keys() { - for rel in self - .list_rels(*spcnode, *dbnode, lsn, ctx) - .await - .context("list rels")? - { - if cancel.is_cancelled() { + for rel in self.list_rels(*spcnode, *dbnode, lsn, ctx).await? { + if self.cancel.is_cancelled() { return Err(CalculateLogicalSizeError::Cancelled); } let relsize_key = rel_size_to_key(rel); - let mut buf = self - .get(relsize_key, lsn, ctx) - .await - .with_context(|| format!("read relation size of {rel:?}"))?; + let mut buf = self.get(relsize_key, lsn, ctx).await?; let relsize = buf.get_u32_le(); total_size += relsize as u64; @@ -603,11 +656,11 @@ impl Timeline { /// Get a KeySpace that covers all the Keys that are in use at the given LSN. /// Anything that's not listed maybe removed from the underlying storage (from /// that LSN forwards). - pub async fn collect_keyspace( + pub(crate) async fn collect_keyspace( &self, lsn: Lsn, ctx: &RequestContext, - ) -> anyhow::Result { + ) -> Result { // Iterate through key ranges, greedily packing them into partitions let mut result = KeySpaceAccum::new(); @@ -616,7 +669,7 @@ impl Timeline { // Fetch list of database dirs and iterate them let buf = self.get(DBDIR_KEY, lsn, ctx).await?; - let dbdir = DbDirectory::des(&buf).context("deserialization failure")?; + let dbdir = DbDirectory::des(&buf)?; let mut dbs: Vec<(Oid, Oid)> = dbdir.dbdirs.keys().cloned().collect(); dbs.sort_unstable(); @@ -649,7 +702,7 @@ impl Timeline { let slrudir_key = slru_dir_to_key(kind); result.add_key(slrudir_key); let buf = self.get(slrudir_key, lsn, ctx).await?; - let dir = SlruSegmentDirectory::des(&buf).context("deserialization failure")?; + let dir = SlruSegmentDirectory::des(&buf)?; let mut segments: Vec = dir.segments.iter().cloned().collect(); segments.sort_unstable(); for segno in segments { @@ -667,7 +720,7 @@ impl Timeline { // Then pg_twophase result.add_key(TWOPHASEDIR_KEY); let buf = self.get(TWOPHASEDIR_KEY, lsn, ctx).await?; - let twophase_dir = TwoPhaseDirectory::des(&buf).context("deserialization failure")?; + let twophase_dir = TwoPhaseDirectory::des(&buf)?; let mut xids: Vec = twophase_dir.xids.iter().cloned().collect(); xids.sort_unstable(); for xid in xids { diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 7a94c3449d..24f47df92e 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -1,106 +1,11 @@ use crate::walrecord::NeonWalRecord; -use anyhow::{bail, Result}; -use byteorder::{ByteOrder, BE}; +use anyhow::Result; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use std::fmt; use std::ops::{AddAssign, Range}; use std::time::Duration; -/// Key used in the Repository kv-store. -/// -/// The Repository treats this as an opaque struct, but see the code in pgdatadir_mapping.rs -/// for what we actually store in these fields. -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] -pub struct Key { - pub field1: u8, - pub field2: u32, - pub field3: u32, - pub field4: u32, - pub field5: u8, - pub field6: u32, -} - -pub const KEY_SIZE: usize = 18; - -impl Key { - /// 'field2' is used to store tablespaceid for relations and small enum numbers for other relish. - /// As long as Neon does not support tablespace (because of lack of access to local file system), - /// we can assume that only some predefined namespace OIDs are used which can fit in u16 - pub fn to_i128(&self) -> i128 { - assert!(self.field2 < 0xFFFF || self.field2 == 0xFFFFFFFF || self.field2 == 0x22222222); - (((self.field1 & 0xf) as i128) << 120) - | (((self.field2 & 0xFFFF) as i128) << 104) - | ((self.field3 as i128) << 72) - | ((self.field4 as i128) << 40) - | ((self.field5 as i128) << 32) - | self.field6 as i128 - } - - pub const fn from_i128(x: i128) -> Self { - Key { - field1: ((x >> 120) & 0xf) as u8, - field2: ((x >> 104) & 0xFFFF) as u32, - field3: (x >> 72) as u32, - field4: (x >> 40) as u32, - field5: (x >> 32) as u8, - field6: x as u32, - } - } - - pub fn next(&self) -> Key { - self.add(1) - } - - pub fn add(&self, x: u32) -> Key { - let mut key = *self; - - let r = key.field6.overflowing_add(x); - key.field6 = r.0; - if r.1 { - let r = key.field5.overflowing_add(1); - key.field5 = r.0; - if r.1 { - let r = key.field4.overflowing_add(1); - key.field4 = r.0; - if r.1 { - let r = key.field3.overflowing_add(1); - key.field3 = r.0; - if r.1 { - let r = key.field2.overflowing_add(1); - key.field2 = r.0; - if r.1 { - let r = key.field1.overflowing_add(1); - key.field1 = r.0; - assert!(!r.1); - } - } - } - } - } - key - } - - pub fn from_slice(b: &[u8]) -> Self { - Key { - field1: b[0], - field2: u32::from_be_bytes(b[1..5].try_into().unwrap()), - field3: u32::from_be_bytes(b[5..9].try_into().unwrap()), - field4: u32::from_be_bytes(b[9..13].try_into().unwrap()), - field5: b[13], - field6: u32::from_be_bytes(b[14..18].try_into().unwrap()), - } - } - - pub fn write_to_byte_slice(&self, buf: &mut [u8]) { - buf[0] = self.field1; - BE::write_u32(&mut buf[1..5], self.field2); - BE::write_u32(&mut buf[5..9], self.field3); - BE::write_u32(&mut buf[9..13], self.field4); - buf[13] = self.field5; - BE::write_u32(&mut buf[14..18], self.field6); - } -} +pub use pageserver_api::key::{Key, KEY_SIZE}; pub fn key_range_size(key_range: &Range) -> u32 { let start = key_range.start; @@ -129,51 +34,9 @@ pub fn singleton_range(key: Key) -> Range { key..key.next() } -impl fmt::Display for Key { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{:02X}{:08X}{:08X}{:08X}{:02X}{:08X}", - self.field1, self.field2, self.field3, self.field4, self.field5, self.field6 - ) - } -} - -impl Key { - pub const MIN: Key = Key { - field1: u8::MIN, - field2: u32::MIN, - field3: u32::MIN, - field4: u32::MIN, - field5: u8::MIN, - field6: u32::MIN, - }; - pub const MAX: Key = Key { - field1: u8::MAX, - field2: u32::MAX, - field3: u32::MAX, - field4: u32::MAX, - field5: u8::MAX, - field6: u32::MAX, - }; - - pub fn from_hex(s: &str) -> Result { - if s.len() != 36 { - bail!("parse error"); - } - Ok(Key { - field1: u8::from_str_radix(&s[0..2], 16)?, - field2: u32::from_str_radix(&s[2..10], 16)?, - field3: u32::from_str_radix(&s[10..18], 16)?, - field4: u32::from_str_radix(&s[18..26], 16)?, - field5: u8::from_str_radix(&s[26..28], 16)?, - field6: u32::from_str_radix(&s[28..36], 16)?, - }) - } -} - /// A 'value' stored for a one Key. #[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(test, derive(PartialEq))] pub enum Value { /// An Image value contains a full copy of the value Image(Bytes), @@ -197,6 +60,70 @@ impl Value { } } +#[cfg(test)] +mod test { + use super::*; + + use bytes::Bytes; + use utils::bin_ser::BeSer; + + macro_rules! roundtrip { + ($orig:expr, $expected:expr) => {{ + let orig: Value = $orig; + + let actual = Value::ser(&orig).unwrap(); + let expected: &[u8] = &$expected; + + assert_eq!(utils::Hex(&actual), utils::Hex(expected)); + + let deser = Value::des(&actual).unwrap(); + + assert_eq!(orig, deser); + }}; + } + + #[test] + fn image_roundtrip() { + let image = Bytes::from_static(b"foobar"); + let image = Value::Image(image); + + #[rustfmt::skip] + let expected = [ + // top level discriminator of 4 bytes + 0x00, 0x00, 0x00, 0x00, + // 8 byte length + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // foobar + 0x66, 0x6f, 0x6f, 0x62, 0x61, 0x72 + ]; + + roundtrip!(image, expected); + } + + #[test] + fn walrecord_postgres_roundtrip() { + let rec = NeonWalRecord::Postgres { + will_init: true, + rec: Bytes::from_static(b"foobar"), + }; + let rec = Value::WalRecord(rec); + + #[rustfmt::skip] + let expected = [ + // flattened discriminator of total 8 bytes + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + // will_init + 0x01, + // 8 byte length + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + // foobar + 0x66, 0x6f, 0x6f, 0x62, 0x61, 0x72 + ]; + + roundtrip!(rec, expected); + } +} + /// /// Result of performing GC /// diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 017322ffb2..4270b6edb0 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -299,10 +299,6 @@ pub enum TaskKind { #[derive(Default)] struct MutableTaskState { - /// Tenant and timeline that this task is associated with. - tenant_id: Option, - timeline_id: Option, - /// Handle for waiting for the task to exit. It can be None, if the /// the task has already exited. join_handle: Option>, @@ -319,6 +315,11 @@ struct PageServerTask { // To request task shutdown, just cancel this token. cancel: CancellationToken, + /// Tasks may optionally be launched for a particular tenant/timeline, enabling + /// later cancelling tasks for that tenant/timeline in [`shutdown_tasks`] + tenant_id: Option, + timeline_id: Option, + mutable: Mutex, } @@ -344,11 +345,9 @@ where kind, name: name.to_string(), cancel: cancel.clone(), - mutable: Mutex::new(MutableTaskState { - tenant_id, - timeline_id, - join_handle: None, - }), + tenant_id, + timeline_id, + mutable: Mutex::new(MutableTaskState { join_handle: None }), }); TASKS.lock().unwrap().insert(task_id, Arc::clone(&task)); @@ -418,8 +417,6 @@ async fn task_finish( let mut shutdown_process = false; { - let task_mut = task.mutable.lock().unwrap(); - match result { Ok(Ok(())) => { debug!("Task '{}' exited normally", task_name); @@ -428,13 +425,13 @@ async fn task_finish( if shutdown_process_on_error { error!( "Shutting down: task '{}' tenant_id: {:?}, timeline_id: {:?} exited with error: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); shutdown_process = true; } else { error!( "Task '{}' tenant_id: {:?}, timeline_id: {:?} exited with error: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); } } @@ -442,13 +439,13 @@ async fn task_finish( if shutdown_process_on_error { error!( "Shutting down: task '{}' tenant_id: {:?}, timeline_id: {:?} panicked: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); shutdown_process = true; } else { error!( "Task '{}' tenant_id: {:?}, timeline_id: {:?} panicked: {:?}", - task_name, task_mut.tenant_id, task_mut.timeline_id, err + task_name, task.tenant_id, task.timeline_id, err ); } } @@ -460,17 +457,6 @@ async fn task_finish( } } -// expected to be called from the task of the given id. -pub fn associate_with(tenant_id: Option, timeline_id: Option) { - CURRENT_TASK.with(|ct| { - let mut task_mut = ct.mutable.lock().unwrap(); - task_mut.tenant_id = tenant_id; - task_mut.timeline_id = timeline_id; - }); -} - -/// Is there a task running that matches the criteria - /// Signal and wait for tasks to shut down. /// /// @@ -493,17 +479,16 @@ pub async fn shutdown_tasks( { let tasks = TASKS.lock().unwrap(); for task in tasks.values() { - let task_mut = task.mutable.lock().unwrap(); if (kind.is_none() || Some(task.kind) == kind) - && (tenant_id.is_none() || task_mut.tenant_id == tenant_id) - && (timeline_id.is_none() || task_mut.timeline_id == timeline_id) + && (tenant_id.is_none() || task.tenant_id == tenant_id) + && (timeline_id.is_none() || task.timeline_id == timeline_id) { task.cancel.cancel(); victim_tasks.push(( Arc::clone(task), task.kind, - task_mut.tenant_id, - task_mut.timeline_id, + task.tenant_id, + task.timeline_id, )); } } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 3a426ac87b..04fe9db76a 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -26,6 +26,7 @@ use tracing::*; use utils::completion; use utils::crashsafe::path_with_suffix_extension; use utils::fs_ext; +use utils::sync::gate::Gate; use std::cmp::min; use std::collections::hash_map::Entry; @@ -54,6 +55,8 @@ use self::config::TenantConf; use self::delete::DeleteTenantFlow; use self::metadata::LoadMetadataError; use self::metadata::TimelineMetadata; +use self::mgr::GetActiveTenantError; +use self::mgr::GetTenantError; use self::mgr::TenantsMap; use self::remote_timeline_client::RemoteTimelineClient; use self::timeline::uninit::TimelineUninitMark; @@ -252,6 +255,20 @@ pub struct Tenant { eviction_task_tenant_state: tokio::sync::Mutex, pub(crate) delete_progress: Arc>, + + // Cancellation token fires when we have entered shutdown(). This is a parent of + // Timelines' cancellation token. + pub(crate) cancel: CancellationToken, + + // Users of the Tenant such as the page service must take this Gate to avoid + // trying to use a Tenant which is shutting down. + pub(crate) gate: Gate, +} + +impl std::fmt::Debug for Tenant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} ({})", self.tenant_id, self.current_state()) + } } pub(crate) enum WalRedoManager { @@ -274,6 +291,16 @@ impl From for WalRedoManager { } impl WalRedoManager { + pub(crate) fn maybe_quiesce(&self, idle_timeout: Duration) { + match self { + Self::Prod(mgr) => mgr.maybe_quiesce(idle_timeout), + #[cfg(test)] + Self::Test(_) => { + // Not applicable to test redo manager + } + } + } + pub async fn request_redo( &self, key: crate::repository::Key, @@ -359,34 +386,6 @@ impl Debug for SetStoppingError { } } -#[derive(Debug, thiserror::Error)] -pub(crate) enum WaitToBecomeActiveError { - WillNotBecomeActive { - tenant_id: TenantId, - state: TenantState, - }, - TenantDropped { - tenant_id: TenantId, - }, -} - -impl std::fmt::Display for WaitToBecomeActiveError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - WaitToBecomeActiveError::WillNotBecomeActive { tenant_id, state } => { - write!( - f, - "Tenant {} will not become active. Current state: {:?}", - tenant_id, state - ) - } - WaitToBecomeActiveError::TenantDropped { tenant_id } => { - write!(f, "Tenant {tenant_id} will not become active (dropped)") - } - } - } -} - #[derive(thiserror::Error, Debug)] pub enum CreateTimelineError { #[error("a timeline with the given ID already exists")] @@ -395,6 +394,8 @@ pub enum CreateTimelineError { AncestorLsn(anyhow::Error), #[error("ancestor timeline is not active")] AncestorNotActive, + #[error("tenant shutting down")] + ShuttingDown, #[error(transparent)] Other(#[from] anyhow::Error), } @@ -526,7 +527,7 @@ impl Tenant { resources: TenantSharedResources, attached_conf: AttachedTenantConf, init_order: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { @@ -1524,6 +1525,11 @@ impl Tenant { ))); } + let _gate = self + .gate + .enter() + .map_err(|_| CreateTimelineError::ShuttingDown)?; + if let Ok(existing) = self.get_timeline(new_timeline_id, false) { debug!("timeline {new_timeline_id} already exists"); @@ -1653,22 +1659,16 @@ impl Tenant { /// This function is periodically called by compactor task. /// Also it can be explicitly requested per timeline through page server /// api's 'compact' command. - pub async fn compaction_iteration( + async fn compaction_iteration( &self, cancel: &CancellationToken, ctx: &RequestContext, - ) -> anyhow::Result<()> { - // Don't start doing work during shutdown - if let TenantState::Stopping { .. } = self.current_state() { + ) -> anyhow::Result<(), timeline::CompactionError> { + // Don't start doing work during shutdown, or when broken, we do not need those in the logs + if !self.is_active() { return Ok(()); } - // We should only be called once the tenant has activated. - anyhow::ensure!( - self.is_active(), - "Cannot run compaction iteration on inactive tenant" - ); - { let conf = self.tenant_conf.read().unwrap(); if !conf.location.may_delete_layers_hint() || !conf.location.may_upload_layers_hint() { @@ -1808,6 +1808,7 @@ impl Tenant { freeze_and_flush: bool, ) -> Result<(), completion::Barrier> { span::debug_assert_current_span_has_tenant_id(); + // Set tenant (and its timlines) to Stoppping state. // // Since we can only transition into Stopping state after activation is complete, @@ -1833,6 +1834,7 @@ impl Tenant { } Err(SetStoppingError::AlreadyStopping(other)) => { // give caller the option to wait for this this shutdown + info!("Tenant::shutdown: AlreadyStopping"); return Err(other); } }; @@ -1843,9 +1845,16 @@ impl Tenant { timelines.values().for_each(|timeline| { let timeline = Arc::clone(timeline); let span = Span::current(); - js.spawn(async move { timeline.shutdown(freeze_and_flush).instrument(span).await }); + js.spawn(async move { + if freeze_and_flush { + timeline.flush_and_shutdown().instrument(span).await + } else { + timeline.shutdown().instrument(span).await + } + }); }) }; + tracing::info!("Waiting for timelines..."); while let Some(res) = js.join_next().await { match res { Ok(()) => {} @@ -1855,12 +1864,21 @@ impl Tenant { } } + // We cancel the Tenant's cancellation token _after_ the timelines have all shut down. This permits + // them to continue to do work during their shutdown methods, e.g. flushing data. + tracing::debug!("Cancelling CancellationToken"); + self.cancel.cancel(); + // shutdown all tenant and timeline tasks: gc, compaction, page service // No new tasks will be started for this tenant because it's in `Stopping` state. // // this will additionally shutdown and await all timeline tasks. + tracing::debug!("Waiting for tasks..."); task_mgr::shutdown_tasks(None, Some(self.tenant_id), None).await; + // Wait for any in-flight operations to complete + self.gate.close().await; + Ok(()) } @@ -2021,7 +2039,7 @@ impl Tenant { self.state.subscribe() } - pub(crate) async fn wait_to_become_active(&self) -> Result<(), WaitToBecomeActiveError> { + pub(crate) async fn wait_to_become_active(&self) -> Result<(), GetActiveTenantError> { let mut receiver = self.state.subscribe(); loop { let current_state = receiver.borrow_and_update().clone(); @@ -2029,11 +2047,9 @@ impl Tenant { TenantState::Loading | TenantState::Attaching | TenantState::Activating(_) => { // in these states, there's a chance that we can reach ::Active receiver.changed().await.map_err( - |_e: tokio::sync::watch::error::RecvError| { - WaitToBecomeActiveError::TenantDropped { - tenant_id: self.tenant_id, - } - }, + |_e: tokio::sync::watch::error::RecvError| + // Tenant existed but was dropped: report it as non-existent + GetActiveTenantError::NotFound(GetTenantError::NotFound(self.tenant_id)) )?; } TenantState::Active { .. } => { @@ -2041,10 +2057,7 @@ impl Tenant { } TenantState::Broken { .. } | TenantState::Stopping { .. } => { // There's no chance the tenant can transition back into ::Active - return Err(WaitToBecomeActiveError::WillNotBecomeActive { - tenant_id: self.tenant_id, - state: current_state, - }); + return Err(GetActiveTenantError::WillNotBecomeActive(current_state)); } } } @@ -2110,6 +2123,9 @@ where } impl Tenant { + pub fn get_tenant_id(&self) -> TenantId { + self.tenant_id + } pub fn tenant_specific_overrides(&self) -> TenantConfOpt { self.tenant_conf.read().unwrap().tenant_conf } @@ -2267,6 +2283,7 @@ impl Tenant { initial_logical_size_can_start.cloned(), initial_logical_size_attempt.cloned().flatten(), state, + self.cancel.child_token(), ); Ok(timeline) @@ -2356,6 +2373,8 @@ impl Tenant { cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)), eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()), delete_progress: Arc::new(tokio::sync::Mutex::new(DeleteTenantFlow::default())), + cancel: CancellationToken::default(), + gate: Gate::new(format!("Tenant<{tenant_id}>")), } } @@ -3519,10 +3538,6 @@ pub(crate) mod harness { let remote_fs_dir = conf.workdir.join("localfs"); std::fs::create_dir_all(&remote_fs_dir).unwrap(); let config = RemoteStorageConfig { - // TODO: why not remote_storage::DEFAULT_REMOTE_STORAGE_MAX_CONCURRENT_SYNCS, - max_concurrent_syncs: std::num::NonZeroUsize::new(2_000_000).unwrap(), - // TODO: why not remote_storage::DEFAULT_REMOTE_STORAGE_MAX_SYNC_ERRORS, - max_sync_errors: std::num::NonZeroU32::new(3_000_000).unwrap(), storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), }; let remote_storage = GenericRemoteStorage::from_config(&config).unwrap(); @@ -3692,7 +3707,7 @@ mod tests { use tokio_util::sync::CancellationToken; static TEST_KEY: Lazy = - Lazy::new(|| Key::from_slice(&hex!("112222222233333333444444445500000001"))); + Lazy::new(|| Key::from_slice(&hex!("010000000033333333444444445500000001"))); #[tokio::test] async fn test_basic() -> anyhow::Result<()> { @@ -3788,9 +3803,9 @@ mod tests { let writer = tline.writer().await; #[allow(non_snake_case)] - let TEST_KEY_A: Key = Key::from_hex("112222222233333333444444445500000001").unwrap(); + let TEST_KEY_A: Key = Key::from_hex("110000000033333333444444445500000001").unwrap(); #[allow(non_snake_case)] - let TEST_KEY_B: Key = Key::from_hex("112222222233333333444444445500000002").unwrap(); + let TEST_KEY_B: Key = Key::from_hex("110000000033333333444444445500000002").unwrap(); // Insert a value on the timeline writer @@ -4236,11 +4251,7 @@ mod tests { metadata_bytes[8] ^= 1; std::fs::write(metadata_path, metadata_bytes)?; - let err = harness - .try_load_local(&ctx) - .await - .err() - .expect("should fail"); + let err = harness.try_load_local(&ctx).await.expect_err("should fail"); // get all the stack with all .context, not only the last one let message = format!("{err:#}"); let expected = "failed to load metadata"; @@ -4374,7 +4385,7 @@ mod tests { let mut keyspace = KeySpaceAccum::new(); - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut blknum = 0; for _ in 0..50 { for _ in 0..10000 { @@ -4420,7 +4431,7 @@ mod tests { const NUM_KEYS: usize = 1000; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut keyspace = KeySpaceAccum::new(); @@ -4501,7 +4512,7 @@ mod tests { const NUM_KEYS: usize = 1000; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let mut keyspace = KeySpaceAccum::new(); @@ -4592,7 +4603,7 @@ mod tests { const NUM_KEYS: usize = 100; const NUM_TLINES: usize = 50; - let mut test_key = Key::from_hex("012222222233333333444444445500000000").unwrap(); + let mut test_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); // Track page mutation lsns across different timelines. let mut updated = [[Lsn(0); NUM_KEYS]; NUM_TLINES]; @@ -4726,7 +4737,7 @@ mod tests { // Keeps uninit mark in place let raw_tline = tline.raw_timeline().unwrap(); raw_tline - .shutdown(false) + .shutdown() .instrument(info_span!("test_shutdown", tenant_id=%raw_tline.tenant_id)) .await; std::mem::forget(tline); diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index bedf09a40c..6de2e95055 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -327,7 +327,7 @@ mod tests { let mut sz: u16 = rng.gen(); // Make 50% of the arrays small if rng.gen() { - sz |= 63; + sz &= 63; } random_array(sz.into()) }) diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 87b48e4bee..066f239ff0 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -21,7 +21,7 @@ use crate::{ }; use super::{ - mgr::{GetTenantError, TenantsMap}, + mgr::{GetTenantError, TenantSlotError, TenantSlotUpsertError, TenantsMap}, remote_timeline_client::{FAILED_REMOTE_OP_RETRIES, FAILED_UPLOAD_WARN_THRESHOLD}, span, timeline::delete::DeleteTimelineFlow, @@ -33,12 +33,21 @@ pub(crate) enum DeleteTenantError { #[error("GetTenant {0}")] Get(#[from] GetTenantError), + #[error("Tenant not attached")] + NotAttached, + #[error("Invalid state {0}. Expected Active or Broken")] InvalidState(TenantState), #[error("Tenant deletion is already in progress")] AlreadyInProgress, + #[error("Tenant map slot error {0}")] + SlotError(#[from] TenantSlotError), + + #[error("Tenant map slot upsert error {0}")] + SlotUpsertError(#[from] TenantSlotUpsertError), + #[error("Timeline {0}")] Timeline(#[from] DeleteTimelineError), @@ -273,12 +282,12 @@ impl DeleteTenantFlow { pub(crate) async fn run( conf: &'static PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, - tenant_id: TenantId, + tenants: &'static std::sync::RwLock, + tenant: Arc, ) -> Result<(), DeleteTenantError> { span::debug_assert_current_span_has_tenant_id(); - let (tenant, mut guard) = Self::prepare(tenants, tenant_id).await?; + let mut guard = Self::prepare(&tenant).await?; if let Err(e) = Self::run_inner(&mut guard, conf, remote_storage.as_ref(), &tenant).await { tenant.set_broken(format!("{e:#}")).await; @@ -378,7 +387,7 @@ impl DeleteTenantFlow { guard: DeletionGuard, tenant: &Arc, preload: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, init_order: Option, ctx: &RequestContext, ) -> Result<(), DeleteTenantError> { @@ -405,15 +414,8 @@ impl DeleteTenantFlow { } async fn prepare( - tenants: &tokio::sync::RwLock, - tenant_id: TenantId, - ) -> Result<(Arc, tokio::sync::OwnedMutexGuard), DeleteTenantError> { - let m = tenants.read().await; - - let tenant = m - .get(&tenant_id) - .ok_or(GetTenantError::NotFound(tenant_id))?; - + tenant: &Arc, + ) -> Result, DeleteTenantError> { // FIXME: unsure about active only. Our init jobs may not be cancellable properly, // so at least for now allow deletions only for active tenants. TODO recheck // Broken and Stopping is needed for retries. @@ -447,14 +449,14 @@ impl DeleteTenantFlow { ))); } - Ok((Arc::clone(tenant), guard)) + Ok(guard) } fn schedule_background( guard: OwnedMutexGuard, conf: &'static PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, tenant: Arc, ) { let tenant_id = tenant.tenant_id; @@ -487,7 +489,7 @@ impl DeleteTenantFlow { mut guard: OwnedMutexGuard, conf: &PageServerConf, remote_storage: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, tenant: &Arc, ) -> Result<(), DeleteTenantError> { // Tree sort timelines, schedule delete for them. Mention retries from the console side. @@ -535,10 +537,18 @@ impl DeleteTenantFlow { .await .context("cleanup_remaining_fs_traces")?; - let mut locked = tenants.write().await; - if locked.remove(&tenant.tenant_id).is_none() { - warn!("Tenant got removed from tenants map during deletion"); - }; + { + let mut locked = tenants.write().unwrap(); + if locked.remove(&tenant.tenant_id).is_none() { + warn!("Tenant got removed from tenants map during deletion"); + }; + + // FIXME: we should not be modifying this from outside of mgr.rs. + // This will go away when we simplify deletion (https://github.com/neondatabase/neon/issues/5080) + crate::metrics::TENANT_MANAGER + .tenant_slots + .set(locked.len() as u64); + } *guard = Self::Finished; diff --git a/pageserver/src/tenant/disk_btree_test_data.rs b/pageserver/src/tenant/disk_btree_test_data.rs index 9462573f03..4f835f55af 100644 --- a/pageserver/src/tenant/disk_btree_test_data.rs +++ b/pageserver/src/tenant/disk_btree_test_data.rs @@ -10,2004 +10,2004 @@ use hex_literal::hex; /// by a delta layer, for evaluating how well the prefix compression works. #[rustfmt::skip] pub static TEST_DATA: [([u8; 26], u64); 2000] = [ - (hex!("0122222222333333334444444455000000000000000000000010"), 0x004001), - (hex!("0122222222333333334444444455000000000000000000007cb0"), 0x0040a1), - (hex!("0122222222333333334444444455000000010000000000000020"), 0x004141), - (hex!("0122222222333333334444444455000000020000000000000030"), 0x0041e1), - (hex!("01222222223333333344444444550000000200000000000051a0"), 0x004281), - (hex!("0122222222333333334444444455000000030000000000000040"), 0x004321), - (hex!("0122222222333333334444444455000000030000000000006cf0"), 0x0043c1), - (hex!("0122222222333333334444444455000000030000000000007140"), 0x004461), - (hex!("0122222222333333334444444455000000040000000000000050"), 0x004501), - (hex!("01222222223333333344444444550000000400000000000047f0"), 0x0045a1), - (hex!("01222222223333333344444444550000000400000000000072b0"), 0x004641), - (hex!("0122222222333333334444444455000000050000000000000060"), 0x0046e1), - (hex!("0122222222333333334444444455000000050000000000005550"), 0x004781), - (hex!("0122222222333333334444444455000000060000000000000070"), 0x004821), - (hex!("01222222223333333344444444550000000600000000000044a0"), 0x0048c1), - (hex!("0122222222333333334444444455000000060000000000006870"), 0x004961), - (hex!("0122222222333333334444444455000000070000000000000080"), 0x004a01), - (hex!("0122222222333333334444444455000000080000000000000090"), 0x004aa1), - (hex!("0122222222333333334444444455000000080000000000004150"), 0x004b41), - (hex!("01222222223333333344444444550000000900000000000000a0"), 0x004be1), - (hex!("01222222223333333344444444550000000a00000000000000b0"), 0x004c81), - (hex!("01222222223333333344444444550000000a0000000000006680"), 0x004d21), - (hex!("01222222223333333344444444550000000b00000000000000c0"), 0x004dc1), - (hex!("01222222223333333344444444550000000b0000000000006230"), 0x004e61), - (hex!("01222222223333333344444444550000000c00000000000000d0"), 0x004f01), - (hex!("01222222223333333344444444550000000d00000000000000e0"), 0x004fa1), - (hex!("01222222223333333344444444550000000e00000000000000f0"), 0x005041), - (hex!("01222222223333333344444444550000000e0000000000006000"), 0x0050e1), - (hex!("01222222223333333344444444550000000f0000000000000100"), 0x005181), - (hex!("01222222223333333344444444550000000f00000000000053c0"), 0x005221), - (hex!("01222222223333333344444444550000000f0000000000006580"), 0x0052c1), - (hex!("0122222222333333334444444455000000100000000000000110"), 0x005361), - (hex!("01222222223333333344444444550000001000000000000046c0"), 0x005401), - (hex!("0122222222333333334444444455000000100000000000004e40"), 0x0054a1), - (hex!("0122222222333333334444444455000000110000000000000120"), 0x005541), - (hex!("0122222222333333334444444455000000120000000000000130"), 0x0055e1), - (hex!("01222222223333333344444444550000001200000000000066d0"), 0x005681), - (hex!("0122222222333333334444444455000000130000000000000140"), 0x005721), - (hex!("0122222222333333334444444455000000130000000000007710"), 0x0057c1), - (hex!("0122222222333333334444444455000000140000000000000150"), 0x005861), - (hex!("0122222222333333334444444455000000140000000000006c40"), 0x005901), - (hex!("0122222222333333334444444455000000150000000000000160"), 0x0059a1), - (hex!("0122222222333333334444444455000000150000000000005990"), 0x005a41), - (hex!("0122222222333333334444444455000000160000000000000170"), 0x005ae1), - (hex!("0122222222333333334444444455000000160000000000005530"), 0x005b81), - (hex!("0122222222333333334444444455000000170000000000000180"), 0x005c21), - (hex!("0122222222333333334444444455000000170000000000004290"), 0x005cc1), - (hex!("0122222222333333334444444455000000180000000000000190"), 0x005d61), - (hex!("01222222223333333344444444550000001800000000000051c0"), 0x005e01), - (hex!("01222222223333333344444444550000001900000000000001a0"), 0x005ea1), - (hex!("0122222222333333334444444455000000190000000000005420"), 0x005f41), - (hex!("0122222222333333334444444455000000190000000000005770"), 0x005fe1), - (hex!("01222222223333333344444444550000001900000000000079d0"), 0x006081), - (hex!("01222222223333333344444444550000001a00000000000001b0"), 0x006121), - (hex!("01222222223333333344444444550000001a0000000000006f70"), 0x0061c1), - (hex!("01222222223333333344444444550000001a0000000000007150"), 0x006261), - (hex!("01222222223333333344444444550000001b00000000000001c0"), 0x006301), - (hex!("01222222223333333344444444550000001b0000000000005070"), 0x0063a1), - (hex!("01222222223333333344444444550000001c00000000000001d0"), 0x006441), - (hex!("01222222223333333344444444550000001d00000000000001e0"), 0x0064e1), - (hex!("01222222223333333344444444550000001e00000000000001f0"), 0x006581), - (hex!("01222222223333333344444444550000001e0000000000005650"), 0x006621), - (hex!("01222222223333333344444444550000001f0000000000000200"), 0x0066c1), - (hex!("01222222223333333344444444550000001f0000000000006ca0"), 0x006761), - (hex!("0122222222333333334444444455000000200000000000000210"), 0x006801), - (hex!("0122222222333333334444444455000000200000000000005fc0"), 0x0068a1), - (hex!("0122222222333333334444444455000000210000000000000220"), 0x006941), - (hex!("0122222222333333334444444455000000210000000000006430"), 0x0069e1), - (hex!("0122222222333333334444444455000000220000000000000230"), 0x006a81), - (hex!("01222222223333333344444444550000002200000000000040e0"), 0x006b21), - (hex!("0122222222333333334444444455000000230000000000000240"), 0x006bc1), - (hex!("01222222223333333344444444550000002300000000000042d0"), 0x006c61), - (hex!("0122222222333333334444444455000000240000000000000250"), 0x006d01), - (hex!("0122222222333333334444444455000000250000000000000260"), 0x006da1), - (hex!("01222222223333333344444444550000002500000000000058c0"), 0x006e41), - (hex!("0122222222333333334444444455000000260000000000000270"), 0x006ee1), - (hex!("0122222222333333334444444455000000260000000000004020"), 0x006f81), - (hex!("0122222222333333334444444455000000270000000000000280"), 0x007021), - (hex!("0122222222333333334444444455000000280000000000000290"), 0x0070c1), - (hex!("0122222222333333334444444455000000280000000000007c00"), 0x007161), - (hex!("01222222223333333344444444550000002900000000000002a0"), 0x007201), - (hex!("01222222223333333344444444550000002a00000000000002b0"), 0x0072a1), - (hex!("01222222223333333344444444550000002b00000000000002c0"), 0x007341), - (hex!("01222222223333333344444444550000002c00000000000002d0"), 0x0073e1), - (hex!("01222222223333333344444444550000002c00000000000041b0"), 0x007481), - (hex!("01222222223333333344444444550000002c0000000000004c30"), 0x007521), - (hex!("01222222223333333344444444550000002d00000000000002e0"), 0x0075c1), - (hex!("01222222223333333344444444550000002d0000000000005e40"), 0x007661), - (hex!("01222222223333333344444444550000002d0000000000006990"), 0x007701), - (hex!("01222222223333333344444444550000002e00000000000002f0"), 0x0077a1), - (hex!("01222222223333333344444444550000002f0000000000000300"), 0x007841), - (hex!("01222222223333333344444444550000002f0000000000004a70"), 0x0078e1), - (hex!("01222222223333333344444444550000002f0000000000006b40"), 0x007981), - (hex!("0122222222333333334444444455000000300000000000000310"), 0x007a21), - (hex!("0122222222333333334444444455000000310000000000000320"), 0x007ac1), - (hex!("0122222222333333334444444455000000320000000000000330"), 0x007b61), - (hex!("01222222223333333344444444550000003200000000000041a0"), 0x007c01), - (hex!("0122222222333333334444444455000000320000000000007340"), 0x007ca1), - (hex!("0122222222333333334444444455000000320000000000007730"), 0x007d41), - (hex!("0122222222333333334444444455000000330000000000000340"), 0x007de1), - (hex!("01222222223333333344444444550000003300000000000055a0"), 0x007e81), - (hex!("0122222222333333334444444455000000340000000000000350"), 0x007f21), - (hex!("0122222222333333334444444455000000350000000000000360"), 0x007fc1), - (hex!("01222222223333333344444444550000003500000000000077a0"), 0x008061), - (hex!("0122222222333333334444444455000000360000000000000370"), 0x008101), - (hex!("0122222222333333334444444455000000370000000000000380"), 0x0081a1), - (hex!("0122222222333333334444444455000000380000000000000390"), 0x008241), - (hex!("01222222223333333344444444550000003900000000000003a0"), 0x0082e1), - (hex!("01222222223333333344444444550000003a00000000000003b0"), 0x008381), - (hex!("01222222223333333344444444550000003a00000000000071c0"), 0x008421), - (hex!("01222222223333333344444444550000003b00000000000003c0"), 0x0084c1), - (hex!("01222222223333333344444444550000003c00000000000003d0"), 0x008561), - (hex!("01222222223333333344444444550000003d00000000000003e0"), 0x008601), - (hex!("01222222223333333344444444550000003e00000000000003f0"), 0x0086a1), - (hex!("01222222223333333344444444550000003e00000000000062e0"), 0x008741), - (hex!("01222222223333333344444444550000003f0000000000000400"), 0x0087e1), - (hex!("0122222222333333334444444455000000400000000000000410"), 0x008881), - (hex!("0122222222333333334444444455000000400000000000004460"), 0x008921), - (hex!("0122222222333333334444444455000000400000000000005b90"), 0x0089c1), - (hex!("01222222223333333344444444550000004000000000000079b0"), 0x008a61), - (hex!("0122222222333333334444444455000000410000000000000420"), 0x008b01), - (hex!("0122222222333333334444444455000000420000000000000430"), 0x008ba1), - (hex!("0122222222333333334444444455000000420000000000005640"), 0x008c41), - (hex!("0122222222333333334444444455000000430000000000000440"), 0x008ce1), - (hex!("01222222223333333344444444550000004300000000000072a0"), 0x008d81), - (hex!("0122222222333333334444444455000000440000000000000450"), 0x008e21), - (hex!("0122222222333333334444444455000000450000000000000460"), 0x008ec1), - (hex!("0122222222333333334444444455000000450000000000005750"), 0x008f61), - (hex!("01222222223333333344444444550000004500000000000077b0"), 0x009001), - (hex!("0122222222333333334444444455000000460000000000000470"), 0x0090a1), - (hex!("0122222222333333334444444455000000470000000000000480"), 0x009141), - (hex!("0122222222333333334444444455000000480000000000000490"), 0x0091e1), - (hex!("01222222223333333344444444550000004800000000000069e0"), 0x009281), - (hex!("01222222223333333344444444550000004900000000000004a0"), 0x009321), - (hex!("0122222222333333334444444455000000490000000000007370"), 0x0093c1), - (hex!("01222222223333333344444444550000004a00000000000004b0"), 0x009461), - (hex!("01222222223333333344444444550000004a0000000000005cb0"), 0x009501), - (hex!("01222222223333333344444444550000004b00000000000004c0"), 0x0095a1), - (hex!("01222222223333333344444444550000004c00000000000004d0"), 0x009641), - (hex!("01222222223333333344444444550000004c0000000000004880"), 0x0096e1), - (hex!("01222222223333333344444444550000004c0000000000007a40"), 0x009781), - (hex!("01222222223333333344444444550000004d00000000000004e0"), 0x009821), - (hex!("01222222223333333344444444550000004d0000000000006390"), 0x0098c1), - (hex!("01222222223333333344444444550000004e00000000000004f0"), 0x009961), - (hex!("01222222223333333344444444550000004e0000000000004db0"), 0x009a01), - (hex!("01222222223333333344444444550000004f0000000000000500"), 0x009aa1), - (hex!("0122222222333333334444444455000000500000000000000510"), 0x009b41), - (hex!("0122222222333333334444444455000000510000000000000520"), 0x009be1), - (hex!("01222222223333333344444444550000005100000000000069c0"), 0x009c81), - (hex!("0122222222333333334444444455000000520000000000000530"), 0x009d21), - (hex!("0122222222333333334444444455000000520000000000006e60"), 0x009dc1), - (hex!("01222222223333333344444444550000005200000000000070c0"), 0x009e61), - (hex!("0122222222333333334444444455000000530000000000000540"), 0x009f01), - (hex!("0122222222333333334444444455000000530000000000005840"), 0x009fa1), - (hex!("0122222222333333334444444455000000540000000000000550"), 0x00a041), - (hex!("01222222223333333344444444550000005400000000000043e0"), 0x00a0e1), - (hex!("01222222223333333344444444550000005400000000000074e0"), 0x00a181), - (hex!("0122222222333333334444444455000000550000000000000560"), 0x00a221), - (hex!("0122222222333333334444444455000000550000000000003ee0"), 0x00a2c1), - (hex!("0122222222333333334444444455000000560000000000000570"), 0x00a361), - (hex!("0122222222333333334444444455000000570000000000000580"), 0x00a401), - (hex!("0122222222333333334444444455000000570000000000007030"), 0x00a4a1), - (hex!("0122222222333333334444444455000000580000000000000590"), 0x00a541), - (hex!("0122222222333333334444444455000000580000000000005340"), 0x00a5e1), - (hex!("01222222223333333344444444550000005800000000000059f0"), 0x00a681), - (hex!("0122222222333333334444444455000000580000000000006930"), 0x00a721), - (hex!("01222222223333333344444444550000005900000000000005a0"), 0x00a7c1), - (hex!("0122222222333333334444444455000000590000000000003f90"), 0x00a861), - (hex!("01222222223333333344444444550000005a00000000000005b0"), 0x00a901), - (hex!("01222222223333333344444444550000005b00000000000005c0"), 0x00a9a1), - (hex!("01222222223333333344444444550000005b00000000000062c0"), 0x00aa41), - (hex!("01222222223333333344444444550000005c00000000000005d0"), 0x00aae1), - (hex!("01222222223333333344444444550000005c0000000000005a70"), 0x00ab81), - (hex!("01222222223333333344444444550000005c0000000000005dd0"), 0x00ac21), - (hex!("01222222223333333344444444550000005d00000000000005e0"), 0x00acc1), - (hex!("01222222223333333344444444550000005d0000000000005730"), 0x00ad61), - (hex!("01222222223333333344444444550000005e00000000000005f0"), 0x00ae01), - (hex!("01222222223333333344444444550000005e0000000000004f40"), 0x00aea1), - (hex!("01222222223333333344444444550000005f0000000000000600"), 0x00af41), - (hex!("0122222222333333334444444455000000600000000000000610"), 0x00afe1), - (hex!("0122222222333333334444444455000000600000000000007c40"), 0x00b081), - (hex!("0122222222333333334444444455000000610000000000000620"), 0x00b121), - (hex!("0122222222333333334444444455000000610000000000007860"), 0x00b1c1), - (hex!("0122222222333333334444444455000000620000000000000630"), 0x00b261), - (hex!("0122222222333333334444444455000000620000000000005050"), 0x00b301), - (hex!("0122222222333333334444444455000000630000000000000640"), 0x00b3a1), - (hex!("0122222222333333334444444455000000640000000000000650"), 0x00b441), - (hex!("0122222222333333334444444455000000650000000000000660"), 0x00b4e1), - (hex!("0122222222333333334444444455000000650000000000005330"), 0x00b581), - (hex!("0122222222333333334444444455000000660000000000000670"), 0x00b621), - (hex!("0122222222333333334444444455000000660000000000004e20"), 0x00b6c1), - (hex!("0122222222333333334444444455000000660000000000005ee0"), 0x00b761), - (hex!("0122222222333333334444444455000000660000000000006360"), 0x00b801), - (hex!("0122222222333333334444444455000000670000000000000680"), 0x00b8a1), - (hex!("0122222222333333334444444455000000670000000000004040"), 0x00b941), - (hex!("0122222222333333334444444455000000680000000000000690"), 0x00b9e1), - (hex!("0122222222333333334444444455000000680000000000003f80"), 0x00ba81), - (hex!("01222222223333333344444444550000006800000000000041e0"), 0x00bb21), - (hex!("01222222223333333344444444550000006900000000000006a0"), 0x00bbc1), - (hex!("0122222222333333334444444455000000690000000000006080"), 0x00bc61), - (hex!("01222222223333333344444444550000006a00000000000006b0"), 0x00bd01), - (hex!("01222222223333333344444444550000006a00000000000042f0"), 0x00bda1), - (hex!("01222222223333333344444444550000006b00000000000006c0"), 0x00be41), - (hex!("01222222223333333344444444550000006b00000000000052f0"), 0x00bee1), - (hex!("01222222223333333344444444550000006b0000000000005980"), 0x00bf81), - (hex!("01222222223333333344444444550000006b0000000000006170"), 0x00c021), - (hex!("01222222223333333344444444550000006c00000000000006d0"), 0x00c0c1), - (hex!("01222222223333333344444444550000006d00000000000006e0"), 0x00c161), - (hex!("01222222223333333344444444550000006d0000000000006fb0"), 0x00c201), - (hex!("01222222223333333344444444550000006e00000000000006f0"), 0x00c2a1), - (hex!("01222222223333333344444444550000006e00000000000065b0"), 0x00c341), - (hex!("01222222223333333344444444550000006e0000000000007970"), 0x00c3e1), - (hex!("01222222223333333344444444550000006f0000000000000700"), 0x00c481), - (hex!("01222222223333333344444444550000006f0000000000005900"), 0x00c521), - (hex!("01222222223333333344444444550000006f0000000000006d90"), 0x00c5c1), - (hex!("0122222222333333334444444455000000700000000000000710"), 0x00c661), - (hex!("01222222223333333344444444550000007000000000000045c0"), 0x00c701), - (hex!("0122222222333333334444444455000000700000000000004d40"), 0x00c7a1), - (hex!("0122222222333333334444444455000000710000000000000720"), 0x00c841), - (hex!("0122222222333333334444444455000000710000000000004dc0"), 0x00c8e1), - (hex!("0122222222333333334444444455000000710000000000007550"), 0x00c981), - (hex!("0122222222333333334444444455000000720000000000000730"), 0x00ca21), - (hex!("0122222222333333334444444455000000720000000000003ec0"), 0x00cac1), - (hex!("01222222223333333344444444550000007200000000000045a0"), 0x00cb61), - (hex!("0122222222333333334444444455000000720000000000006770"), 0x00cc01), - (hex!("0122222222333333334444444455000000720000000000006bc0"), 0x00cca1), - (hex!("0122222222333333334444444455000000730000000000000740"), 0x00cd41), - (hex!("0122222222333333334444444455000000730000000000005250"), 0x00cde1), - (hex!("01222222223333333344444444550000007300000000000075f0"), 0x00ce81), - (hex!("0122222222333333334444444455000000740000000000000750"), 0x00cf21), - (hex!("0122222222333333334444444455000000740000000000003ff0"), 0x00cfc1), - (hex!("01222222223333333344444444550000007400000000000079e0"), 0x00d061), - (hex!("0122222222333333334444444455000000750000000000000760"), 0x00d101), - (hex!("0122222222333333334444444455000000750000000000004310"), 0x00d1a1), - (hex!("0122222222333333334444444455000000760000000000000770"), 0x00d241), - (hex!("0122222222333333334444444455000000770000000000000780"), 0x00d2e1), - (hex!("01222222223333333344444444550000007700000000000062f0"), 0x00d381), - (hex!("0122222222333333334444444455000000770000000000006940"), 0x00d421), - (hex!("0122222222333333334444444455000000780000000000000790"), 0x00d4c1), - (hex!("01222222223333333344444444550000007900000000000007a0"), 0x00d561), - (hex!("0122222222333333334444444455000000790000000000007af0"), 0x00d601), - (hex!("01222222223333333344444444550000007a00000000000007b0"), 0x00d6a1), - (hex!("01222222223333333344444444550000007b00000000000007c0"), 0x00d741), - (hex!("01222222223333333344444444550000007b00000000000067e0"), 0x00d7e1), - (hex!("01222222223333333344444444550000007b0000000000007890"), 0x00d881), - (hex!("01222222223333333344444444550000007c00000000000007d0"), 0x00d921), - (hex!("01222222223333333344444444550000007d00000000000007e0"), 0x00d9c1), - (hex!("01222222223333333344444444550000007e00000000000007f0"), 0x00da61), - (hex!("01222222223333333344444444550000007f0000000000000800"), 0x00db01), - (hex!("01222222223333333344444444550000007f0000000000005be0"), 0x00dba1), - (hex!("0122222222333333334444444455000000800000000000000810"), 0x00dc41), - (hex!("0122222222333333334444444455000000810000000000000820"), 0x00dce1), - (hex!("0122222222333333334444444455000000810000000000007190"), 0x00dd81), - (hex!("0122222222333333334444444455000000820000000000000830"), 0x00de21), - (hex!("0122222222333333334444444455000000820000000000004ab0"), 0x00dec1), - (hex!("0122222222333333334444444455000000830000000000000840"), 0x00df61), - (hex!("0122222222333333334444444455000000830000000000006720"), 0x00e001), - (hex!("0122222222333333334444444455000000840000000000000850"), 0x00e0a1), - (hex!("0122222222333333334444444455000000850000000000000860"), 0x00e141), - (hex!("01222222223333333344444444550000008500000000000054f0"), 0x00e1e1), - (hex!("0122222222333333334444444455000000850000000000007920"), 0x00e281), - (hex!("0122222222333333334444444455000000860000000000000870"), 0x00e321), - (hex!("01222222223333333344444444550000008600000000000060e0"), 0x00e3c1), - (hex!("0122222222333333334444444455000000860000000000006be0"), 0x00e461), - (hex!("0122222222333333334444444455000000870000000000000880"), 0x00e501), - (hex!("0122222222333333334444444455000000870000000000006820"), 0x00e5a1), - (hex!("0122222222333333334444444455000000880000000000000890"), 0x00e641), - (hex!("01222222223333333344444444550000008900000000000008a0"), 0x00e6e1), - (hex!("0122222222333333334444444455000000890000000000007c30"), 0x00e781), - (hex!("01222222223333333344444444550000008a00000000000008b0"), 0x00e821), - (hex!("01222222223333333344444444550000008b00000000000008c0"), 0x00e8c1), - (hex!("01222222223333333344444444550000008b0000000000005910"), 0x00e961), - (hex!("01222222223333333344444444550000008b0000000000006fe0"), 0x00ea01), - (hex!("01222222223333333344444444550000008c00000000000008d0"), 0x00eaa1), - (hex!("01222222223333333344444444550000008c0000000000006800"), 0x00eb41), - (hex!("01222222223333333344444444550000008d00000000000008e0"), 0x00ebe1), - (hex!("01222222223333333344444444550000008d0000000000005810"), 0x00ec81), - (hex!("01222222223333333344444444550000008d0000000000007c90"), 0x00ed21), - (hex!("01222222223333333344444444550000008e00000000000008f0"), 0x00edc1), - (hex!("01222222223333333344444444550000008e00000000000058f0"), 0x00ee61), - (hex!("01222222223333333344444444550000008f0000000000000900"), 0x00ef01), - (hex!("01222222223333333344444444550000008f0000000000005a30"), 0x00efa1), - (hex!("0122222222333333334444444455000000900000000000000910"), 0x00f041), - (hex!("0122222222333333334444444455000000900000000000006130"), 0x00f0e1), - (hex!("0122222222333333334444444455000000900000000000006550"), 0x00f181), - (hex!("0122222222333333334444444455000000910000000000000920"), 0x00f221), - (hex!("01222222223333333344444444550000009100000000000079f0"), 0x00f2c1), - (hex!("0122222222333333334444444455000000920000000000000930"), 0x00f361), - (hex!("0122222222333333334444444455000000920000000000005620"), 0x00f401), - (hex!("0122222222333333334444444455000000920000000000005e90"), 0x00f4a1), - (hex!("01222222223333333344444444550000009200000000000063d0"), 0x00f541), - (hex!("01222222223333333344444444550000009200000000000076c0"), 0x00f5e1), - (hex!("0122222222333333334444444455000000930000000000000940"), 0x00f681), - (hex!("01222222223333333344444444550000009300000000000044e0"), 0x00f721), - (hex!("0122222222333333334444444455000000940000000000000950"), 0x00f7c1), - (hex!("0122222222333333334444444455000000940000000000007a30"), 0x00f861), - (hex!("0122222222333333334444444455000000950000000000000960"), 0x00f901), - (hex!("0122222222333333334444444455000000950000000000007a70"), 0x00f9a1), - (hex!("0122222222333333334444444455000000960000000000000970"), 0x00fa41), - (hex!("0122222222333333334444444455000000970000000000000980"), 0x00fae1), - (hex!("0122222222333333334444444455000000970000000000007330"), 0x00fb81), - (hex!("0122222222333333334444444455000000980000000000000990"), 0x00fc21), - (hex!("0122222222333333334444444455000000980000000000005af0"), 0x00fcc1), - (hex!("0122222222333333334444444455000000980000000000007ae0"), 0x00fd61), - (hex!("01222222223333333344444444550000009900000000000009a0"), 0x00fe01), - (hex!("0122222222333333334444444455000000990000000000005160"), 0x00fea1), - (hex!("0122222222333333334444444455000000990000000000006850"), 0x00ff41), - (hex!("01222222223333333344444444550000009a00000000000009b0"), 0x00ffe1), - (hex!("01222222223333333344444444550000009b00000000000009c0"), 0x010081), - (hex!("01222222223333333344444444550000009b0000000000005010"), 0x010121), - (hex!("01222222223333333344444444550000009c00000000000009d0"), 0x0101c1), - (hex!("01222222223333333344444444550000009c00000000000042e0"), 0x010261), - (hex!("01222222223333333344444444550000009d00000000000009e0"), 0x010301), - (hex!("01222222223333333344444444550000009d00000000000057f0"), 0x0103a1), - (hex!("01222222223333333344444444550000009e00000000000009f0"), 0x010441), - (hex!("01222222223333333344444444550000009e0000000000004ef0"), 0x0104e1), - (hex!("01222222223333333344444444550000009f0000000000000a00"), 0x010581), - (hex!("01222222223333333344444444550000009f0000000000006110"), 0x010621), - (hex!("0122222222333333334444444455000000a00000000000000a10"), 0x0106c1), - (hex!("0122222222333333334444444455000000a10000000000000a20"), 0x010761), - (hex!("0122222222333333334444444455000000a100000000000040d0"), 0x010801), - (hex!("0122222222333333334444444455000000a10000000000007670"), 0x0108a1), - (hex!("0122222222333333334444444455000000a20000000000000a30"), 0x010941), - (hex!("0122222222333333334444444455000000a200000000000074d0"), 0x0109e1), - (hex!("0122222222333333334444444455000000a30000000000000a40"), 0x010a81), - (hex!("0122222222333333334444444455000000a30000000000004c90"), 0x010b21), - (hex!("0122222222333333334444444455000000a40000000000000a50"), 0x010bc1), - (hex!("0122222222333333334444444455000000a50000000000000a60"), 0x010c61), - (hex!("0122222222333333334444444455000000a60000000000000a70"), 0x010d01), - (hex!("0122222222333333334444444455000000a60000000000006d80"), 0x010da1), - (hex!("0122222222333333334444444455000000a60000000000007830"), 0x010e41), - (hex!("0122222222333333334444444455000000a70000000000000a80"), 0x010ee1), - (hex!("0122222222333333334444444455000000a700000000000064f0"), 0x010f81), - (hex!("0122222222333333334444444455000000a80000000000000a90"), 0x011021), - (hex!("0122222222333333334444444455000000a90000000000000aa0"), 0x0110c1), - (hex!("0122222222333333334444444455000000a90000000000005e30"), 0x011161), - (hex!("0122222222333333334444444455000000aa0000000000000ab0"), 0x011201), - (hex!("0122222222333333334444444455000000ab0000000000000ac0"), 0x0112a1), - (hex!("0122222222333333334444444455000000ac0000000000000ad0"), 0x011341), - (hex!("0122222222333333334444444455000000ac0000000000006d20"), 0x0113e1), - (hex!("0122222222333333334444444455000000ac0000000000007000"), 0x011481), - (hex!("0122222222333333334444444455000000ad0000000000000ae0"), 0x011521), - (hex!("0122222222333333334444444455000000ae0000000000000af0"), 0x0115c1), - (hex!("0122222222333333334444444455000000ae0000000000004a10"), 0x011661), - (hex!("0122222222333333334444444455000000af0000000000000b00"), 0x011701), - (hex!("0122222222333333334444444455000000af0000000000004e10"), 0x0117a1), - (hex!("0122222222333333334444444455000000b00000000000000b10"), 0x011841), - (hex!("0122222222333333334444444455000000b00000000000004280"), 0x0118e1), - (hex!("0122222222333333334444444455000000b000000000000077e0"), 0x011981), - (hex!("0122222222333333334444444455000000b10000000000000b20"), 0x011a21), - (hex!("0122222222333333334444444455000000b20000000000000b30"), 0x011ac1), - (hex!("0122222222333333334444444455000000b30000000000000b40"), 0x011b61), - (hex!("0122222222333333334444444455000000b30000000000004bc0"), 0x011c01), - (hex!("0122222222333333334444444455000000b40000000000000b50"), 0x011ca1), - (hex!("0122222222333333334444444455000000b50000000000000b60"), 0x011d41), - (hex!("0122222222333333334444444455000000b50000000000004fa0"), 0x011de1), - (hex!("0122222222333333334444444455000000b50000000000006a60"), 0x011e81), - (hex!("0122222222333333334444444455000000b60000000000000b70"), 0x011f21), - (hex!("0122222222333333334444444455000000b60000000000005630"), 0x011fc1), - (hex!("0122222222333333334444444455000000b70000000000000b80"), 0x012061), - (hex!("0122222222333333334444444455000000b80000000000000b90"), 0x012101), - (hex!("0122222222333333334444444455000000b80000000000006f80"), 0x0121a1), - (hex!("0122222222333333334444444455000000b90000000000000ba0"), 0x012241), - (hex!("0122222222333333334444444455000000ba0000000000000bb0"), 0x0122e1), - (hex!("0122222222333333334444444455000000bb0000000000000bc0"), 0x012381), - (hex!("0122222222333333334444444455000000bb00000000000047c0"), 0x012421), - (hex!("0122222222333333334444444455000000bb0000000000006060"), 0x0124c1), - (hex!("0122222222333333334444444455000000bc0000000000000bd0"), 0x012561), - (hex!("0122222222333333334444444455000000bd0000000000000be0"), 0x012601), - (hex!("0122222222333333334444444455000000bd0000000000004e80"), 0x0126a1), - (hex!("0122222222333333334444444455000000be0000000000000bf0"), 0x012741), - (hex!("0122222222333333334444444455000000bf0000000000000c00"), 0x0127e1), - (hex!("0122222222333333334444444455000000bf00000000000047a0"), 0x012881), - (hex!("0122222222333333334444444455000000bf0000000000006da0"), 0x012921), - (hex!("0122222222333333334444444455000000c00000000000000c10"), 0x0129c1), - (hex!("0122222222333333334444444455000000c10000000000000c20"), 0x012a61), - (hex!("0122222222333333334444444455000000c20000000000000c30"), 0x012b01), - (hex!("0122222222333333334444444455000000c20000000000004bd0"), 0x012ba1), - (hex!("0122222222333333334444444455000000c20000000000006ac0"), 0x012c41), - (hex!("0122222222333333334444444455000000c30000000000000c40"), 0x012ce1), - (hex!("0122222222333333334444444455000000c30000000000004660"), 0x012d81), - (hex!("0122222222333333334444444455000000c40000000000000c50"), 0x012e21), - (hex!("0122222222333333334444444455000000c50000000000000c60"), 0x012ec1), - (hex!("0122222222333333334444444455000000c60000000000000c70"), 0x012f61), - (hex!("0122222222333333334444444455000000c60000000000005880"), 0x013001), - (hex!("0122222222333333334444444455000000c60000000000006b70"), 0x0130a1), - (hex!("0122222222333333334444444455000000c70000000000000c80"), 0x013141), - (hex!("0122222222333333334444444455000000c80000000000000c90"), 0x0131e1), - (hex!("0122222222333333334444444455000000c80000000000005310"), 0x013281), - (hex!("0122222222333333334444444455000000c80000000000005db0"), 0x013321), - (hex!("0122222222333333334444444455000000c80000000000007040"), 0x0133c1), - (hex!("0122222222333333334444444455000000c80000000000007290"), 0x013461), - (hex!("0122222222333333334444444455000000c90000000000000ca0"), 0x013501), - (hex!("0122222222333333334444444455000000c90000000000004fe0"), 0x0135a1), - (hex!("0122222222333333334444444455000000ca0000000000000cb0"), 0x013641), - (hex!("0122222222333333334444444455000000ca0000000000006140"), 0x0136e1), - (hex!("0122222222333333334444444455000000ca0000000000007700"), 0x013781), - (hex!("0122222222333333334444444455000000cb0000000000000cc0"), 0x013821), - (hex!("0122222222333333334444444455000000cc0000000000000cd0"), 0x0138c1), - (hex!("0122222222333333334444444455000000cd0000000000000ce0"), 0x013961), - (hex!("0122222222333333334444444455000000cd0000000000003f20"), 0x013a01), - (hex!("0122222222333333334444444455000000cd00000000000040f0"), 0x013aa1), - (hex!("0122222222333333334444444455000000cd0000000000004ec0"), 0x013b41), - (hex!("0122222222333333334444444455000000ce0000000000000cf0"), 0x013be1), - (hex!("0122222222333333334444444455000000ce0000000000007200"), 0x013c81), - (hex!("0122222222333333334444444455000000cf0000000000000d00"), 0x013d21), - (hex!("0122222222333333334444444455000000cf00000000000046a0"), 0x013dc1), - (hex!("0122222222333333334444444455000000cf0000000000005960"), 0x013e61), - (hex!("0122222222333333334444444455000000d00000000000000d10"), 0x013f01), - (hex!("0122222222333333334444444455000000d00000000000005f30"), 0x013fa1), - (hex!("0122222222333333334444444455000000d10000000000000d20"), 0x014041), - (hex!("0122222222333333334444444455000000d10000000000007a00"), 0x0140e1), - (hex!("0122222222333333334444444455000000d20000000000000d30"), 0x014181), - (hex!("0122222222333333334444444455000000d30000000000000d40"), 0x014221), - (hex!("0122222222333333334444444455000000d40000000000000d50"), 0x0142c1), - (hex!("0122222222333333334444444455000000d50000000000000d60"), 0x014361), - (hex!("0122222222333333334444444455000000d50000000000004960"), 0x014401), - (hex!("0122222222333333334444444455000000d500000000000055d0"), 0x0144a1), - (hex!("0122222222333333334444444455000000d500000000000067d0"), 0x014541), - (hex!("0122222222333333334444444455000000d60000000000000d70"), 0x0145e1), - (hex!("0122222222333333334444444455000000d70000000000000d80"), 0x014681), - (hex!("0122222222333333334444444455000000d80000000000000d90"), 0x014721), - (hex!("0122222222333333334444444455000000d800000000000065f0"), 0x0147c1), - (hex!("0122222222333333334444444455000000d90000000000000da0"), 0x014861), - (hex!("0122222222333333334444444455000000d90000000000004980"), 0x014901), - (hex!("0122222222333333334444444455000000da0000000000000db0"), 0x0149a1), - (hex!("0122222222333333334444444455000000da00000000000048c0"), 0x014a41), - (hex!("0122222222333333334444444455000000da00000000000072c0"), 0x014ae1), - (hex!("0122222222333333334444444455000000da00000000000076b0"), 0x014b81), - (hex!("0122222222333333334444444455000000db0000000000000dc0"), 0x014c21), - (hex!("0122222222333333334444444455000000dc0000000000000dd0"), 0x014cc1), - (hex!("0122222222333333334444444455000000dc00000000000040a0"), 0x014d61), - (hex!("0122222222333333334444444455000000dc00000000000074c0"), 0x014e01), - (hex!("0122222222333333334444444455000000dd0000000000000de0"), 0x014ea1), - (hex!("0122222222333333334444444455000000dd0000000000004e50"), 0x014f41), - (hex!("0122222222333333334444444455000000dd0000000000007270"), 0x014fe1), - (hex!("0122222222333333334444444455000000de0000000000000df0"), 0x015081), - (hex!("0122222222333333334444444455000000de00000000000078d0"), 0x015121), - (hex!("0122222222333333334444444455000000df0000000000000e00"), 0x0151c1), - (hex!("0122222222333333334444444455000000df0000000000004d30"), 0x015261), - (hex!("0122222222333333334444444455000000df0000000000006c30"), 0x015301), - (hex!("0122222222333333334444444455000000e00000000000000e10"), 0x0153a1), - (hex!("0122222222333333334444444455000000e00000000000005d30"), 0x015441), - (hex!("0122222222333333334444444455000000e10000000000000e20"), 0x0154e1), - (hex!("0122222222333333334444444455000000e10000000000004610"), 0x015581), - (hex!("0122222222333333334444444455000000e100000000000051d0"), 0x015621), - (hex!("0122222222333333334444444455000000e10000000000005f10"), 0x0156c1), - (hex!("0122222222333333334444444455000000e20000000000000e30"), 0x015761), - (hex!("0122222222333333334444444455000000e20000000000007a90"), 0x015801), - (hex!("0122222222333333334444444455000000e30000000000000e40"), 0x0158a1), - (hex!("0122222222333333334444444455000000e30000000000005ae0"), 0x015941), - (hex!("0122222222333333334444444455000000e40000000000000e50"), 0x0159e1), - (hex!("0122222222333333334444444455000000e50000000000000e60"), 0x015a81), - (hex!("0122222222333333334444444455000000e50000000000004700"), 0x015b21), - (hex!("0122222222333333334444444455000000e500000000000065d0"), 0x015bc1), - (hex!("0122222222333333334444444455000000e60000000000000e70"), 0x015c61), - (hex!("0122222222333333334444444455000000e60000000000004fd0"), 0x015d01), - (hex!("0122222222333333334444444455000000e70000000000000e80"), 0x015da1), - (hex!("0122222222333333334444444455000000e70000000000005150"), 0x015e41), - (hex!("0122222222333333334444444455000000e70000000000005920"), 0x015ee1), - (hex!("0122222222333333334444444455000000e80000000000000e90"), 0x015f81), - (hex!("0122222222333333334444444455000000e80000000000004320"), 0x016021), - (hex!("0122222222333333334444444455000000e80000000000005ec0"), 0x0160c1), - (hex!("0122222222333333334444444455000000e90000000000000ea0"), 0x016161), - (hex!("0122222222333333334444444455000000e900000000000043b0"), 0x016201), - (hex!("0122222222333333334444444455000000ea0000000000000eb0"), 0x0162a1), - (hex!("0122222222333333334444444455000000ea0000000000003ea0"), 0x016341), - (hex!("0122222222333333334444444455000000ea0000000000004f50"), 0x0163e1), - (hex!("0122222222333333334444444455000000ea0000000000007520"), 0x016481), - (hex!("0122222222333333334444444455000000eb0000000000000ec0"), 0x016521), - (hex!("0122222222333333334444444455000000ec0000000000000ed0"), 0x0165c1), - (hex!("0122222222333333334444444455000000ec0000000000006670"), 0x016661), - (hex!("0122222222333333334444444455000000ed0000000000000ee0"), 0x016701), - (hex!("0122222222333333334444444455000000ee0000000000000ef0"), 0x0167a1), - (hex!("0122222222333333334444444455000000ee0000000000004d10"), 0x016841), - (hex!("0122222222333333334444444455000000ef0000000000000f00"), 0x0168e1), - (hex!("0122222222333333334444444455000000f00000000000000f10"), 0x016981), - (hex!("0122222222333333334444444455000000f00000000000007220"), 0x016a21), - (hex!("0122222222333333334444444455000000f00000000000007540"), 0x016ac1), - (hex!("0122222222333333334444444455000000f10000000000000f20"), 0x016b61), - (hex!("0122222222333333334444444455000000f100000000000066f0"), 0x016c01), - (hex!("0122222222333333334444444455000000f20000000000000f30"), 0x016ca1), - (hex!("0122222222333333334444444455000000f20000000000007810"), 0x016d41), - (hex!("0122222222333333334444444455000000f30000000000000f40"), 0x016de1), - (hex!("0122222222333333334444444455000000f30000000000007b70"), 0x016e81), - (hex!("0122222222333333334444444455000000f40000000000000f50"), 0x016f21), - (hex!("0122222222333333334444444455000000f400000000000059c0"), 0x016fc1), - (hex!("0122222222333333334444444455000000f50000000000000f60"), 0x017061), - (hex!("0122222222333333334444444455000000f50000000000003fb0"), 0x017101), - (hex!("0122222222333333334444444455000000f50000000000005740"), 0x0171a1), - (hex!("0122222222333333334444444455000000f500000000000064d0"), 0x017241), - (hex!("0122222222333333334444444455000000f50000000000006960"), 0x0172e1), - (hex!("0122222222333333334444444455000000f60000000000000f70"), 0x017381), - (hex!("0122222222333333334444444455000000f60000000000006d00"), 0x017421), - (hex!("0122222222333333334444444455000000f70000000000000f80"), 0x0174c1), - (hex!("0122222222333333334444444455000000f80000000000000f90"), 0x017561), - (hex!("0122222222333333334444444455000000f90000000000000fa0"), 0x017601), - (hex!("0122222222333333334444444455000000fa0000000000000fb0"), 0x0176a1), - (hex!("0122222222333333334444444455000000fa00000000000067b0"), 0x017741), - (hex!("0122222222333333334444444455000000fb0000000000000fc0"), 0x0177e1), - (hex!("0122222222333333334444444455000000fb0000000000004eb0"), 0x017881), - (hex!("0122222222333333334444444455000000fb0000000000006ef0"), 0x017921), - (hex!("0122222222333333334444444455000000fc0000000000000fd0"), 0x0179c1), - (hex!("0122222222333333334444444455000000fc0000000000004470"), 0x017a61), - (hex!("0122222222333333334444444455000000fc0000000000005940"), 0x017b01), - (hex!("0122222222333333334444444455000000fd0000000000000fe0"), 0x017ba1), - (hex!("0122222222333333334444444455000000fe0000000000000ff0"), 0x017c41), - (hex!("0122222222333333334444444455000000ff0000000000001000"), 0x017ce1), - (hex!("0122222222333333334444444455000000ff0000000000005690"), 0x017d81), - (hex!("0122222222333333334444444455000001000000000000001010"), 0x017e21), - (hex!("0122222222333333334444444455000001000000000000005210"), 0x017ec1), - (hex!("01222222223333333344444444550000010000000000000070a0"), 0x017f61), - (hex!("0122222222333333334444444455000001010000000000001020"), 0x018001), - (hex!("0122222222333333334444444455000001010000000000006b80"), 0x0180a1), - (hex!("0122222222333333334444444455000001020000000000001030"), 0x018141), - (hex!("0122222222333333334444444455000001030000000000001040"), 0x0181e1), - (hex!("0122222222333333334444444455000001030000000000004c80"), 0x018281), - (hex!("0122222222333333334444444455000001040000000000001050"), 0x018321), - (hex!("0122222222333333334444444455000001040000000000004850"), 0x0183c1), - (hex!("01222222223333333344444444550000010400000000000057b0"), 0x018461), - (hex!("0122222222333333334444444455000001050000000000001060"), 0x018501), - (hex!("01222222223333333344444444550000010500000000000048d0"), 0x0185a1), - (hex!("0122222222333333334444444455000001050000000000007870"), 0x018641), - (hex!("0122222222333333334444444455000001060000000000001070"), 0x0186e1), - (hex!("0122222222333333334444444455000001060000000000004f90"), 0x018781), - (hex!("0122222222333333334444444455000001060000000000006270"), 0x018821), - (hex!("0122222222333333334444444455000001070000000000001080"), 0x0188c1), - (hex!("01222222223333333344444444550000010700000000000063b0"), 0x018961), - (hex!("0122222222333333334444444455000001080000000000001090"), 0x018a01), - (hex!("01222222223333333344444444550000010900000000000010a0"), 0x018aa1), - (hex!("0122222222333333334444444455000001090000000000006f40"), 0x018b41), - (hex!("01222222223333333344444444550000010a00000000000010b0"), 0x018be1), - (hex!("01222222223333333344444444550000010a0000000000006640"), 0x018c81), - (hex!("01222222223333333344444444550000010b00000000000010c0"), 0x018d21), - (hex!("01222222223333333344444444550000010c00000000000010d0"), 0x018dc1), - (hex!("01222222223333333344444444550000010d00000000000010e0"), 0x018e61), - (hex!("01222222223333333344444444550000010e00000000000010f0"), 0x018f01), - (hex!("01222222223333333344444444550000010e0000000000005c40"), 0x018fa1), - (hex!("01222222223333333344444444550000010e0000000000007ba0"), 0x019041), - (hex!("01222222223333333344444444550000010f0000000000001100"), 0x0190e1), - (hex!("01222222223333333344444444550000010f0000000000005c30"), 0x019181), - (hex!("0122222222333333334444444455000001100000000000001110"), 0x019221), - (hex!("0122222222333333334444444455000001100000000000007640"), 0x0192c1), - (hex!("0122222222333333334444444455000001110000000000001120"), 0x019361), - (hex!("01222222223333333344444444550000011100000000000052c0"), 0x019401), - (hex!("0122222222333333334444444455000001110000000000005710"), 0x0194a1), - (hex!("0122222222333333334444444455000001110000000000006a00"), 0x019541), - (hex!("0122222222333333334444444455000001120000000000001130"), 0x0195e1), - (hex!("0122222222333333334444444455000001130000000000001140"), 0x019681), - (hex!("0122222222333333334444444455000001140000000000001150"), 0x019721), - (hex!("0122222222333333334444444455000001140000000000003fa0"), 0x0197c1), - (hex!("01222222223333333344444444550000011400000000000054b0"), 0x019861), - (hex!("0122222222333333334444444455000001140000000000006070"), 0x019901), - (hex!("0122222222333333334444444455000001150000000000001160"), 0x0199a1), - (hex!("0122222222333333334444444455000001150000000000005320"), 0x019a41), - (hex!("0122222222333333334444444455000001150000000000006600"), 0x019ae1), - (hex!("0122222222333333334444444455000001150000000000006df0"), 0x019b81), - (hex!("01222222223333333344444444550000011500000000000079c0"), 0x019c21), - (hex!("0122222222333333334444444455000001160000000000001170"), 0x019cc1), - (hex!("0122222222333333334444444455000001170000000000001180"), 0x019d61), - (hex!("0122222222333333334444444455000001170000000000004a60"), 0x019e01), - (hex!("01222222223333333344444444550000011700000000000063c0"), 0x019ea1), - (hex!("0122222222333333334444444455000001180000000000001190"), 0x019f41), - (hex!("0122222222333333334444444455000001180000000000004530"), 0x019fe1), - (hex!("01222222223333333344444444550000011800000000000077c0"), 0x01a081), - (hex!("01222222223333333344444444550000011900000000000011a0"), 0x01a121), - (hex!("01222222223333333344444444550000011a00000000000011b0"), 0x01a1c1), - (hex!("01222222223333333344444444550000011a00000000000041c0"), 0x01a261), - (hex!("01222222223333333344444444550000011a00000000000061e0"), 0x01a301), - (hex!("01222222223333333344444444550000011b00000000000011c0"), 0x01a3a1), - (hex!("01222222223333333344444444550000011c00000000000011d0"), 0x01a441), - (hex!("01222222223333333344444444550000011c0000000000005f90"), 0x01a4e1), - (hex!("01222222223333333344444444550000011d00000000000011e0"), 0x01a581), - (hex!("01222222223333333344444444550000011d0000000000004160"), 0x01a621), - (hex!("01222222223333333344444444550000011e00000000000011f0"), 0x01a6c1), - (hex!("01222222223333333344444444550000011e00000000000056d0"), 0x01a761), - (hex!("01222222223333333344444444550000011f0000000000001200"), 0x01a801), - (hex!("01222222223333333344444444550000011f0000000000004510"), 0x01a8a1), - (hex!("0122222222333333334444444455000001200000000000001210"), 0x01a941), - (hex!("0122222222333333334444444455000001210000000000001220"), 0x01a9e1), - (hex!("0122222222333333334444444455000001210000000000005140"), 0x01aa81), - (hex!("0122222222333333334444444455000001210000000000006710"), 0x01ab21), - (hex!("0122222222333333334444444455000001210000000000006f50"), 0x01abc1), - (hex!("0122222222333333334444444455000001220000000000001230"), 0x01ac61), - (hex!("0122222222333333334444444455000001220000000000005570"), 0x01ad01), - (hex!("0122222222333333334444444455000001220000000000007ac0"), 0x01ada1), - (hex!("0122222222333333334444444455000001230000000000001240"), 0x01ae41), - (hex!("0122222222333333334444444455000001240000000000001250"), 0x01aee1), - (hex!("0122222222333333334444444455000001240000000000006cd0"), 0x01af81), - (hex!("0122222222333333334444444455000001250000000000001260"), 0x01b021), - (hex!("01222222223333333344444444550000012500000000000046b0"), 0x01b0c1), - (hex!("0122222222333333334444444455000001250000000000005eb0"), 0x01b161), - (hex!("0122222222333333334444444455000001260000000000001270"), 0x01b201), - (hex!("0122222222333333334444444455000001260000000000004630"), 0x01b2a1), - (hex!("0122222222333333334444444455000001270000000000001280"), 0x01b341), - (hex!("0122222222333333334444444455000001270000000000004ff0"), 0x01b3e1), - (hex!("0122222222333333334444444455000001270000000000006ec0"), 0x01b481), - (hex!("0122222222333333334444444455000001280000000000001290"), 0x01b521), - (hex!("01222222223333333344444444550000012900000000000012a0"), 0x01b5c1), - (hex!("0122222222333333334444444455000001290000000000005f60"), 0x01b661), - (hex!("01222222223333333344444444550000012a00000000000012b0"), 0x01b701), - (hex!("01222222223333333344444444550000012a0000000000005480"), 0x01b7a1), - (hex!("01222222223333333344444444550000012b00000000000012c0"), 0x01b841), - (hex!("01222222223333333344444444550000012b00000000000065a0"), 0x01b8e1), - (hex!("01222222223333333344444444550000012b00000000000066c0"), 0x01b981), - (hex!("01222222223333333344444444550000012c00000000000012d0"), 0x01ba21), - (hex!("01222222223333333344444444550000012c00000000000064b0"), 0x01bac1), - (hex!("01222222223333333344444444550000012d00000000000012e0"), 0x01bb61), - (hex!("01222222223333333344444444550000012d00000000000049c0"), 0x01bc01), - (hex!("01222222223333333344444444550000012d0000000000004bf0"), 0x01bca1), - (hex!("01222222223333333344444444550000012e00000000000012f0"), 0x01bd41), - (hex!("01222222223333333344444444550000012e0000000000005ed0"), 0x01bde1), - (hex!("01222222223333333344444444550000012f0000000000001300"), 0x01be81), - (hex!("01222222223333333344444444550000012f00000000000049a0"), 0x01bf21), - (hex!("0122222222333333334444444455000001300000000000001310"), 0x01bfc1), - (hex!("0122222222333333334444444455000001300000000000007840"), 0x01c061), - (hex!("0122222222333333334444444455000001310000000000001320"), 0x01c101), - (hex!("0122222222333333334444444455000001310000000000005f70"), 0x01c1a1), - (hex!("0122222222333333334444444455000001320000000000001330"), 0x01c241), - (hex!("0122222222333333334444444455000001320000000000005a00"), 0x01c2e1), - (hex!("0122222222333333334444444455000001330000000000001340"), 0x01c381), - (hex!("0122222222333333334444444455000001330000000000006c70"), 0x01c421), - (hex!("0122222222333333334444444455000001340000000000001350"), 0x01c4c1), - (hex!("0122222222333333334444444455000001340000000000005c60"), 0x01c561), - (hex!("0122222222333333334444444455000001350000000000001360"), 0x01c601), - (hex!("0122222222333333334444444455000001350000000000004f10"), 0x01c6a1), - (hex!("0122222222333333334444444455000001360000000000001370"), 0x01c741), - (hex!("0122222222333333334444444455000001360000000000004c60"), 0x01c7e1), - (hex!("0122222222333333334444444455000001370000000000001380"), 0x01c881), - (hex!("0122222222333333334444444455000001380000000000001390"), 0x01c921), - (hex!("01222222223333333344444444550000013900000000000013a0"), 0x01c9c1), - (hex!("0122222222333333334444444455000001390000000000004ea0"), 0x01ca61), - (hex!("01222222223333333344444444550000013a00000000000013b0"), 0x01cb01), - (hex!("01222222223333333344444444550000013a0000000000007350"), 0x01cba1), - (hex!("01222222223333333344444444550000013b00000000000013c0"), 0x01cc41), - (hex!("01222222223333333344444444550000013c00000000000013d0"), 0x01cce1), - (hex!("01222222223333333344444444550000013c0000000000007050"), 0x01cd81), - (hex!("01222222223333333344444444550000013d00000000000013e0"), 0x01ce21), - (hex!("01222222223333333344444444550000013d0000000000006bd0"), 0x01cec1), - (hex!("01222222223333333344444444550000013e00000000000013f0"), 0x01cf61), - (hex!("01222222223333333344444444550000013e00000000000058e0"), 0x01d001), - (hex!("01222222223333333344444444550000013f0000000000001400"), 0x01d0a1), - (hex!("01222222223333333344444444550000013f0000000000004740"), 0x01d141), - (hex!("0122222222333333334444444455000001400000000000001410"), 0x01d1e1), - (hex!("0122222222333333334444444455000001400000000000003f10"), 0x01d281), - (hex!("0122222222333333334444444455000001400000000000006d40"), 0x01d321), - (hex!("01222222223333333344444444550000014000000000000072d0"), 0x01d3c1), - (hex!("0122222222333333334444444455000001410000000000001420"), 0x01d461), - (hex!("0122222222333333334444444455000001420000000000001430"), 0x01d501), - (hex!("0122222222333333334444444455000001430000000000001440"), 0x01d5a1), - (hex!("0122222222333333334444444455000001440000000000001450"), 0x01d641), - (hex!("0122222222333333334444444455000001450000000000001460"), 0x01d6e1), - (hex!("0122222222333333334444444455000001460000000000001470"), 0x01d781), - (hex!("01222222223333333344444444550000014600000000000055c0"), 0x01d821), - (hex!("0122222222333333334444444455000001470000000000001480"), 0x01d8c1), - (hex!("0122222222333333334444444455000001470000000000004570"), 0x01d961), - (hex!("0122222222333333334444444455000001470000000000004be0"), 0x01da01), - (hex!("0122222222333333334444444455000001480000000000001490"), 0x01daa1), - (hex!("0122222222333333334444444455000001480000000000005360"), 0x01db41), - (hex!("01222222223333333344444444550000014900000000000014a0"), 0x01dbe1), - (hex!("01222222223333333344444444550000014a00000000000014b0"), 0x01dc81), - (hex!("01222222223333333344444444550000014a00000000000053d0"), 0x01dd21), - (hex!("01222222223333333344444444550000014b00000000000014c0"), 0x01ddc1), - (hex!("01222222223333333344444444550000014b0000000000005950"), 0x01de61), - (hex!("01222222223333333344444444550000014c00000000000014d0"), 0x01df01), - (hex!("01222222223333333344444444550000014c0000000000004f60"), 0x01dfa1), - (hex!("01222222223333333344444444550000014d00000000000014e0"), 0x01e041), - (hex!("01222222223333333344444444550000014d0000000000004520"), 0x01e0e1), - (hex!("01222222223333333344444444550000014d0000000000005200"), 0x01e181), - (hex!("01222222223333333344444444550000014e00000000000014f0"), 0x01e221), - (hex!("01222222223333333344444444550000014e0000000000005bd0"), 0x01e2c1), - (hex!("01222222223333333344444444550000014f0000000000001500"), 0x01e361), - (hex!("01222222223333333344444444550000014f00000000000060d0"), 0x01e401), - (hex!("0122222222333333334444444455000001500000000000001510"), 0x01e4a1), - (hex!("01222222223333333344444444550000015000000000000075e0"), 0x01e541), - (hex!("0122222222333333334444444455000001510000000000001520"), 0x01e5e1), - (hex!("0122222222333333334444444455000001510000000000005c00"), 0x01e681), - (hex!("0122222222333333334444444455000001510000000000006af0"), 0x01e721), - (hex!("0122222222333333334444444455000001510000000000007b80"), 0x01e7c1), - (hex!("0122222222333333334444444455000001520000000000001530"), 0x01e861), - (hex!("0122222222333333334444444455000001520000000000004c70"), 0x01e901), - (hex!("0122222222333333334444444455000001530000000000001540"), 0x01e9a1), - (hex!("0122222222333333334444444455000001540000000000001550"), 0x01ea41), - (hex!("0122222222333333334444444455000001540000000000007cd0"), 0x01eae1), - (hex!("0122222222333333334444444455000001550000000000001560"), 0x01eb81), - (hex!("0122222222333333334444444455000001550000000000004ae0"), 0x01ec21), - (hex!("01222222223333333344444444550000015500000000000068c0"), 0x01ecc1), - (hex!("0122222222333333334444444455000001560000000000001570"), 0x01ed61), - (hex!("01222222223333333344444444550000015600000000000064a0"), 0x01ee01), - (hex!("0122222222333333334444444455000001570000000000001580"), 0x01eea1), - (hex!("0122222222333333334444444455000001580000000000001590"), 0x01ef41), - (hex!("0122222222333333334444444455000001580000000000006d30"), 0x01efe1), - (hex!("01222222223333333344444444550000015800000000000074f0"), 0x01f081), - (hex!("01222222223333333344444444550000015900000000000015a0"), 0x01f121), - (hex!("01222222223333333344444444550000015900000000000053a0"), 0x01f1c1), - (hex!("01222222223333333344444444550000015900000000000055e0"), 0x01f261), - (hex!("0122222222333333334444444455000001590000000000006210"), 0x01f301), - (hex!("01222222223333333344444444550000015900000000000067c0"), 0x01f3a1), - (hex!("01222222223333333344444444550000015a00000000000015b0"), 0x01f441), - (hex!("01222222223333333344444444550000015b00000000000015c0"), 0x01f4e1), - (hex!("01222222223333333344444444550000015c00000000000015d0"), 0x01f581), - (hex!("01222222223333333344444444550000015c0000000000004d80"), 0x01f621), - (hex!("01222222223333333344444444550000015c00000000000073f0"), 0x01f6c1), - (hex!("01222222223333333344444444550000015d00000000000015e0"), 0x01f761), - (hex!("01222222223333333344444444550000015e00000000000015f0"), 0x01f801), - (hex!("01222222223333333344444444550000015e0000000000004120"), 0x01f8a1), - (hex!("01222222223333333344444444550000015e0000000000004350"), 0x01f941), - (hex!("01222222223333333344444444550000015e0000000000007c50"), 0x01f9e1), - (hex!("01222222223333333344444444550000015f0000000000001600"), 0x01fa81), - (hex!("0122222222333333334444444455000001600000000000001610"), 0x01fb21), - (hex!("0122222222333333334444444455000001600000000000004840"), 0x01fbc1), - (hex!("0122222222333333334444444455000001600000000000004b10"), 0x01fc61), - (hex!("0122222222333333334444444455000001600000000000007060"), 0x01fd01), - (hex!("0122222222333333334444444455000001610000000000001620"), 0x01fda1), - (hex!("0122222222333333334444444455000001610000000000005300"), 0x01fe41), - (hex!("0122222222333333334444444455000001620000000000001630"), 0x01fee1), - (hex!("0122222222333333334444444455000001620000000000006530"), 0x01ff81), - (hex!("0122222222333333334444444455000001630000000000001640"), 0x020021), - (hex!("0122222222333333334444444455000001640000000000001650"), 0x0200c1), - (hex!("0122222222333333334444444455000001650000000000001660"), 0x020161), - (hex!("0122222222333333334444444455000001660000000000001670"), 0x020201), - (hex!("0122222222333333334444444455000001670000000000001680"), 0x0202a1), - (hex!("0122222222333333334444444455000001670000000000007310"), 0x020341), - (hex!("0122222222333333334444444455000001680000000000001690"), 0x0203e1), - (hex!("0122222222333333334444444455000001680000000000007b50"), 0x020481), - (hex!("01222222223333333344444444550000016900000000000016a0"), 0x020521), - (hex!("01222222223333333344444444550000016900000000000049d0"), 0x0205c1), - (hex!("01222222223333333344444444550000016a00000000000016b0"), 0x020661), - (hex!("01222222223333333344444444550000016a00000000000078b0"), 0x020701), - (hex!("01222222223333333344444444550000016b00000000000016c0"), 0x0207a1), - (hex!("01222222223333333344444444550000016b0000000000004100"), 0x020841), - (hex!("01222222223333333344444444550000016c00000000000016d0"), 0x0208e1), - (hex!("01222222223333333344444444550000016c0000000000006e00"), 0x020981), - (hex!("01222222223333333344444444550000016d00000000000016e0"), 0x020a21), - (hex!("01222222223333333344444444550000016e00000000000016f0"), 0x020ac1), - (hex!("01222222223333333344444444550000016e0000000000004ac0"), 0x020b61), - (hex!("01222222223333333344444444550000016e0000000000007820"), 0x020c01), - (hex!("01222222223333333344444444550000016f0000000000001700"), 0x020ca1), - (hex!("0122222222333333334444444455000001700000000000001710"), 0x020d41), - (hex!("0122222222333333334444444455000001700000000000005830"), 0x020de1), - (hex!("0122222222333333334444444455000001710000000000001720"), 0x020e81), - (hex!("01222222223333333344444444550000017100000000000072f0"), 0x020f21), - (hex!("0122222222333333334444444455000001720000000000001730"), 0x020fc1), - (hex!("0122222222333333334444444455000001720000000000004870"), 0x021061), - (hex!("01222222223333333344444444550000017200000000000070b0"), 0x021101), - (hex!("0122222222333333334444444455000001730000000000001740"), 0x0211a1), - (hex!("0122222222333333334444444455000001740000000000001750"), 0x021241), - (hex!("0122222222333333334444444455000001750000000000001760"), 0x0212e1), - (hex!("0122222222333333334444444455000001750000000000005670"), 0x021381), - (hex!("0122222222333333334444444455000001750000000000005870"), 0x021421), - (hex!("0122222222333333334444444455000001760000000000001770"), 0x0214c1), - (hex!("0122222222333333334444444455000001770000000000001780"), 0x021561), - (hex!("0122222222333333334444444455000001770000000000005000"), 0x021601), - (hex!("0122222222333333334444444455000001770000000000007090"), 0x0216a1), - (hex!("0122222222333333334444444455000001780000000000001790"), 0x021741), - (hex!("01222222223333333344444444550000017800000000000048a0"), 0x0217e1), - (hex!("0122222222333333334444444455000001780000000000006bf0"), 0x021881), - (hex!("01222222223333333344444444550000017900000000000017a0"), 0x021921), - (hex!("01222222223333333344444444550000017900000000000057d0"), 0x0219c1), - (hex!("0122222222333333334444444455000001790000000000006660"), 0x021a61), - (hex!("01222222223333333344444444550000017a00000000000017b0"), 0x021b01), - (hex!("01222222223333333344444444550000017a0000000000004970"), 0x021ba1), - (hex!("01222222223333333344444444550000017a0000000000005dc0"), 0x021c41), - (hex!("01222222223333333344444444550000017b00000000000017c0"), 0x021ce1), - (hex!("01222222223333333344444444550000017b0000000000004ee0"), 0x021d81), - (hex!("01222222223333333344444444550000017b00000000000054c0"), 0x021e21), - (hex!("01222222223333333344444444550000017c00000000000017d0"), 0x021ec1), - (hex!("01222222223333333344444444550000017c0000000000003fc0"), 0x021f61), - (hex!("01222222223333333344444444550000017c00000000000063e0"), 0x022001), - (hex!("01222222223333333344444444550000017c0000000000006520"), 0x0220a1), - (hex!("01222222223333333344444444550000017d00000000000017e0"), 0x022141), - (hex!("01222222223333333344444444550000017d0000000000006220"), 0x0221e1), - (hex!("01222222223333333344444444550000017d0000000000007120"), 0x022281), - (hex!("01222222223333333344444444550000017e00000000000017f0"), 0x022321), - (hex!("01222222223333333344444444550000017f0000000000001800"), 0x0223c1), - (hex!("0122222222333333334444444455000001800000000000001810"), 0x022461), - (hex!("0122222222333333334444444455000001810000000000001820"), 0x022501), - (hex!("01222222223333333344444444550000018100000000000041f0"), 0x0225a1), - (hex!("0122222222333333334444444455000001810000000000007590"), 0x022641), - (hex!("0122222222333333334444444455000001820000000000001830"), 0x0226e1), - (hex!("0122222222333333334444444455000001820000000000004ce0"), 0x022781), - (hex!("0122222222333333334444444455000001830000000000001840"), 0x022821), - (hex!("01222222223333333344444444550000018300000000000042c0"), 0x0228c1), - (hex!("0122222222333333334444444455000001840000000000001850"), 0x022961), - (hex!("0122222222333333334444444455000001840000000000004f70"), 0x022a01), - (hex!("0122222222333333334444444455000001850000000000001860"), 0x022aa1), - (hex!("0122222222333333334444444455000001850000000000006470"), 0x022b41), - (hex!("0122222222333333334444444455000001850000000000007500"), 0x022be1), - (hex!("0122222222333333334444444455000001860000000000001870"), 0x022c81), - (hex!("0122222222333333334444444455000001860000000000004770"), 0x022d21), - (hex!("0122222222333333334444444455000001870000000000001880"), 0x022dc1), - (hex!("0122222222333333334444444455000001870000000000006a30"), 0x022e61), - (hex!("0122222222333333334444444455000001880000000000001890"), 0x022f01), - (hex!("0122222222333333334444444455000001880000000000007410"), 0x022fa1), - (hex!("01222222223333333344444444550000018900000000000018a0"), 0x023041), - (hex!("01222222223333333344444444550000018900000000000044d0"), 0x0230e1), - (hex!("0122222222333333334444444455000001890000000000005ac0"), 0x023181), - (hex!("01222222223333333344444444550000018a00000000000018b0"), 0x023221), - (hex!("01222222223333333344444444550000018a0000000000006260"), 0x0232c1), - (hex!("01222222223333333344444444550000018a0000000000006d70"), 0x023361), - (hex!("01222222223333333344444444550000018b00000000000018c0"), 0x023401), - (hex!("01222222223333333344444444550000018b0000000000004aa0"), 0x0234a1), - (hex!("01222222223333333344444444550000018b0000000000006fd0"), 0x023541), - (hex!("01222222223333333344444444550000018c00000000000018d0"), 0x0235e1), - (hex!("01222222223333333344444444550000018c00000000000051b0"), 0x023681), - (hex!("01222222223333333344444444550000018c0000000000006650"), 0x023721), - (hex!("01222222223333333344444444550000018d00000000000018e0"), 0x0237c1), - (hex!("01222222223333333344444444550000018e00000000000018f0"), 0x023861), - (hex!("01222222223333333344444444550000018e00000000000041d0"), 0x023901), - (hex!("01222222223333333344444444550000018f0000000000001900"), 0x0239a1), - (hex!("01222222223333333344444444550000018f0000000000007600"), 0x023a41), - (hex!("0122222222333333334444444455000001900000000000001910"), 0x023ae1), - (hex!("0122222222333333334444444455000001900000000000005410"), 0x023b81), - (hex!("0122222222333333334444444455000001900000000000006760"), 0x023c21), - (hex!("0122222222333333334444444455000001910000000000001920"), 0x023cc1), - (hex!("0122222222333333334444444455000001920000000000001930"), 0x023d61), - (hex!("0122222222333333334444444455000001920000000000004ca0"), 0x023e01), - (hex!("0122222222333333334444444455000001920000000000005d80"), 0x023ea1), - (hex!("0122222222333333334444444455000001920000000000005fd0"), 0x023f41), - (hex!("01222222223333333344444444550000019200000000000070d0"), 0x023fe1), - (hex!("0122222222333333334444444455000001930000000000001940"), 0x024081), - (hex!("0122222222333333334444444455000001930000000000004010"), 0x024121), - (hex!("0122222222333333334444444455000001930000000000007ca0"), 0x0241c1), - (hex!("0122222222333333334444444455000001940000000000001950"), 0x024261), - (hex!("0122222222333333334444444455000001950000000000001960"), 0x024301), - (hex!("0122222222333333334444444455000001950000000000005380"), 0x0243a1), - (hex!("0122222222333333334444444455000001960000000000001970"), 0x024441), - (hex!("0122222222333333334444444455000001960000000000006de0"), 0x0244e1), - (hex!("0122222222333333334444444455000001970000000000001980"), 0x024581), - (hex!("01222222223333333344444444550000019700000000000048f0"), 0x024621), - (hex!("0122222222333333334444444455000001980000000000001990"), 0x0246c1), - (hex!("0122222222333333334444444455000001980000000000006510"), 0x024761), - (hex!("01222222223333333344444444550000019900000000000019a0"), 0x024801), - (hex!("0122222222333333334444444455000001990000000000007570"), 0x0248a1), - (hex!("0122222222333333334444444455000001990000000000007580"), 0x024941), - (hex!("01222222223333333344444444550000019a00000000000019b0"), 0x0249e1), - (hex!("01222222223333333344444444550000019a0000000000004050"), 0x024a81), - (hex!("01222222223333333344444444550000019a0000000000004ba0"), 0x024b21), - (hex!("01222222223333333344444444550000019a0000000000005540"), 0x024bc1), - (hex!("01222222223333333344444444550000019a00000000000061c0"), 0x024c61), - (hex!("01222222223333333344444444550000019a0000000000007c60"), 0x024d01), - (hex!("01222222223333333344444444550000019b00000000000019c0"), 0x024da1), - (hex!("01222222223333333344444444550000019b0000000000006240"), 0x024e41), - (hex!("01222222223333333344444444550000019c00000000000019d0"), 0x024ee1), - (hex!("01222222223333333344444444550000019d00000000000019e0"), 0x024f81), - (hex!("01222222223333333344444444550000019d0000000000004640"), 0x025021), - (hex!("01222222223333333344444444550000019d00000000000052a0"), 0x0250c1), - (hex!("01222222223333333344444444550000019d00000000000052b0"), 0x025161), - (hex!("01222222223333333344444444550000019e00000000000019f0"), 0x025201), - (hex!("01222222223333333344444444550000019f0000000000001a00"), 0x0252a1), - (hex!("01222222223333333344444444550000019f0000000000006b20"), 0x025341), - (hex!("0122222222333333334444444455000001a00000000000001a10"), 0x0253e1), - (hex!("0122222222333333334444444455000001a10000000000001a20"), 0x025481), - (hex!("0122222222333333334444444455000001a10000000000005460"), 0x025521), - (hex!("0122222222333333334444444455000001a10000000000005d20"), 0x0255c1), - (hex!("0122222222333333334444444455000001a100000000000068f0"), 0x025661), - (hex!("0122222222333333334444444455000001a20000000000001a30"), 0x025701), - (hex!("0122222222333333334444444455000001a20000000000007170"), 0x0257a1), - (hex!("0122222222333333334444444455000001a30000000000001a40"), 0x025841), - (hex!("0122222222333333334444444455000001a40000000000001a50"), 0x0258e1), - (hex!("0122222222333333334444444455000001a50000000000001a60"), 0x025981), - (hex!("0122222222333333334444444455000001a60000000000001a70"), 0x025a21), - (hex!("0122222222333333334444444455000001a70000000000001a80"), 0x025ac1), - (hex!("0122222222333333334444444455000001a70000000000005a90"), 0x025b61), - (hex!("0122222222333333334444444455000001a70000000000006440"), 0x025c01), - (hex!("0122222222333333334444444455000001a80000000000001a90"), 0x025ca1), - (hex!("0122222222333333334444444455000001a80000000000004800"), 0x025d41), - (hex!("0122222222333333334444444455000001a90000000000001aa0"), 0x025de1), - (hex!("0122222222333333334444444455000001aa0000000000001ab0"), 0x025e81), - (hex!("0122222222333333334444444455000001aa0000000000005b60"), 0x025f21), - (hex!("0122222222333333334444444455000001ab0000000000001ac0"), 0x025fc1), - (hex!("0122222222333333334444444455000001ab0000000000006700"), 0x026061), - (hex!("0122222222333333334444444455000001ab00000000000071d0"), 0x026101), - (hex!("0122222222333333334444444455000001ac0000000000001ad0"), 0x0261a1), - (hex!("0122222222333333334444444455000001ac0000000000007380"), 0x026241), - (hex!("0122222222333333334444444455000001ad0000000000001ae0"), 0x0262e1), - (hex!("0122222222333333334444444455000001ad0000000000006350"), 0x026381), - (hex!("0122222222333333334444444455000001ae0000000000001af0"), 0x026421), - (hex!("0122222222333333334444444455000001af0000000000001b00"), 0x0264c1), - (hex!("0122222222333333334444444455000001af0000000000007390"), 0x026561), - (hex!("0122222222333333334444444455000001b00000000000001b10"), 0x026601), - (hex!("0122222222333333334444444455000001b10000000000001b20"), 0x0266a1), - (hex!("0122222222333333334444444455000001b10000000000005cc0"), 0x026741), - (hex!("0122222222333333334444444455000001b20000000000001b30"), 0x0267e1), - (hex!("0122222222333333334444444455000001b20000000000004fb0"), 0x026881), - (hex!("0122222222333333334444444455000001b30000000000001b40"), 0x026921), - (hex!("0122222222333333334444444455000001b40000000000001b50"), 0x0269c1), - (hex!("0122222222333333334444444455000001b50000000000001b60"), 0x026a61), - (hex!("0122222222333333334444444455000001b60000000000001b70"), 0x026b01), - (hex!("0122222222333333334444444455000001b600000000000048e0"), 0x026ba1), - (hex!("0122222222333333334444444455000001b70000000000001b80"), 0x026c41), - (hex!("0122222222333333334444444455000001b70000000000005ca0"), 0x026ce1), - (hex!("0122222222333333334444444455000001b70000000000007900"), 0x026d81), - (hex!("0122222222333333334444444455000001b80000000000001b90"), 0x026e21), - (hex!("0122222222333333334444444455000001b80000000000004d90"), 0x026ec1), - (hex!("0122222222333333334444444455000001b90000000000001ba0"), 0x026f61), - (hex!("0122222222333333334444444455000001b90000000000003f40"), 0x027001), - (hex!("0122222222333333334444444455000001ba0000000000001bb0"), 0x0270a1), - (hex!("0122222222333333334444444455000001ba00000000000042a0"), 0x027141), - (hex!("0122222222333333334444444455000001ba00000000000067f0"), 0x0271e1), - (hex!("0122222222333333334444444455000001ba00000000000073a0"), 0x027281), - (hex!("0122222222333333334444444455000001bb0000000000001bc0"), 0x027321), - (hex!("0122222222333333334444444455000001bb0000000000004a00"), 0x0273c1), - (hex!("0122222222333333334444444455000001bb0000000000005e00"), 0x027461), - (hex!("0122222222333333334444444455000001bc0000000000001bd0"), 0x027501), - (hex!("0122222222333333334444444455000001bc0000000000004230"), 0x0275a1), - (hex!("0122222222333333334444444455000001bc0000000000005860"), 0x027641), - (hex!("0122222222333333334444444455000001bd0000000000001be0"), 0x0276e1), - (hex!("0122222222333333334444444455000001bd0000000000007c70"), 0x027781), - (hex!("0122222222333333334444444455000001be0000000000001bf0"), 0x027821), - (hex!("0122222222333333334444444455000001be0000000000007770"), 0x0278c1), - (hex!("0122222222333333334444444455000001be0000000000007cf0"), 0x027961), - (hex!("0122222222333333334444444455000001bf0000000000001c00"), 0x027a01), - (hex!("0122222222333333334444444455000001bf0000000000006490"), 0x027aa1), - (hex!("0122222222333333334444444455000001c00000000000001c10"), 0x027b41), - (hex!("0122222222333333334444444455000001c10000000000001c20"), 0x027be1), - (hex!("0122222222333333334444444455000001c10000000000004600"), 0x027c81), - (hex!("0122222222333333334444444455000001c20000000000001c30"), 0x027d21), - (hex!("0122222222333333334444444455000001c20000000000006e30"), 0x027dc1), - (hex!("0122222222333333334444444455000001c30000000000001c40"), 0x027e61), - (hex!("0122222222333333334444444455000001c40000000000001c50"), 0x027f01), - (hex!("0122222222333333334444444455000001c50000000000001c60"), 0x027fa1), - (hex!("0122222222333333334444444455000001c60000000000001c70"), 0x028041), - (hex!("0122222222333333334444444455000001c60000000000004240"), 0x0280e1), - (hex!("0122222222333333334444444455000001c60000000000005bb0"), 0x028181), - (hex!("0122222222333333334444444455000001c70000000000001c80"), 0x028221), - (hex!("0122222222333333334444444455000001c80000000000001c90"), 0x0282c1), - (hex!("0122222222333333334444444455000001c90000000000001ca0"), 0x028361), - (hex!("0122222222333333334444444455000001c90000000000006730"), 0x028401), - (hex!("0122222222333333334444444455000001ca0000000000001cb0"), 0x0284a1), - (hex!("0122222222333333334444444455000001ca00000000000070f0"), 0x028541), - (hex!("0122222222333333334444444455000001cb0000000000001cc0"), 0x0285e1), - (hex!("0122222222333333334444444455000001cb00000000000071a0"), 0x028681), - (hex!("0122222222333333334444444455000001cc0000000000001cd0"), 0x028721), - (hex!("0122222222333333334444444455000001cc0000000000005280"), 0x0287c1), - (hex!("0122222222333333334444444455000001cc0000000000005d90"), 0x028861), - (hex!("0122222222333333334444444455000001cd0000000000001ce0"), 0x028901), - (hex!("0122222222333333334444444455000001cd00000000000069b0"), 0x0289a1), - (hex!("0122222222333333334444444455000001ce0000000000001cf0"), 0x028a41), - (hex!("0122222222333333334444444455000001ce0000000000004540"), 0x028ae1), - (hex!("0122222222333333334444444455000001cf0000000000001d00"), 0x028b81), - (hex!("0122222222333333334444444455000001cf00000000000076a0"), 0x028c21), - (hex!("0122222222333333334444444455000001d00000000000001d10"), 0x028cc1), - (hex!("0122222222333333334444444455000001d000000000000060a0"), 0x028d61), - (hex!("0122222222333333334444444455000001d10000000000001d20"), 0x028e01), - (hex!("0122222222333333334444444455000001d20000000000001d30"), 0x028ea1), - (hex!("0122222222333333334444444455000001d30000000000001d40"), 0x028f41), - (hex!("0122222222333333334444444455000001d30000000000004000"), 0x028fe1), - (hex!("0122222222333333334444444455000001d30000000000004140"), 0x029081), - (hex!("0122222222333333334444444455000001d30000000000006790"), 0x029121), - (hex!("0122222222333333334444444455000001d40000000000001d50"), 0x0291c1), - (hex!("0122222222333333334444444455000001d50000000000001d60"), 0x029261), - (hex!("0122222222333333334444444455000001d60000000000001d70"), 0x029301), - (hex!("0122222222333333334444444455000001d60000000000004b50"), 0x0293a1), - (hex!("0122222222333333334444444455000001d60000000000007430"), 0x029441), - (hex!("0122222222333333334444444455000001d70000000000001d80"), 0x0294e1), - (hex!("0122222222333333334444444455000001d70000000000006920"), 0x029581), - (hex!("0122222222333333334444444455000001d80000000000001d90"), 0x029621), - (hex!("0122222222333333334444444455000001d80000000000005b30"), 0x0296c1), - (hex!("0122222222333333334444444455000001d90000000000001da0"), 0x029761), - (hex!("0122222222333333334444444455000001da0000000000001db0"), 0x029801), - (hex!("0122222222333333334444444455000001da0000000000004af0"), 0x0298a1), - (hex!("0122222222333333334444444455000001da0000000000007240"), 0x029941), - (hex!("0122222222333333334444444455000001da0000000000007470"), 0x0299e1), - (hex!("0122222222333333334444444455000001db0000000000001dc0"), 0x029a81), - (hex!("0122222222333333334444444455000001db00000000000045d0"), 0x029b21), - (hex!("0122222222333333334444444455000001dc0000000000001dd0"), 0x029bc1), - (hex!("0122222222333333334444444455000001dd0000000000001de0"), 0x029c61), - (hex!("0122222222333333334444444455000001dd0000000000004bb0"), 0x029d01), - (hex!("0122222222333333334444444455000001dd0000000000004cd0"), 0x029da1), - (hex!("0122222222333333334444444455000001dd0000000000006100"), 0x029e41), - (hex!("0122222222333333334444444455000001dd0000000000007bb0"), 0x029ee1), - (hex!("0122222222333333334444444455000001de0000000000001df0"), 0x029f81), - (hex!("0122222222333333334444444455000001de0000000000004260"), 0x02a021), - (hex!("0122222222333333334444444455000001de0000000000006040"), 0x02a0c1), - (hex!("0122222222333333334444444455000001df0000000000001e00"), 0x02a161), - (hex!("0122222222333333334444444455000001df0000000000005fa0"), 0x02a201), - (hex!("0122222222333333334444444455000001df0000000000006a70"), 0x02a2a1), - (hex!("0122222222333333334444444455000001df0000000000006dc0"), 0x02a341), - (hex!("0122222222333333334444444455000001e00000000000001e10"), 0x02a3e1), - (hex!("0122222222333333334444444455000001e00000000000007010"), 0x02a481), - (hex!("0122222222333333334444444455000001e10000000000001e20"), 0x02a521), - (hex!("0122222222333333334444444455000001e10000000000005720"), 0x02a5c1), - (hex!("0122222222333333334444444455000001e10000000000006830"), 0x02a661), - (hex!("0122222222333333334444444455000001e20000000000001e30"), 0x02a701), - (hex!("0122222222333333334444444455000001e20000000000005100"), 0x02a7a1), - (hex!("0122222222333333334444444455000001e30000000000001e40"), 0x02a841), - (hex!("0122222222333333334444444455000001e40000000000001e50"), 0x02a8e1), - (hex!("0122222222333333334444444455000001e40000000000003f30"), 0x02a981), - (hex!("0122222222333333334444444455000001e40000000000005220"), 0x02aa21), - (hex!("0122222222333333334444444455000001e50000000000001e60"), 0x02aac1), - (hex!("0122222222333333334444444455000001e50000000000006f60"), 0x02ab61), - (hex!("0122222222333333334444444455000001e60000000000001e70"), 0x02ac01), - (hex!("0122222222333333334444444455000001e60000000000006c80"), 0x02aca1), - (hex!("0122222222333333334444444455000001e70000000000001e80"), 0x02ad41), - (hex!("0122222222333333334444444455000001e80000000000001e90"), 0x02ade1), - (hex!("0122222222333333334444444455000001e80000000000004e30"), 0x02ae81), - (hex!("0122222222333333334444444455000001e90000000000001ea0"), 0x02af21), - (hex!("0122222222333333334444444455000001e90000000000005470"), 0x02afc1), - (hex!("0122222222333333334444444455000001ea0000000000001eb0"), 0x02b061), - (hex!("0122222222333333334444444455000001ea0000000000007980"), 0x02b101), - (hex!("0122222222333333334444444455000001eb0000000000001ec0"), 0x02b1a1), - (hex!("0122222222333333334444444455000001eb0000000000004390"), 0x02b241), - (hex!("0122222222333333334444444455000001eb0000000000005970"), 0x02b2e1), - (hex!("0122222222333333334444444455000001ec0000000000001ed0"), 0x02b381), - (hex!("0122222222333333334444444455000001ec0000000000005d50"), 0x02b421), - (hex!("0122222222333333334444444455000001ec00000000000076e0"), 0x02b4c1), - (hex!("0122222222333333334444444455000001ed0000000000001ee0"), 0x02b561), - (hex!("0122222222333333334444444455000001ed0000000000006190"), 0x02b601), - (hex!("0122222222333333334444444455000001ee0000000000001ef0"), 0x02b6a1), - (hex!("0122222222333333334444444455000001ee0000000000004900"), 0x02b741), - (hex!("0122222222333333334444444455000001ef0000000000001f00"), 0x02b7e1), - (hex!("0122222222333333334444444455000001ef0000000000006c60"), 0x02b881), - (hex!("0122222222333333334444444455000001f00000000000001f10"), 0x02b921), - (hex!("0122222222333333334444444455000001f00000000000006950"), 0x02b9c1), - (hex!("0122222222333333334444444455000001f10000000000001f20"), 0x02ba61), - (hex!("0122222222333333334444444455000001f10000000000006400"), 0x02bb01), - (hex!("0122222222333333334444444455000001f20000000000001f30"), 0x02bba1), - (hex!("0122222222333333334444444455000001f20000000000006f00"), 0x02bc41), - (hex!("0122222222333333334444444455000001f20000000000007b10"), 0x02bce1), - (hex!("0122222222333333334444444455000001f30000000000001f40"), 0x02bd81), - (hex!("0122222222333333334444444455000001f40000000000001f50"), 0x02be21), - (hex!("0122222222333333334444444455000001f50000000000001f60"), 0x02bec1), - (hex!("0122222222333333334444444455000001f500000000000044f0"), 0x02bf61), - (hex!("0122222222333333334444444455000001f60000000000001f70"), 0x02c001), - (hex!("0122222222333333334444444455000001f70000000000001f80"), 0x02c0a1), - (hex!("0122222222333333334444444455000001f70000000000004ad0"), 0x02c141), - (hex!("0122222222333333334444444455000001f80000000000001f90"), 0x02c1e1), - (hex!("0122222222333333334444444455000001f90000000000001fa0"), 0x02c281), - (hex!("0122222222333333334444444455000001f90000000000003f60"), 0x02c321), - (hex!("0122222222333333334444444455000001f90000000000004a80"), 0x02c3c1), - (hex!("0122222222333333334444444455000001fa0000000000001fb0"), 0x02c461), - (hex!("0122222222333333334444444455000001fa0000000000006f90"), 0x02c501), - (hex!("0122222222333333334444444455000001fb0000000000001fc0"), 0x02c5a1), - (hex!("0122222222333333334444444455000001fc0000000000001fd0"), 0x02c641), - (hex!("0122222222333333334444444455000001fc0000000000004a90"), 0x02c6e1), - (hex!("0122222222333333334444444455000001fd0000000000001fe0"), 0x02c781), - (hex!("0122222222333333334444444455000001fd0000000000005f50"), 0x02c821), - (hex!("0122222222333333334444444455000001fe0000000000001ff0"), 0x02c8c1), - (hex!("0122222222333333334444444455000001ff0000000000002000"), 0x02c961), - (hex!("0122222222333333334444444455000002000000000000002010"), 0x02ca01), - (hex!("0122222222333333334444444455000002000000000000005f00"), 0x02caa1), - (hex!("0122222222333333334444444455000002000000000000006840"), 0x02cb41), - (hex!("0122222222333333334444444455000002010000000000002020"), 0x02cbe1), - (hex!("0122222222333333334444444455000002020000000000002030"), 0x02cc81), - (hex!("0122222222333333334444444455000002030000000000002040"), 0x02cd21), - (hex!("0122222222333333334444444455000002040000000000002050"), 0x02cdc1), - (hex!("01222222223333333344444444550000020400000000000051f0"), 0x02ce61), - (hex!("0122222222333333334444444455000002050000000000002060"), 0x02cf01), - (hex!("0122222222333333334444444455000002060000000000002070"), 0x02cfa1), - (hex!("0122222222333333334444444455000002060000000000005c80"), 0x02d041), - (hex!("01222222223333333344444444550000020600000000000061d0"), 0x02d0e1), - (hex!("01222222223333333344444444550000020600000000000078c0"), 0x02d181), - (hex!("0122222222333333334444444455000002070000000000002080"), 0x02d221), - (hex!("0122222222333333334444444455000002070000000000006ba0"), 0x02d2c1), - (hex!("0122222222333333334444444455000002080000000000002090"), 0x02d361), - (hex!("01222222223333333344444444550000020900000000000020a0"), 0x02d401), - (hex!("01222222223333333344444444550000020900000000000067a0"), 0x02d4a1), - (hex!("01222222223333333344444444550000020a00000000000020b0"), 0x02d541), - (hex!("01222222223333333344444444550000020a0000000000004950"), 0x02d5e1), - (hex!("01222222223333333344444444550000020a0000000000004de0"), 0x02d681), - (hex!("01222222223333333344444444550000020b00000000000020c0"), 0x02d721), - (hex!("01222222223333333344444444550000020b0000000000004b00"), 0x02d7c1), - (hex!("01222222223333333344444444550000020c00000000000020d0"), 0x02d861), - (hex!("01222222223333333344444444550000020d00000000000020e0"), 0x02d901), - (hex!("01222222223333333344444444550000020e00000000000020f0"), 0x02d9a1), - (hex!("01222222223333333344444444550000020f0000000000002100"), 0x02da41), - (hex!("0122222222333333334444444455000002100000000000002110"), 0x02dae1), - (hex!("0122222222333333334444444455000002110000000000002120"), 0x02db81), - (hex!("0122222222333333334444444455000002110000000000004490"), 0x02dc21), - (hex!("0122222222333333334444444455000002120000000000002130"), 0x02dcc1), - (hex!("0122222222333333334444444455000002130000000000002140"), 0x02dd61), - (hex!("01222222223333333344444444550000021300000000000046d0"), 0x02de01), - (hex!("01222222223333333344444444550000021300000000000046e0"), 0x02dea1), - (hex!("0122222222333333334444444455000002130000000000004b70"), 0x02df41), - (hex!("0122222222333333334444444455000002140000000000002150"), 0x02dfe1), - (hex!("0122222222333333334444444455000002140000000000006c50"), 0x02e081), - (hex!("0122222222333333334444444455000002150000000000002160"), 0x02e121), - (hex!("01222222223333333344444444550000021500000000000043c0"), 0x02e1c1), - (hex!("0122222222333333334444444455000002160000000000002170"), 0x02e261), - (hex!("01222222223333333344444444550000021600000000000055b0"), 0x02e301), - (hex!("0122222222333333334444444455000002160000000000006150"), 0x02e3a1), - (hex!("0122222222333333334444444455000002170000000000002180"), 0x02e441), - (hex!("01222222223333333344444444550000021700000000000053b0"), 0x02e4e1), - (hex!("0122222222333333334444444455000002170000000000007460"), 0x02e581), - (hex!("0122222222333333334444444455000002180000000000002190"), 0x02e621), - (hex!("01222222223333333344444444550000021900000000000021a0"), 0x02e6c1), - (hex!("01222222223333333344444444550000021a00000000000021b0"), 0x02e761), - (hex!("01222222223333333344444444550000021a0000000000007650"), 0x02e801), - (hex!("01222222223333333344444444550000021b00000000000021c0"), 0x02e8a1), - (hex!("01222222223333333344444444550000021b0000000000004b20"), 0x02e941), - (hex!("01222222223333333344444444550000021c00000000000021d0"), 0x02e9e1), - (hex!("01222222223333333344444444550000021c0000000000007610"), 0x02ea81), - (hex!("01222222223333333344444444550000021d00000000000021e0"), 0x02eb21), - (hex!("01222222223333333344444444550000021d0000000000005f40"), 0x02ebc1), - (hex!("01222222223333333344444444550000021e00000000000021f0"), 0x02ec61), - (hex!("01222222223333333344444444550000021e0000000000005a50"), 0x02ed01), - (hex!("01222222223333333344444444550000021e0000000000005ff0"), 0x02eda1), - (hex!("01222222223333333344444444550000021f0000000000002200"), 0x02ee41), - (hex!("01222222223333333344444444550000021f00000000000043a0"), 0x02eee1), - (hex!("01222222223333333344444444550000021f0000000000004cb0"), 0x02ef81), - (hex!("01222222223333333344444444550000021f0000000000004e00"), 0x02f021), - (hex!("0122222222333333334444444455000002200000000000002210"), 0x02f0c1), - (hex!("0122222222333333334444444455000002210000000000002220"), 0x02f161), - (hex!("0122222222333333334444444455000002210000000000006290"), 0x02f201), - (hex!("0122222222333333334444444455000002210000000000007230"), 0x02f2a1), - (hex!("0122222222333333334444444455000002220000000000002230"), 0x02f341), - (hex!("0122222222333333334444444455000002220000000000006ea0"), 0x02f3e1), - (hex!("0122222222333333334444444455000002230000000000002240"), 0x02f481), - (hex!("0122222222333333334444444455000002230000000000004710"), 0x02f521), - (hex!("0122222222333333334444444455000002240000000000002250"), 0x02f5c1), - (hex!("0122222222333333334444444455000002250000000000002260"), 0x02f661), - (hex!("0122222222333333334444444455000002260000000000002270"), 0x02f701), - (hex!("0122222222333333334444444455000002260000000000005b40"), 0x02f7a1), - (hex!("0122222222333333334444444455000002260000000000006300"), 0x02f841), - (hex!("0122222222333333334444444455000002270000000000002280"), 0x02f8e1), - (hex!("0122222222333333334444444455000002270000000000005b80"), 0x02f981), - (hex!("0122222222333333334444444455000002280000000000002290"), 0x02fa21), - (hex!("0122222222333333334444444455000002280000000000003ed0"), 0x02fac1), - (hex!("0122222222333333334444444455000002280000000000004550"), 0x02fb61), - (hex!("01222222223333333344444444550000022800000000000077d0"), 0x02fc01), - (hex!("01222222223333333344444444550000022900000000000022a0"), 0x02fca1), - (hex!("0122222222333333334444444455000002290000000000006480"), 0x02fd41), - (hex!("01222222223333333344444444550000022a00000000000022b0"), 0x02fde1), - (hex!("01222222223333333344444444550000022a0000000000005450"), 0x02fe81), - (hex!("01222222223333333344444444550000022b00000000000022c0"), 0x02ff21), - (hex!("01222222223333333344444444550000022b0000000000006dd0"), 0x02ffc1), - (hex!("01222222223333333344444444550000022c00000000000022d0"), 0x030061), - (hex!("01222222223333333344444444550000022c0000000000006890"), 0x030101), - (hex!("01222222223333333344444444550000022d00000000000022e0"), 0x0301a1), - (hex!("01222222223333333344444444550000022e00000000000022f0"), 0x030241), - (hex!("01222222223333333344444444550000022e0000000000004f20"), 0x0302e1), - (hex!("01222222223333333344444444550000022f0000000000002300"), 0x030381), - (hex!("01222222223333333344444444550000022f0000000000005260"), 0x030421), - (hex!("01222222223333333344444444550000022f00000000000053f0"), 0x0304c1), - (hex!("0122222222333333334444444455000002300000000000002310"), 0x030561), - (hex!("01222222223333333344444444550000023000000000000050e0"), 0x030601), - (hex!("0122222222333333334444444455000002310000000000002320"), 0x0306a1), - (hex!("0122222222333333334444444455000002310000000000007800"), 0x030741), - (hex!("0122222222333333334444444455000002320000000000002330"), 0x0307e1), - (hex!("0122222222333333334444444455000002330000000000002340"), 0x030881), - (hex!("0122222222333333334444444455000002330000000000004d70"), 0x030921), - (hex!("0122222222333333334444444455000002330000000000005cf0"), 0x0309c1), - (hex!("0122222222333333334444444455000002340000000000002350"), 0x030a61), - (hex!("0122222222333333334444444455000002350000000000002360"), 0x030b01), - (hex!("0122222222333333334444444455000002350000000000006970"), 0x030ba1), - (hex!("0122222222333333334444444455000002360000000000002370"), 0x030c41), - (hex!("0122222222333333334444444455000002360000000000005270"), 0x030ce1), - (hex!("0122222222333333334444444455000002370000000000002380"), 0x030d81), - (hex!("0122222222333333334444444455000002370000000000005d70"), 0x030e21), - (hex!("0122222222333333334444444455000002380000000000002390"), 0x030ec1), - (hex!("01222222223333333344444444550000023800000000000069a0"), 0x030f61), - (hex!("01222222223333333344444444550000023900000000000023a0"), 0x031001), - (hex!("01222222223333333344444444550000023900000000000052e0"), 0x0310a1), - (hex!("0122222222333333334444444455000002390000000000005a10"), 0x031141), - (hex!("0122222222333333334444444455000002390000000000007440"), 0x0311e1), - (hex!("01222222223333333344444444550000023a00000000000023b0"), 0x031281), - (hex!("01222222223333333344444444550000023a0000000000003f00"), 0x031321), - (hex!("01222222223333333344444444550000023a0000000000004430"), 0x0313c1), - (hex!("01222222223333333344444444550000023a0000000000007070"), 0x031461), - (hex!("01222222223333333344444444550000023a00000000000074a0"), 0x031501), - (hex!("01222222223333333344444444550000023b00000000000023c0"), 0x0315a1), - (hex!("01222222223333333344444444550000023b0000000000004730"), 0x031641), - (hex!("01222222223333333344444444550000023b00000000000068b0"), 0x0316e1), - (hex!("01222222223333333344444444550000023c00000000000023d0"), 0x031781), - (hex!("01222222223333333344444444550000023c0000000000004680"), 0x031821), - (hex!("01222222223333333344444444550000023d00000000000023e0"), 0x0318c1), - (hex!("01222222223333333344444444550000023d00000000000059a0"), 0x031961), - (hex!("01222222223333333344444444550000023e00000000000023f0"), 0x031a01), - (hex!("01222222223333333344444444550000023f0000000000002400"), 0x031aa1), - (hex!("0122222222333333334444444455000002400000000000002410"), 0x031b41), - (hex!("0122222222333333334444444455000002400000000000004920"), 0x031be1), - (hex!("01222222223333333344444444550000024000000000000066e0"), 0x031c81), - (hex!("01222222223333333344444444550000024000000000000076f0"), 0x031d21), - (hex!("01222222223333333344444444550000024000000000000078e0"), 0x031dc1), - (hex!("0122222222333333334444444455000002410000000000002420"), 0x031e61), - (hex!("0122222222333333334444444455000002420000000000002430"), 0x031f01), - (hex!("0122222222333333334444444455000002420000000000006590"), 0x031fa1), - (hex!("0122222222333333334444444455000002430000000000002440"), 0x032041), - (hex!("0122222222333333334444444455000002430000000000004d00"), 0x0320e1), - (hex!("0122222222333333334444444455000002440000000000002450"), 0x032181), - (hex!("0122222222333333334444444455000002440000000000005f80"), 0x032221), - (hex!("0122222222333333334444444455000002450000000000002460"), 0x0322c1), - (hex!("0122222222333333334444444455000002450000000000004940"), 0x032361), - (hex!("0122222222333333334444444455000002460000000000002470"), 0x032401), - (hex!("0122222222333333334444444455000002470000000000002480"), 0x0324a1), - (hex!("0122222222333333334444444455000002470000000000004dd0"), 0x032541), - (hex!("0122222222333333334444444455000002470000000000005930"), 0x0325e1), - (hex!("01222222223333333344444444550000024700000000000061b0"), 0x032681), - (hex!("0122222222333333334444444455000002470000000000007740"), 0x032721), - (hex!("0122222222333333334444444455000002480000000000002490"), 0x0327c1), - (hex!("0122222222333333334444444455000002480000000000004890"), 0x032861), - (hex!("01222222223333333344444444550000024900000000000024a0"), 0x032901), - (hex!("01222222223333333344444444550000024a00000000000024b0"), 0x0329a1), - (hex!("01222222223333333344444444550000024b00000000000024c0"), 0x032a41), - (hex!("01222222223333333344444444550000024c00000000000024d0"), 0x032ae1), - (hex!("01222222223333333344444444550000024d00000000000024e0"), 0x032b81), - (hex!("01222222223333333344444444550000024d0000000000004070"), 0x032c21), - (hex!("01222222223333333344444444550000024e00000000000024f0"), 0x032cc1), - (hex!("01222222223333333344444444550000024e00000000000066a0"), 0x032d61), - (hex!("01222222223333333344444444550000024e0000000000006ab0"), 0x032e01), - (hex!("01222222223333333344444444550000024f0000000000002500"), 0x032ea1), - (hex!("0122222222333333334444444455000002500000000000002510"), 0x032f41), - (hex!("0122222222333333334444444455000002510000000000002520"), 0x032fe1), - (hex!("0122222222333333334444444455000002510000000000007320"), 0x033081), - (hex!("0122222222333333334444444455000002520000000000002530"), 0x033121), - (hex!("0122222222333333334444444455000002520000000000006410"), 0x0331c1), - (hex!("0122222222333333334444444455000002530000000000002540"), 0x033261), - (hex!("0122222222333333334444444455000002530000000000005110"), 0x033301), - (hex!("0122222222333333334444444455000002540000000000002550"), 0x0333a1), - (hex!("01222222223333333344444444550000025400000000000040c0"), 0x033441), - (hex!("0122222222333333334444444455000002540000000000006a40"), 0x0334e1), - (hex!("0122222222333333334444444455000002550000000000002560"), 0x033581), - (hex!("0122222222333333334444444455000002550000000000005190"), 0x033621), - (hex!("0122222222333333334444444455000002560000000000002570"), 0x0336c1), - (hex!("01222222223333333344444444550000025600000000000061f0"), 0x033761), - (hex!("0122222222333333334444444455000002570000000000002580"), 0x033801), - (hex!("0122222222333333334444444455000002580000000000002590"), 0x0338a1), - (hex!("01222222223333333344444444550000025800000000000043d0"), 0x033941), - (hex!("01222222223333333344444444550000025900000000000025a0"), 0x0339e1), - (hex!("0122222222333333334444444455000002590000000000006bb0"), 0x033a81), - (hex!("01222222223333333344444444550000025a00000000000025b0"), 0x033b21), - (hex!("01222222223333333344444444550000025a0000000000005fb0"), 0x033bc1), - (hex!("01222222223333333344444444550000025a00000000000064c0"), 0x033c61), - (hex!("01222222223333333344444444550000025b00000000000025c0"), 0x033d01), - (hex!("01222222223333333344444444550000025b0000000000005c10"), 0x033da1), - (hex!("01222222223333333344444444550000025c00000000000025d0"), 0x033e41), - (hex!("01222222223333333344444444550000025c0000000000007d00"), 0x033ee1), - (hex!("01222222223333333344444444550000025d00000000000025e0"), 0x033f81), - (hex!("01222222223333333344444444550000025e00000000000025f0"), 0x034021), - (hex!("01222222223333333344444444550000025e00000000000045e0"), 0x0340c1), - (hex!("01222222223333333344444444550000025e0000000000006ee0"), 0x034161), - (hex!("01222222223333333344444444550000025f0000000000002600"), 0x034201), - (hex!("01222222223333333344444444550000025f00000000000050b0"), 0x0342a1), - (hex!("01222222223333333344444444550000025f0000000000007690"), 0x034341), - (hex!("0122222222333333334444444455000002600000000000002610"), 0x0343e1), - (hex!("0122222222333333334444444455000002600000000000007b60"), 0x034481), - (hex!("0122222222333333334444444455000002610000000000002620"), 0x034521), - (hex!("0122222222333333334444444455000002620000000000002630"), 0x0345c1), - (hex!("0122222222333333334444444455000002630000000000002640"), 0x034661), - (hex!("0122222222333333334444444455000002640000000000002650"), 0x034701), - (hex!("0122222222333333334444444455000002650000000000002660"), 0x0347a1), - (hex!("0122222222333333334444444455000002650000000000006180"), 0x034841), - (hex!("0122222222333333334444444455000002660000000000002670"), 0x0348e1), - (hex!("0122222222333333334444444455000002660000000000005430"), 0x034981), - (hex!("0122222222333333334444444455000002660000000000007a60"), 0x034a21), - (hex!("0122222222333333334444444455000002670000000000002680"), 0x034ac1), - (hex!("01222222223333333344444444550000026700000000000077f0"), 0x034b61), - (hex!("0122222222333333334444444455000002680000000000002690"), 0x034c01), - (hex!("01222222223333333344444444550000026900000000000026a0"), 0x034ca1), - (hex!("01222222223333333344444444550000026a00000000000026b0"), 0x034d41), - (hex!("01222222223333333344444444550000026a0000000000007530"), 0x034de1), - (hex!("01222222223333333344444444550000026b00000000000026c0"), 0x034e81), - (hex!("01222222223333333344444444550000026b00000000000058b0"), 0x034f21), - (hex!("01222222223333333344444444550000026b00000000000066b0"), 0x034fc1), - (hex!("01222222223333333344444444550000026b0000000000006b10"), 0x035061), - (hex!("01222222223333333344444444550000026c00000000000026d0"), 0x035101), - (hex!("01222222223333333344444444550000026d00000000000026e0"), 0x0351a1), - (hex!("01222222223333333344444444550000026d0000000000004210"), 0x035241), - (hex!("01222222223333333344444444550000026d0000000000005490"), 0x0352e1), - (hex!("01222222223333333344444444550000026d0000000000005e60"), 0x035381), - (hex!("01222222223333333344444444550000026d00000000000068e0"), 0x035421), - (hex!("01222222223333333344444444550000026d0000000000007020"), 0x0354c1), - (hex!("01222222223333333344444444550000026d0000000000007300"), 0x035561), - (hex!("01222222223333333344444444550000026e00000000000026f0"), 0x035601), - (hex!("01222222223333333344444444550000026f0000000000002700"), 0x0356a1), - (hex!("01222222223333333344444444550000026f0000000000004910"), 0x035741), - (hex!("0122222222333333334444444455000002700000000000002710"), 0x0357e1), - (hex!("0122222222333333334444444455000002710000000000002720"), 0x035881), - (hex!("01222222223333333344444444550000027100000000000050c0"), 0x035921), - (hex!("0122222222333333334444444455000002720000000000002730"), 0x0359c1), - (hex!("0122222222333333334444444455000002730000000000002740"), 0x035a61), - (hex!("0122222222333333334444444455000002740000000000002750"), 0x035b01), - (hex!("0122222222333333334444444455000002740000000000007490"), 0x035ba1), - (hex!("0122222222333333334444444455000002750000000000002760"), 0x035c41), - (hex!("0122222222333333334444444455000002760000000000002770"), 0x035ce1), - (hex!("0122222222333333334444444455000002760000000000004790"), 0x035d81), - (hex!("0122222222333333334444444455000002770000000000002780"), 0x035e21), - (hex!("01222222223333333344444444550000027700000000000050a0"), 0x035ec1), - (hex!("0122222222333333334444444455000002780000000000002790"), 0x035f61), - (hex!("0122222222333333334444444455000002780000000000004330"), 0x036001), - (hex!("0122222222333333334444444455000002780000000000006b00"), 0x0360a1), - (hex!("01222222223333333344444444550000027900000000000027a0"), 0x036141), - (hex!("01222222223333333344444444550000027a00000000000027b0"), 0x0361e1), - (hex!("01222222223333333344444444550000027b00000000000027c0"), 0x036281), - (hex!("01222222223333333344444444550000027b0000000000004930"), 0x036321), - (hex!("01222222223333333344444444550000027b0000000000006250"), 0x0363c1), - (hex!("01222222223333333344444444550000027c00000000000027d0"), 0x036461), - (hex!("01222222223333333344444444550000027d00000000000027e0"), 0x036501), - (hex!("01222222223333333344444444550000027d0000000000005ce0"), 0x0365a1), - (hex!("01222222223333333344444444550000027d0000000000005fe0"), 0x036641), - (hex!("01222222223333333344444444550000027e00000000000027f0"), 0x0366e1), - (hex!("01222222223333333344444444550000027f0000000000002800"), 0x036781), - (hex!("01222222223333333344444444550000027f0000000000003e90"), 0x036821), - (hex!("01222222223333333344444444550000027f0000000000007910"), 0x0368c1), - (hex!("0122222222333333334444444455000002800000000000002810"), 0x036961), - (hex!("0122222222333333334444444455000002800000000000004990"), 0x036a01), - (hex!("0122222222333333334444444455000002800000000000006160"), 0x036aa1), - (hex!("0122222222333333334444444455000002800000000000006740"), 0x036b41), - (hex!("0122222222333333334444444455000002810000000000002820"), 0x036be1), - (hex!("0122222222333333334444444455000002820000000000002830"), 0x036c81), - (hex!("0122222222333333334444444455000002820000000000005170"), 0x036d21), - (hex!("0122222222333333334444444455000002830000000000002840"), 0x036dc1), - (hex!("0122222222333333334444444455000002840000000000002850"), 0x036e61), - (hex!("0122222222333333334444444455000002840000000000004810"), 0x036f01), - (hex!("0122222222333333334444444455000002840000000000006aa0"), 0x036fa1), - (hex!("0122222222333333334444444455000002850000000000002860"), 0x037041), - (hex!("0122222222333333334444444455000002860000000000002870"), 0x0370e1), - (hex!("0122222222333333334444444455000002860000000000005080"), 0x037181), - (hex!("0122222222333333334444444455000002870000000000002880"), 0x037221), - (hex!("0122222222333333334444444455000002870000000000004e60"), 0x0372c1), - (hex!("0122222222333333334444444455000002880000000000002890"), 0x037361), - (hex!("0122222222333333334444444455000002880000000000005060"), 0x037401), - (hex!("0122222222333333334444444455000002880000000000006f20"), 0x0374a1), - (hex!("01222222223333333344444444550000028900000000000028a0"), 0x037541), - (hex!("01222222223333333344444444550000028900000000000047e0"), 0x0375e1), - (hex!("01222222223333333344444444550000028a00000000000028b0"), 0x037681), - (hex!("01222222223333333344444444550000028a0000000000005ab0"), 0x037721), - (hex!("01222222223333333344444444550000028a0000000000007130"), 0x0377c1), - (hex!("01222222223333333344444444550000028a0000000000007660"), 0x037861), - (hex!("01222222223333333344444444550000028b00000000000028c0"), 0x037901), - (hex!("01222222223333333344444444550000028b00000000000054e0"), 0x0379a1), - (hex!("01222222223333333344444444550000028c00000000000028d0"), 0x037a41), - (hex!("01222222223333333344444444550000028c00000000000046f0"), 0x037ae1), - (hex!("01222222223333333344444444550000028c00000000000061a0"), 0x037b81), - (hex!("01222222223333333344444444550000028d00000000000028e0"), 0x037c21), - (hex!("01222222223333333344444444550000028e00000000000028f0"), 0x037cc1), - (hex!("01222222223333333344444444550000028e0000000000004130"), 0x037d61), - (hex!("01222222223333333344444444550000028f0000000000002900"), 0x037e01), - (hex!("01222222223333333344444444550000028f0000000000007510"), 0x037ea1), - (hex!("0122222222333333334444444455000002900000000000002910"), 0x037f41), - (hex!("0122222222333333334444444455000002900000000000004a40"), 0x037fe1), - (hex!("0122222222333333334444444455000002910000000000002920"), 0x038081), - (hex!("0122222222333333334444444455000002920000000000002930"), 0x038121), - (hex!("0122222222333333334444444455000002920000000000004e90"), 0x0381c1), - (hex!("0122222222333333334444444455000002930000000000002940"), 0x038261), - (hex!("0122222222333333334444444455000002930000000000006880"), 0x038301), - (hex!("0122222222333333334444444455000002940000000000002950"), 0x0383a1), - (hex!("0122222222333333334444444455000002940000000000007bc0"), 0x038441), - (hex!("0122222222333333334444444455000002950000000000002960"), 0x0384e1), - (hex!("0122222222333333334444444455000002960000000000002970"), 0x038581), - (hex!("01222222223333333344444444550000029600000000000059d0"), 0x038621), - (hex!("0122222222333333334444444455000002970000000000002980"), 0x0386c1), - (hex!("0122222222333333334444444455000002970000000000004a50"), 0x038761), - (hex!("0122222222333333334444444455000002970000000000005f20"), 0x038801), - (hex!("01222222223333333344444444550000029700000000000068d0"), 0x0388a1), - (hex!("0122222222333333334444444455000002980000000000002990"), 0x038941), - (hex!("0122222222333333334444444455000002980000000000004370"), 0x0389e1), - (hex!("0122222222333333334444444455000002980000000000004420"), 0x038a81), - (hex!("01222222223333333344444444550000029900000000000029a0"), 0x038b21), - (hex!("01222222223333333344444444550000029a00000000000029b0"), 0x038bc1), - (hex!("01222222223333333344444444550000029a0000000000006010"), 0x038c61), - (hex!("01222222223333333344444444550000029a0000000000006980"), 0x038d01), - (hex!("01222222223333333344444444550000029b00000000000029c0"), 0x038da1), - (hex!("01222222223333333344444444550000029c00000000000029d0"), 0x038e41), - (hex!("01222222223333333344444444550000029c0000000000007480"), 0x038ee1), - (hex!("01222222223333333344444444550000029d00000000000029e0"), 0x038f81), - (hex!("01222222223333333344444444550000029d0000000000005030"), 0x039021), - (hex!("01222222223333333344444444550000029d0000000000007780"), 0x0390c1), - (hex!("01222222223333333344444444550000029d0000000000007a50"), 0x039161), - (hex!("01222222223333333344444444550000029e00000000000029f0"), 0x039201), - (hex!("01222222223333333344444444550000029e00000000000074b0"), 0x0392a1), - (hex!("01222222223333333344444444550000029f0000000000002a00"), 0x039341), - (hex!("0122222222333333334444444455000002a00000000000002a10"), 0x0393e1), - (hex!("0122222222333333334444444455000002a10000000000002a20"), 0x039481), - (hex!("0122222222333333334444444455000002a20000000000002a30"), 0x039521), - (hex!("0122222222333333334444444455000002a20000000000004c50"), 0x0395c1), - (hex!("0122222222333333334444444455000002a20000000000006f10"), 0x039661), - (hex!("0122222222333333334444444455000002a30000000000002a40"), 0x039701), - (hex!("0122222222333333334444444455000002a40000000000002a50"), 0x0397a1), - (hex!("0122222222333333334444444455000002a40000000000005d60"), 0x039841), - (hex!("0122222222333333334444444455000002a50000000000002a60"), 0x0398e1), - (hex!("0122222222333333334444444455000002a50000000000005440"), 0x039981), - (hex!("0122222222333333334444444455000002a50000000000005890"), 0x039a21), - (hex!("0122222222333333334444444455000002a60000000000002a70"), 0x039ac1), - (hex!("0122222222333333334444444455000002a70000000000002a80"), 0x039b61), - (hex!("0122222222333333334444444455000002a700000000000054a0"), 0x039c01), - (hex!("0122222222333333334444444455000002a70000000000007280"), 0x039ca1), - (hex!("0122222222333333334444444455000002a80000000000002a90"), 0x039d41), - (hex!("0122222222333333334444444455000002a90000000000002aa0"), 0x039de1), - (hex!("0122222222333333334444444455000002aa0000000000002ab0"), 0x039e81), - (hex!("0122222222333333334444444455000002ab0000000000002ac0"), 0x039f21), - (hex!("0122222222333333334444444455000002ab0000000000006c90"), 0x039fc1), - (hex!("0122222222333333334444444455000002ac0000000000002ad0"), 0x03a061), - (hex!("0122222222333333334444444455000002ac0000000000006db0"), 0x03a101), - (hex!("0122222222333333334444444455000002ad0000000000002ae0"), 0x03a1a1), - (hex!("0122222222333333334444444455000002ad00000000000065e0"), 0x03a241), - (hex!("0122222222333333334444444455000002ad0000000000007b40"), 0x03a2e1), - (hex!("0122222222333333334444444455000002ae0000000000002af0"), 0x03a381), - (hex!("0122222222333333334444444455000002ae0000000000004d20"), 0x03a421), - (hex!("0122222222333333334444444455000002ae0000000000006f30"), 0x03a4c1), - (hex!("0122222222333333334444444455000002af0000000000002b00"), 0x03a561), - (hex!("0122222222333333334444444455000002b00000000000002b10"), 0x03a601), - (hex!("0122222222333333334444444455000002b00000000000004560"), 0x03a6a1), - (hex!("0122222222333333334444444455000002b00000000000005800"), 0x03a741), - (hex!("0122222222333333334444444455000002b00000000000005a60"), 0x03a7e1), - (hex!("0122222222333333334444444455000002b10000000000002b20"), 0x03a881), - (hex!("0122222222333333334444444455000002b10000000000007b30"), 0x03a921), - (hex!("0122222222333333334444444455000002b20000000000002b30"), 0x03a9c1), - (hex!("0122222222333333334444444455000002b20000000000004440"), 0x03aa61), - (hex!("0122222222333333334444444455000002b20000000000004f80"), 0x03ab01), - (hex!("0122222222333333334444444455000002b20000000000005020"), 0x03aba1), - (hex!("0122222222333333334444444455000002b30000000000002b40"), 0x03ac41), - (hex!("0122222222333333334444444455000002b40000000000002b50"), 0x03ace1), - (hex!("0122222222333333334444444455000002b50000000000002b60"), 0x03ad81), - (hex!("0122222222333333334444444455000002b500000000000059e0"), 0x03ae21), - (hex!("0122222222333333334444444455000002b60000000000002b70"), 0x03aec1), - (hex!("0122222222333333334444444455000002b70000000000002b80"), 0x03af61), - (hex!("0122222222333333334444444455000002b80000000000002b90"), 0x03b001), - (hex!("0122222222333333334444444455000002b80000000000004590"), 0x03b0a1), - (hex!("0122222222333333334444444455000002b800000000000047d0"), 0x03b141), - (hex!("0122222222333333334444444455000002b80000000000006030"), 0x03b1e1), - (hex!("0122222222333333334444444455000002b80000000000006a20"), 0x03b281), - (hex!("0122222222333333334444444455000002b80000000000006a90"), 0x03b321), - (hex!("0122222222333333334444444455000002b90000000000002ba0"), 0x03b3c1), - (hex!("0122222222333333334444444455000002ba0000000000002bb0"), 0x03b461), - (hex!("0122222222333333334444444455000002ba0000000000006e80"), 0x03b501), - (hex!("0122222222333333334444444455000002bb0000000000002bc0"), 0x03b5a1), - (hex!("0122222222333333334444444455000002bc0000000000002bd0"), 0x03b641), - (hex!("0122222222333333334444444455000002bc0000000000004b30"), 0x03b6e1), - (hex!("0122222222333333334444444455000002bd0000000000002be0"), 0x03b781), - (hex!("0122222222333333334444444455000002bd0000000000005e10"), 0x03b821), - (hex!("0122222222333333334444444455000002be0000000000002bf0"), 0x03b8c1), - (hex!("0122222222333333334444444455000002bf0000000000002c00"), 0x03b961), - (hex!("0122222222333333334444444455000002c00000000000002c10"), 0x03ba01), - (hex!("0122222222333333334444444455000002c10000000000002c20"), 0x03baa1), - (hex!("0122222222333333334444444455000002c10000000000003ef0"), 0x03bb41), - (hex!("0122222222333333334444444455000002c20000000000002c30"), 0x03bbe1), - (hex!("0122222222333333334444444455000002c200000000000056e0"), 0x03bc81), - (hex!("0122222222333333334444444455000002c30000000000002c40"), 0x03bd21), - (hex!("0122222222333333334444444455000002c30000000000004b60"), 0x03bdc1), - (hex!("0122222222333333334444444455000002c40000000000002c50"), 0x03be61), - (hex!("0122222222333333334444444455000002c400000000000045f0"), 0x03bf01), - (hex!("0122222222333333334444444455000002c40000000000005290"), 0x03bfa1), - (hex!("0122222222333333334444444455000002c50000000000002c60"), 0x03c041), - (hex!("0122222222333333334444444455000002c60000000000002c70"), 0x03c0e1), - (hex!("0122222222333333334444444455000002c60000000000006ae0"), 0x03c181), - (hex!("0122222222333333334444444455000002c70000000000002c80"), 0x03c221), - (hex!("0122222222333333334444444455000002c70000000000005680"), 0x03c2c1), - (hex!("0122222222333333334444444455000002c70000000000006e10"), 0x03c361), - (hex!("0122222222333333334444444455000002c80000000000002c90"), 0x03c401), - (hex!("0122222222333333334444444455000002c90000000000002ca0"), 0x03c4a1), - (hex!("0122222222333333334444444455000002ca0000000000002cb0"), 0x03c541), - (hex!("0122222222333333334444444455000002cb0000000000002cc0"), 0x03c5e1), - (hex!("0122222222333333334444444455000002cc0000000000002cd0"), 0x03c681), - (hex!("0122222222333333334444444455000002cc0000000000005b50"), 0x03c721), - (hex!("0122222222333333334444444455000002cd0000000000002ce0"), 0x03c7c1), - (hex!("0122222222333333334444444455000002ce0000000000002cf0"), 0x03c861), - (hex!("0122222222333333334444444455000002ce00000000000043f0"), 0x03c901), - (hex!("0122222222333333334444444455000002ce0000000000006420"), 0x03c9a1), - (hex!("0122222222333333334444444455000002cf0000000000002d00"), 0x03ca41), - (hex!("0122222222333333334444444455000002d00000000000002d10"), 0x03cae1), - (hex!("0122222222333333334444444455000002d10000000000002d20"), 0x03cb81), - (hex!("0122222222333333334444444455000002d10000000000005370"), 0x03cc21), - (hex!("0122222222333333334444444455000002d20000000000002d30"), 0x03ccc1), - (hex!("0122222222333333334444444455000002d20000000000005ef0"), 0x03cd61), - (hex!("0122222222333333334444444455000002d20000000000006570"), 0x03ce01), - (hex!("0122222222333333334444444455000002d30000000000002d40"), 0x03cea1), - (hex!("0122222222333333334444444455000002d30000000000007360"), 0x03cf41), - (hex!("0122222222333333334444444455000002d40000000000002d50"), 0x03cfe1), - (hex!("0122222222333333334444444455000002d400000000000079a0"), 0x03d081), - (hex!("0122222222333333334444444455000002d50000000000002d60"), 0x03d121), - (hex!("0122222222333333334444444455000002d50000000000004250"), 0x03d1c1), - (hex!("0122222222333333334444444455000002d50000000000006050"), 0x03d261), - (hex!("0122222222333333334444444455000002d60000000000002d70"), 0x03d301), - (hex!("0122222222333333334444444455000002d60000000000007080"), 0x03d3a1), - (hex!("0122222222333333334444444455000002d70000000000002d80"), 0x03d441), - (hex!("0122222222333333334444444455000002d80000000000002d90"), 0x03d4e1), - (hex!("0122222222333333334444444455000002d80000000000007110"), 0x03d581), - (hex!("0122222222333333334444444455000002d800000000000073c0"), 0x03d621), - (hex!("0122222222333333334444444455000002d800000000000075a0"), 0x03d6c1), - (hex!("0122222222333333334444444455000002d90000000000002da0"), 0x03d761), - (hex!("0122222222333333334444444455000002d90000000000004860"), 0x03d801), - (hex!("0122222222333333334444444455000002d90000000000006b60"), 0x03d8a1), - (hex!("0122222222333333334444444455000002da0000000000002db0"), 0x03d941), - (hex!("0122222222333333334444444455000002da0000000000006630"), 0x03d9e1), - (hex!("0122222222333333334444444455000002db0000000000002dc0"), 0x03da81), - (hex!("0122222222333333334444444455000002dc0000000000002dd0"), 0x03db21), - (hex!("0122222222333333334444444455000002dc0000000000004830"), 0x03dbc1), - (hex!("0122222222333333334444444455000002dd0000000000002de0"), 0x03dc61), - (hex!("0122222222333333334444444455000002de0000000000002df0"), 0x03dd01), - (hex!("0122222222333333334444444455000002de0000000000004f00"), 0x03dda1), - (hex!("0122222222333333334444444455000002df0000000000002e00"), 0x03de41), - (hex!("0122222222333333334444444455000002e00000000000002e10"), 0x03dee1), - (hex!("0122222222333333334444444455000002e10000000000002e20"), 0x03df81), - (hex!("0122222222333333334444444455000002e10000000000006e90"), 0x03e021), - (hex!("0122222222333333334444444455000002e20000000000002e30"), 0x03e0c1), - (hex!("0122222222333333334444444455000002e200000000000053e0"), 0x03e161), - (hex!("0122222222333333334444444455000002e30000000000002e40"), 0x03e201), - (hex!("0122222222333333334444444455000002e30000000000006020"), 0x03e2a1), - (hex!("0122222222333333334444444455000002e30000000000006540"), 0x03e341), - (hex!("0122222222333333334444444455000002e40000000000002e50"), 0x03e3e1), - (hex!("0122222222333333334444444455000002e50000000000002e60"), 0x03e481), - (hex!("0122222222333333334444444455000002e50000000000005180"), 0x03e521), - (hex!("0122222222333333334444444455000002e50000000000007bf0"), 0x03e5c1), - (hex!("0122222222333333334444444455000002e60000000000002e70"), 0x03e661), - (hex!("0122222222333333334444444455000002e60000000000005350"), 0x03e701), - (hex!("0122222222333333334444444455000002e60000000000007960"), 0x03e7a1), - (hex!("0122222222333333334444444455000002e70000000000002e80"), 0x03e841), - (hex!("0122222222333333334444444455000002e80000000000002e90"), 0x03e8e1), - (hex!("0122222222333333334444444455000002e90000000000002ea0"), 0x03e981), - (hex!("0122222222333333334444444455000002ea0000000000002eb0"), 0x03ea21), - (hex!("0122222222333333334444444455000002eb0000000000002ec0"), 0x03eac1), - (hex!("0122222222333333334444444455000002ec0000000000002ed0"), 0x03eb61), - (hex!("0122222222333333334444444455000002ec0000000000006c10"), 0x03ec01), - (hex!("0122222222333333334444444455000002ed0000000000002ee0"), 0x03eca1), - (hex!("0122222222333333334444444455000002ed0000000000005590"), 0x03ed41), - (hex!("0122222222333333334444444455000002ed0000000000005cd0"), 0x03ede1), - (hex!("0122222222333333334444444455000002ed0000000000006910"), 0x03ee81), - (hex!("0122222222333333334444444455000002ee0000000000002ef0"), 0x03ef21), - (hex!("0122222222333333334444444455000002ef0000000000002f00"), 0x03efc1), - (hex!("0122222222333333334444444455000002ef0000000000004ed0"), 0x03f061), - (hex!("0122222222333333334444444455000002f00000000000002f10"), 0x03f101), - (hex!("0122222222333333334444444455000002f00000000000004cf0"), 0x03f1a1), - (hex!("0122222222333333334444444455000002f00000000000005d10"), 0x03f241), - (hex!("0122222222333333334444444455000002f00000000000006860"), 0x03f2e1), - (hex!("0122222222333333334444444455000002f00000000000006b50"), 0x03f381), - (hex!("0122222222333333334444444455000002f00000000000007100"), 0x03f421), - (hex!("0122222222333333334444444455000002f00000000000007aa0"), 0x03f4c1), - (hex!("0122222222333333334444444455000002f10000000000002f20"), 0x03f561), - (hex!("0122222222333333334444444455000002f20000000000002f30"), 0x03f601), - (hex!("0122222222333333334444444455000002f200000000000044b0"), 0x03f6a1), - (hex!("0122222222333333334444444455000002f30000000000002f40"), 0x03f741), - (hex!("0122222222333333334444444455000002f300000000000075b0"), 0x03f7e1), - (hex!("0122222222333333334444444455000002f40000000000002f50"), 0x03f881), - (hex!("0122222222333333334444444455000002f400000000000060f0"), 0x03f921), - (hex!("0122222222333333334444444455000002f50000000000002f60"), 0x03f9c1), - (hex!("0122222222333333334444444455000002f50000000000007210"), 0x03fa61), - (hex!("0122222222333333334444444455000002f60000000000002f70"), 0x03fb01), - (hex!("0122222222333333334444444455000002f60000000000006610"), 0x03fba1), - (hex!("0122222222333333334444444455000002f70000000000002f80"), 0x03fc41), - (hex!("0122222222333333334444444455000002f70000000000007560"), 0x03fce1), - (hex!("0122222222333333334444444455000002f80000000000002f90"), 0x03fd81), - (hex!("0122222222333333334444444455000002f80000000000006320"), 0x03fe21), - (hex!("0122222222333333334444444455000002f90000000000002fa0"), 0x03fec1), - (hex!("0122222222333333334444444455000002f90000000000006e50"), 0x03ff61), - (hex!("0122222222333333334444444455000002fa0000000000002fb0"), 0x040001), - (hex!("0122222222333333334444444455000002fb0000000000002fc0"), 0x0400a1), - (hex!("0122222222333333334444444455000002fb0000000000004780"), 0x040141), - (hex!("0122222222333333334444444455000002fc0000000000002fd0"), 0x0401e1), - (hex!("0122222222333333334444444455000002fd0000000000002fe0"), 0x040281), - (hex!("0122222222333333334444444455000002fd0000000000005600"), 0x040321), - (hex!("0122222222333333334444444455000002fd0000000000006c00"), 0x0403c1), - (hex!("0122222222333333334444444455000002fe0000000000002ff0"), 0x040461), - (hex!("0122222222333333334444444455000002ff0000000000003000"), 0x040501), - (hex!("0122222222333333334444444455000003000000000000003010"), 0x0405a1), - (hex!("0122222222333333334444444455000003000000000000004080"), 0x040641), - (hex!("0122222222333333334444444455000003010000000000003020"), 0x0406e1), - (hex!("0122222222333333334444444455000003010000000000006340"), 0x040781), - (hex!("0122222222333333334444444455000003020000000000003030"), 0x040821), - (hex!("0122222222333333334444444455000003020000000000005b00"), 0x0408c1), - (hex!("0122222222333333334444444455000003020000000000007b20"), 0x040961), - (hex!("0122222222333333334444444455000003030000000000003040"), 0x040a01), - (hex!("01222222223333333344444444550000030300000000000056b0"), 0x040aa1), - (hex!("0122222222333333334444444455000003030000000000006280"), 0x040b41), - (hex!("0122222222333333334444444455000003030000000000007ad0"), 0x040be1), - (hex!("0122222222333333334444444455000003040000000000003050"), 0x040c81), - (hex!("0122222222333333334444444455000003040000000000005c50"), 0x040d21), - (hex!("0122222222333333334444444455000003050000000000003060"), 0x040dc1), - (hex!("01222222223333333344444444550000030500000000000072e0"), 0x040e61), - (hex!("0122222222333333334444444455000003060000000000003070"), 0x040f01), - (hex!("0122222222333333334444444455000003060000000000004360"), 0x040fa1), - (hex!("0122222222333333334444444455000003060000000000004380"), 0x041041), - (hex!("0122222222333333334444444455000003060000000000004820"), 0x0410e1), - (hex!("0122222222333333334444444455000003060000000000006d10"), 0x041181), - (hex!("0122222222333333334444444455000003070000000000003080"), 0x041221), - (hex!("0122222222333333334444444455000003070000000000004450"), 0x0412c1), - (hex!("0122222222333333334444444455000003080000000000003090"), 0x041361), - (hex!("0122222222333333334444444455000003080000000000005ad0"), 0x041401), - (hex!("01222222223333333344444444550000030900000000000030a0"), 0x0414a1), - (hex!("01222222223333333344444444550000030a00000000000030b0"), 0x041541), - (hex!("01222222223333333344444444550000030a0000000000007760"), 0x0415e1), - (hex!("01222222223333333344444444550000030b00000000000030c0"), 0x041681), - (hex!("01222222223333333344444444550000030b0000000000007a80"), 0x041721), - (hex!("01222222223333333344444444550000030c00000000000030d0"), 0x0417c1), - (hex!("01222222223333333344444444550000030d00000000000030e0"), 0x041861), - (hex!("01222222223333333344444444550000030d0000000000003eb0"), 0x041901), - (hex!("01222222223333333344444444550000030e00000000000030f0"), 0x0419a1), - (hex!("01222222223333333344444444550000030f0000000000003100"), 0x041a41), - (hex!("01222222223333333344444444550000030f0000000000004690"), 0x041ae1), - (hex!("01222222223333333344444444550000030f0000000000006900"), 0x041b81), - (hex!("0122222222333333334444444455000003100000000000003110"), 0x041c21), - (hex!("01222222223333333344444444550000031000000000000058a0"), 0x041cc1), - (hex!("0122222222333333334444444455000003110000000000003120"), 0x041d61), - (hex!("0122222222333333334444444455000003110000000000004200"), 0x041e01), - (hex!("0122222222333333334444444455000003120000000000003130"), 0x041ea1), - (hex!("0122222222333333334444444455000003130000000000003140"), 0x041f41), - (hex!("0122222222333333334444444455000003130000000000004d50"), 0x041fe1), - (hex!("0122222222333333334444444455000003130000000000005400"), 0x042081), - (hex!("0122222222333333334444444455000003130000000000005520"), 0x042121), - (hex!("0122222222333333334444444455000003140000000000003150"), 0x0421c1), - (hex!("0122222222333333334444444455000003140000000000006450"), 0x042261), - (hex!("0122222222333333334444444455000003150000000000003160"), 0x042301), - (hex!("01222222223333333344444444550000031500000000000062d0"), 0x0423a1), - (hex!("0122222222333333334444444455000003160000000000003170"), 0x042441), - (hex!("0122222222333333334444444455000003160000000000004c40"), 0x0424e1), - (hex!("0122222222333333334444444455000003160000000000007c80"), 0x042581), - (hex!("0122222222333333334444444455000003170000000000003180"), 0x042621), - (hex!("0122222222333333334444444455000003170000000000004400"), 0x0426c1), - (hex!("0122222222333333334444444455000003170000000000005090"), 0x042761), - (hex!("0122222222333333334444444455000003170000000000006cb0"), 0x042801), - (hex!("0122222222333333334444444455000003180000000000003190"), 0x0428a1), - (hex!("0122222222333333334444444455000003180000000000006560"), 0x042941), - (hex!("01222222223333333344444444550000031900000000000031a0"), 0x0429e1), - (hex!("01222222223333333344444444550000031900000000000052d0"), 0x042a81), - (hex!("01222222223333333344444444550000031900000000000057e0"), 0x042b21), - (hex!("01222222223333333344444444550000031a00000000000031b0"), 0x042bc1), - (hex!("01222222223333333344444444550000031a00000000000071e0"), 0x042c61), - (hex!("01222222223333333344444444550000031b00000000000031c0"), 0x042d01), - (hex!("01222222223333333344444444550000031c00000000000031d0"), 0x042da1), - (hex!("01222222223333333344444444550000031c0000000000004480"), 0x042e41), - (hex!("01222222223333333344444444550000031c0000000000005790"), 0x042ee1), - (hex!("01222222223333333344444444550000031c0000000000007be0"), 0x042f81), - (hex!("01222222223333333344444444550000031d00000000000031e0"), 0x043021), - (hex!("01222222223333333344444444550000031d0000000000005560"), 0x0430c1), - (hex!("01222222223333333344444444550000031e00000000000031f0"), 0x043161), - (hex!("01222222223333333344444444550000031f0000000000003200"), 0x043201), - (hex!("01222222223333333344444444550000031f0000000000004190"), 0x0432a1), - (hex!("0122222222333333334444444455000003200000000000003210"), 0x043341), - (hex!("0122222222333333334444444455000003210000000000003220"), 0x0433e1), - (hex!("0122222222333333334444444455000003220000000000003230"), 0x043481), - (hex!("0122222222333333334444444455000003230000000000003240"), 0x043521), - (hex!("01222222223333333344444444550000032300000000000069d0"), 0x0435c1), - (hex!("0122222222333333334444444455000003240000000000003250"), 0x043661), - (hex!("0122222222333333334444444455000003250000000000003260"), 0x043701), - (hex!("01222222223333333344444444550000032500000000000042b0"), 0x0437a1), - (hex!("01222222223333333344444444550000032500000000000064e0"), 0x043841), - (hex!("0122222222333333334444444455000003260000000000003270"), 0x0438e1), - (hex!("0122222222333333334444444455000003270000000000003280"), 0x043981), - (hex!("0122222222333333334444444455000003270000000000005b20"), 0x043a21), - (hex!("0122222222333333334444444455000003270000000000006330"), 0x043ac1), - (hex!("0122222222333333334444444455000003270000000000006810"), 0x043b61), - (hex!("0122222222333333334444444455000003280000000000003290"), 0x043c01), - (hex!("01222222223333333344444444550000032900000000000032a0"), 0x043ca1), - (hex!("01222222223333333344444444550000032900000000000056f0"), 0x043d41), - (hex!("0122222222333333334444444455000003290000000000005e20"), 0x043de1), - (hex!("0122222222333333334444444455000003290000000000005e70"), 0x043e81), - (hex!("01222222223333333344444444550000032a00000000000032b0"), 0x043f21), - (hex!("01222222223333333344444444550000032b00000000000032c0"), 0x043fc1), - (hex!("01222222223333333344444444550000032b0000000000005500"), 0x044061), - (hex!("01222222223333333344444444550000032b0000000000005a20"), 0x044101), - (hex!("01222222223333333344444444550000032c00000000000032d0"), 0x0441a1), - (hex!("01222222223333333344444444550000032c0000000000004060"), 0x044241), - (hex!("01222222223333333344444444550000032c0000000000004760"), 0x0442e1), - (hex!("01222222223333333344444444550000032d00000000000032e0"), 0x044381), - (hex!("01222222223333333344444444550000032d00000000000068a0"), 0x044421), - (hex!("01222222223333333344444444550000032e00000000000032f0"), 0x0444c1), - (hex!("01222222223333333344444444550000032f0000000000003300"), 0x044561), - (hex!("0122222222333333334444444455000003300000000000003310"), 0x044601), - (hex!("0122222222333333334444444455000003300000000000006e40"), 0x0446a1), - (hex!("0122222222333333334444444455000003310000000000003320"), 0x044741), - (hex!("0122222222333333334444444455000003310000000000004620"), 0x0447e1), - (hex!("0122222222333333334444444455000003320000000000003330"), 0x044881), - (hex!("0122222222333333334444444455000003330000000000003340"), 0x044921), - (hex!("0122222222333333334444444455000003330000000000004b80"), 0x0449c1), - (hex!("0122222222333333334444444455000003340000000000003350"), 0x044a61), - (hex!("0122222222333333334444444455000003350000000000003360"), 0x044b01), - (hex!("0122222222333333334444444455000003360000000000003370"), 0x044ba1), - (hex!("0122222222333333334444444455000003370000000000003380"), 0x044c41), - (hex!("0122222222333333334444444455000003380000000000003390"), 0x044ce1), - (hex!("01222222223333333344444444550000033900000000000033a0"), 0x044d81), - (hex!("0122222222333333334444444455000003390000000000006b90"), 0x044e21), - (hex!("01222222223333333344444444550000033a00000000000033b0"), 0x044ec1), - (hex!("01222222223333333344444444550000033a0000000000007420"), 0x044f61), - (hex!("01222222223333333344444444550000033b00000000000033c0"), 0x045001), - (hex!("01222222223333333344444444550000033b0000000000007620"), 0x0450a1), - (hex!("01222222223333333344444444550000033c00000000000033d0"), 0x045141), - (hex!("01222222223333333344444444550000033c0000000000006b30"), 0x0451e1), - (hex!("01222222223333333344444444550000033d00000000000033e0"), 0x045281), - (hex!("01222222223333333344444444550000033e00000000000033f0"), 0x045321), - (hex!("01222222223333333344444444550000033e00000000000048b0"), 0x0453c1), - (hex!("01222222223333333344444444550000033e0000000000004e70"), 0x045461), - (hex!("01222222223333333344444444550000033f0000000000003400"), 0x045501), - (hex!("01222222223333333344444444550000033f0000000000006380"), 0x0455a1), - (hex!("0122222222333333334444444455000003400000000000003410"), 0x045641), - (hex!("0122222222333333334444444455000003410000000000003420"), 0x0456e1), - (hex!("0122222222333333334444444455000003410000000000006090"), 0x045781), - (hex!("0122222222333333334444444455000003420000000000003430"), 0x045821), - (hex!("01222222223333333344444444550000034200000000000073d0"), 0x0458c1), - (hex!("0122222222333333334444444455000003430000000000003440"), 0x045961), - (hex!("0122222222333333334444444455000003430000000000006370"), 0x045a01), - (hex!("01222222223333333344444444550000034300000000000075c0"), 0x045aa1), - (hex!("0122222222333333334444444455000003440000000000003450"), 0x045b41), - (hex!("0122222222333333334444444455000003450000000000003460"), 0x045be1), - (hex!("0122222222333333334444444455000003460000000000003470"), 0x045c81), - (hex!("01222222223333333344444444550000034600000000000055f0"), 0x045d21), - (hex!("0122222222333333334444444455000003470000000000003480"), 0x045dc1), - (hex!("0122222222333333334444444455000003470000000000003fe0"), 0x045e61), - (hex!("0122222222333333334444444455000003480000000000003490"), 0x045f01), - (hex!("0122222222333333334444444455000003480000000000007990"), 0x045fa1), - (hex!("01222222223333333344444444550000034900000000000034a0"), 0x046041), - (hex!("0122222222333333334444444455000003490000000000004410"), 0x0460e1), - (hex!("01222222223333333344444444550000034a00000000000034b0"), 0x046181), - (hex!("01222222223333333344444444550000034a00000000000062a0"), 0x046221), - (hex!("01222222223333333344444444550000034a0000000000007260"), 0x0462c1), - (hex!("01222222223333333344444444550000034b00000000000034c0"), 0x046361), - (hex!("01222222223333333344444444550000034b0000000000005760"), 0x046401), - (hex!("01222222223333333344444444550000034b0000000000006200"), 0x0464a1), - (hex!("01222222223333333344444444550000034c00000000000034d0"), 0x046541), - (hex!("01222222223333333344444444550000034d00000000000034e0"), 0x0465e1), - (hex!("01222222223333333344444444550000034e00000000000034f0"), 0x046681), - (hex!("01222222223333333344444444550000034e0000000000007790"), 0x046721), - (hex!("01222222223333333344444444550000034f0000000000003500"), 0x0467c1), - (hex!("0122222222333333334444444455000003500000000000003510"), 0x046861), - (hex!("0122222222333333334444444455000003510000000000003520"), 0x046901), - (hex!("0122222222333333334444444455000003520000000000003530"), 0x0469a1), - (hex!("01222222223333333344444444550000035200000000000056a0"), 0x046a41), - (hex!("0122222222333333334444444455000003530000000000003540"), 0x046ae1), - (hex!("0122222222333333334444444455000003540000000000003550"), 0x046b81), - (hex!("01222222223333333344444444550000035400000000000047b0"), 0x046c21), - (hex!("0122222222333333334444444455000003550000000000003560"), 0x046cc1), - (hex!("0122222222333333334444444455000003550000000000004500"), 0x046d61), - (hex!("0122222222333333334444444455000003560000000000003570"), 0x046e01), - (hex!("0122222222333333334444444455000003560000000000004fc0"), 0x046ea1), - (hex!("0122222222333333334444444455000003560000000000007160"), 0x046f41), - (hex!("0122222222333333334444444455000003560000000000007400"), 0x046fe1), - (hex!("0122222222333333334444444455000003570000000000003580"), 0x047081), - (hex!("0122222222333333334444444455000003580000000000003590"), 0x047121), - (hex!("0122222222333333334444444455000003580000000000005a80"), 0x0471c1), - (hex!("01222222223333333344444444550000035900000000000035a0"), 0x047261), - (hex!("01222222223333333344444444550000035900000000000073b0"), 0x047301), - (hex!("01222222223333333344444444550000035a00000000000035b0"), 0x0473a1), - (hex!("01222222223333333344444444550000035a0000000000004c20"), 0x047441), - (hex!("01222222223333333344444444550000035b00000000000035c0"), 0x0474e1), - (hex!("01222222223333333344444444550000035b0000000000005120"), 0x047581), - (hex!("01222222223333333344444444550000035c00000000000035d0"), 0x047621), - (hex!("01222222223333333344444444550000035c0000000000004300"), 0x0476c1), - (hex!("01222222223333333344444444550000035c0000000000005a40"), 0x047761), - (hex!("01222222223333333344444444550000035c0000000000006620"), 0x047801), - (hex!("01222222223333333344444444550000035c0000000000006ed0"), 0x0478a1), - (hex!("01222222223333333344444444550000035d00000000000035e0"), 0x047941), - (hex!("01222222223333333344444444550000035d0000000000005df0"), 0x0479e1), - (hex!("01222222223333333344444444550000035e00000000000035f0"), 0x047a81), - (hex!("01222222223333333344444444550000035f0000000000003600"), 0x047b21), - (hex!("01222222223333333344444444550000035f00000000000058d0"), 0x047bc1), - (hex!("0122222222333333334444444455000003600000000000003610"), 0x047c61), - (hex!("0122222222333333334444444455000003600000000000007b90"), 0x047d01), - (hex!("0122222222333333334444444455000003610000000000003620"), 0x047da1), - (hex!("0122222222333333334444444455000003610000000000006ad0"), 0x047e41), - (hex!("0122222222333333334444444455000003620000000000003630"), 0x047ee1), - (hex!("01222222223333333344444444550000036200000000000063a0"), 0x047f81), - (hex!("0122222222333333334444444455000003630000000000003640"), 0x048021), - (hex!("0122222222333333334444444455000003630000000000007250"), 0x0480c1), - (hex!("0122222222333333334444444455000003640000000000003650"), 0x048161), - (hex!("0122222222333333334444444455000003640000000000005510"), 0x048201), - (hex!("0122222222333333334444444455000003640000000000007850"), 0x0482a1), - (hex!("0122222222333333334444444455000003650000000000003660"), 0x048341), - (hex!("0122222222333333334444444455000003660000000000003670"), 0x0483e1), - (hex!("0122222222333333334444444455000003660000000000004650"), 0x048481), - (hex!("01222222223333333344444444550000036600000000000050d0"), 0x048521), - (hex!("0122222222333333334444444455000003660000000000006eb0"), 0x0485c1), - (hex!("0122222222333333334444444455000003670000000000003680"), 0x048661), - (hex!("01222222223333333344444444550000036700000000000071f0"), 0x048701), - (hex!("0122222222333333334444444455000003680000000000003690"), 0x0487a1), - (hex!("01222222223333333344444444550000036900000000000036a0"), 0x048841), - (hex!("0122222222333333334444444455000003690000000000005c70"), 0x0488e1), - (hex!("01222222223333333344444444550000036a00000000000036b0"), 0x048981), - (hex!("01222222223333333344444444550000036a00000000000071b0"), 0x048a21), - (hex!("01222222223333333344444444550000036b00000000000036c0"), 0x048ac1), - (hex!("01222222223333333344444444550000036b0000000000004670"), 0x048b61), - (hex!("01222222223333333344444444550000036c00000000000036d0"), 0x048c01), - (hex!("01222222223333333344444444550000036c0000000000004750"), 0x048ca1), - (hex!("01222222223333333344444444550000036c0000000000006fa0"), 0x048d41), - (hex!("01222222223333333344444444550000036d00000000000036e0"), 0x048de1), - (hex!("01222222223333333344444444550000036d0000000000003f70"), 0x048e81), - (hex!("01222222223333333344444444550000036d0000000000004b90"), 0x048f21), - (hex!("01222222223333333344444444550000036d00000000000057a0"), 0x048fc1), - (hex!("01222222223333333344444444550000036e00000000000036f0"), 0x049061), - (hex!("01222222223333333344444444550000036e00000000000075d0"), 0x049101), - (hex!("01222222223333333344444444550000036f0000000000003700"), 0x0491a1), - (hex!("0122222222333333334444444455000003700000000000003710"), 0x049241), - (hex!("0122222222333333334444444455000003700000000000005aa0"), 0x0492e1), - (hex!("0122222222333333334444444455000003710000000000003720"), 0x049381), - (hex!("0122222222333333334444444455000003710000000000005130"), 0x049421), - (hex!("0122222222333333334444444455000003710000000000006fc0"), 0x0494c1), - (hex!("0122222222333333334444444455000003710000000000007b00"), 0x049561), - (hex!("0122222222333333334444444455000003720000000000003730"), 0x049601), - (hex!("01222222223333333344444444550000037200000000000054d0"), 0x0496a1), - (hex!("0122222222333333334444444455000003730000000000003740"), 0x049741), - (hex!("0122222222333333334444444455000003730000000000004220"), 0x0497e1), - (hex!("0122222222333333334444444455000003740000000000003750"), 0x049881), - (hex!("0122222222333333334444444455000003740000000000004720"), 0x049921), - (hex!("0122222222333333334444444455000003750000000000003760"), 0x0499c1), - (hex!("0122222222333333334444444455000003750000000000004110"), 0x049a61), - (hex!("0122222222333333334444444455000003760000000000003770"), 0x049b01), - (hex!("0122222222333333334444444455000003770000000000003780"), 0x049ba1), - (hex!("0122222222333333334444444455000003780000000000003790"), 0x049c41), - (hex!("0122222222333333334444444455000003780000000000004b40"), 0x049ce1), - (hex!("0122222222333333334444444455000003780000000000005660"), 0x049d81), - (hex!("0122222222333333334444444455000003780000000000005ea0"), 0x049e21), - (hex!("01222222223333333344444444550000037900000000000037a0"), 0x049ec1), - (hex!("01222222223333333344444444550000037a00000000000037b0"), 0x049f61), - (hex!("01222222223333333344444444550000037b00000000000037c0"), 0x04a001), - (hex!("01222222223333333344444444550000037c00000000000037d0"), 0x04a0a1), - (hex!("01222222223333333344444444550000037c0000000000004340"), 0x04a141), - (hex!("01222222223333333344444444550000037c0000000000005230"), 0x04a1e1), - (hex!("01222222223333333344444444550000037d00000000000037e0"), 0x04a281), - (hex!("01222222223333333344444444550000037d00000000000051e0"), 0x04a321), - (hex!("01222222223333333344444444550000037e00000000000037f0"), 0x04a3c1), - (hex!("01222222223333333344444444550000037e0000000000004090"), 0x04a461), - (hex!("01222222223333333344444444550000037e0000000000005c20"), 0x04a501), - (hex!("01222222223333333344444444550000037f0000000000003800"), 0x04a5a1), - (hex!("0122222222333333334444444455000003800000000000003810"), 0x04a641), - (hex!("0122222222333333334444444455000003800000000000007630"), 0x04a6e1), - (hex!("0122222222333333334444444455000003810000000000003820"), 0x04a781), - (hex!("0122222222333333334444444455000003820000000000003830"), 0x04a821), - (hex!("0122222222333333334444444455000003820000000000004170"), 0x04a8c1), - (hex!("0122222222333333334444444455000003830000000000003840"), 0x04a961), - (hex!("0122222222333333334444444455000003840000000000003850"), 0x04aa01), - (hex!("0122222222333333334444444455000003850000000000003860"), 0x04aaa1), - (hex!("0122222222333333334444444455000003850000000000004180"), 0x04ab41), - (hex!("0122222222333333334444444455000003850000000000005c90"), 0x04abe1), - (hex!("0122222222333333334444444455000003850000000000005da0"), 0x04ac81), - (hex!("0122222222333333334444444455000003850000000000006ff0"), 0x04ad21), - (hex!("0122222222333333334444444455000003860000000000003870"), 0x04adc1), - (hex!("01222222223333333344444444550000038600000000000065c0"), 0x04ae61), - (hex!("0122222222333333334444444455000003870000000000003880"), 0x04af01), - (hex!("0122222222333333334444444455000003870000000000007cc0"), 0x04afa1), - (hex!("0122222222333333334444444455000003880000000000003890"), 0x04b041), - (hex!("01222222223333333344444444550000038900000000000038a0"), 0x04b0e1), - (hex!("01222222223333333344444444550000038a00000000000038b0"), 0x04b181), - (hex!("01222222223333333344444444550000038a00000000000073e0"), 0x04b221), - (hex!("01222222223333333344444444550000038b00000000000038c0"), 0x04b2c1), - (hex!("01222222223333333344444444550000038c00000000000038d0"), 0x04b361), - (hex!("01222222223333333344444444550000038d00000000000038e0"), 0x04b401), - (hex!("01222222223333333344444444550000038d00000000000069f0"), 0x04b4a1), - (hex!("01222222223333333344444444550000038d0000000000007680"), 0x04b541), - (hex!("01222222223333333344444444550000038e00000000000038f0"), 0x04b5e1), - (hex!("01222222223333333344444444550000038f0000000000003900"), 0x04b681), - (hex!("01222222223333333344444444550000038f00000000000045b0"), 0x04b721), - (hex!("01222222223333333344444444550000038f0000000000007180"), 0x04b7c1), - (hex!("0122222222333333334444444455000003900000000000003910"), 0x04b861), - (hex!("0122222222333333334444444455000003910000000000003920"), 0x04b901), - (hex!("0122222222333333334444444455000003910000000000004a20"), 0x04b9a1), - (hex!("0122222222333333334444444455000003920000000000003930"), 0x04ba41), - (hex!("01222222223333333344444444550000039200000000000059b0"), 0x04bae1), - (hex!("0122222222333333334444444455000003930000000000003940"), 0x04bb81), - (hex!("0122222222333333334444444455000003930000000000006cc0"), 0x04bc21), - (hex!("0122222222333333334444444455000003940000000000003950"), 0x04bcc1), - (hex!("01222222223333333344444444550000039400000000000056c0"), 0x04bd61), - (hex!("0122222222333333334444444455000003950000000000003960"), 0x04be01), - (hex!("0122222222333333334444444455000003950000000000004cc0"), 0x04bea1), - (hex!("0122222222333333334444444455000003950000000000007720"), 0x04bf41), - (hex!("0122222222333333334444444455000003960000000000003970"), 0x04bfe1), - (hex!("0122222222333333334444444455000003960000000000004da0"), 0x04c081), - (hex!("0122222222333333334444444455000003960000000000004df0"), 0x04c121), - (hex!("0122222222333333334444444455000003960000000000004f30"), 0x04c1c1), - (hex!("01222222223333333344444444550000039600000000000050f0"), 0x04c261), - (hex!("0122222222333333334444444455000003960000000000007940"), 0x04c301), - (hex!("0122222222333333334444444455000003970000000000003980"), 0x04c3a1), - (hex!("0122222222333333334444444455000003970000000000005850"), 0x04c441), - (hex!("0122222222333333334444444455000003970000000000007bd0"), 0x04c4e1), - (hex!("0122222222333333334444444455000003980000000000003990"), 0x04c581), - (hex!("0122222222333333334444444455000003980000000000004c00"), 0x04c621), - (hex!("0122222222333333334444444455000003980000000000005580"), 0x04c6c1), - (hex!("01222222223333333344444444550000039900000000000039a0"), 0x04c761), - (hex!("0122222222333333334444444455000003990000000000005820"), 0x04c801), - (hex!("01222222223333333344444444550000039a00000000000039b0"), 0x04c8a1), - (hex!("01222222223333333344444444550000039b00000000000039c0"), 0x04c941), - (hex!("01222222223333333344444444550000039b0000000000004c10"), 0x04c9e1), - (hex!("01222222223333333344444444550000039b0000000000006460"), 0x04ca81), - (hex!("01222222223333333344444444550000039c00000000000039d0"), 0x04cb21), - (hex!("01222222223333333344444444550000039d00000000000039e0"), 0x04cbc1), - (hex!("01222222223333333344444444550000039d00000000000044c0"), 0x04cc61), - (hex!("01222222223333333344444444550000039d00000000000049e0"), 0x04cd01), - (hex!("01222222223333333344444444550000039e00000000000039f0"), 0x04cda1), - (hex!("01222222223333333344444444550000039f0000000000003a00"), 0x04ce41), - (hex!("0122222222333333334444444455000003a00000000000003a10"), 0x04cee1), - (hex!("0122222222333333334444444455000003a10000000000003a20"), 0x04cf81), - (hex!("0122222222333333334444444455000003a10000000000006a80"), 0x04d021), - (hex!("0122222222333333334444444455000003a20000000000003a30"), 0x04d0c1), - (hex!("0122222222333333334444444455000003a200000000000062b0"), 0x04d161), - (hex!("0122222222333333334444444455000003a30000000000003a40"), 0x04d201), - (hex!("0122222222333333334444444455000003a30000000000006ce0"), 0x04d2a1), - (hex!("0122222222333333334444444455000003a40000000000003a50"), 0x04d341), - (hex!("0122222222333333334444444455000003a50000000000003a60"), 0x04d3e1), - (hex!("0122222222333333334444444455000003a60000000000003a70"), 0x04d481), - (hex!("0122222222333333334444444455000003a60000000000007750"), 0x04d521), - (hex!("0122222222333333334444444455000003a70000000000003a80"), 0x04d5c1), - (hex!("0122222222333333334444444455000003a70000000000005b10"), 0x04d661), - (hex!("0122222222333333334444444455000003a80000000000003a90"), 0x04d701), - (hex!("0122222222333333334444444455000003a80000000000006c20"), 0x04d7a1), - (hex!("0122222222333333334444444455000003a90000000000003aa0"), 0x04d841), - (hex!("0122222222333333334444444455000003a90000000000005b70"), 0x04d8e1), - (hex!("0122222222333333334444444455000003a900000000000070e0"), 0x04d981), - (hex!("0122222222333333334444444455000003aa0000000000003ab0"), 0x04da21), - (hex!("0122222222333333334444444455000003aa00000000000049f0"), 0x04dac1), - (hex!("0122222222333333334444444455000003aa0000000000004d60"), 0x04db61), - (hex!("0122222222333333334444444455000003ab0000000000003ac0"), 0x04dc01), - (hex!("0122222222333333334444444455000003ac0000000000003ad0"), 0x04dca1), - (hex!("0122222222333333334444444455000003ac0000000000004580"), 0x04dd41), - (hex!("0122222222333333334444444455000003ad0000000000003ae0"), 0x04dde1), - (hex!("0122222222333333334444444455000003ae0000000000003af0"), 0x04de81), - (hex!("0122222222333333334444444455000003af0000000000003b00"), 0x04df21), - (hex!("0122222222333333334444444455000003b00000000000003b10"), 0x04dfc1), - (hex!("0122222222333333334444444455000003b10000000000003b20"), 0x04e061), - (hex!("0122222222333333334444444455000003b10000000000003fd0"), 0x04e101), - (hex!("0122222222333333334444444455000003b20000000000003b30"), 0x04e1a1), - (hex!("0122222222333333334444444455000003b30000000000003b40"), 0x04e241), - (hex!("0122222222333333334444444455000003b40000000000003b50"), 0x04e2e1), - (hex!("0122222222333333334444444455000003b40000000000007450"), 0x04e381), - (hex!("0122222222333333334444444455000003b50000000000003b60"), 0x04e421), - (hex!("0122222222333333334444444455000003b60000000000003b70"), 0x04e4c1), - (hex!("0122222222333333334444444455000003b70000000000003b80"), 0x04e561), - (hex!("0122222222333333334444444455000003b70000000000006d50"), 0x04e601), - (hex!("0122222222333333334444444455000003b80000000000003b90"), 0x04e6a1), - (hex!("0122222222333333334444444455000003b800000000000057c0"), 0x04e741), - (hex!("0122222222333333334444444455000003b800000000000078a0"), 0x04e7e1), - (hex!("0122222222333333334444444455000003b90000000000003ba0"), 0x04e881), - (hex!("0122222222333333334444444455000003b90000000000006750"), 0x04e921), - (hex!("0122222222333333334444444455000003ba0000000000003bb0"), 0x04e9c1), - (hex!("0122222222333333334444444455000003ba0000000000007a10"), 0x04ea61), - (hex!("0122222222333333334444444455000003ba0000000000007a20"), 0x04eb01), - (hex!("0122222222333333334444444455000003bb0000000000003bc0"), 0x04eba1), - (hex!("0122222222333333334444444455000003bb0000000000005bc0"), 0x04ec41), - (hex!("0122222222333333334444444455000003bc0000000000003bd0"), 0x04ece1), - (hex!("0122222222333333334444444455000003bc0000000000005e80"), 0x04ed81), - (hex!("0122222222333333334444444455000003bc0000000000007ab0"), 0x04ee21), - (hex!("0122222222333333334444444455000003bd0000000000003be0"), 0x04eec1), - (hex!("0122222222333333334444444455000003bd00000000000049b0"), 0x04ef61), - (hex!("0122222222333333334444444455000003be0000000000003bf0"), 0x04f001), - (hex!("0122222222333333334444444455000003be0000000000005780"), 0x04f0a1), - (hex!("0122222222333333334444444455000003be0000000000007930"), 0x04f141), - (hex!("0122222222333333334444444455000003bf0000000000003c00"), 0x04f1e1), - (hex!("0122222222333333334444444455000003bf0000000000005de0"), 0x04f281), - (hex!("0122222222333333334444444455000003bf00000000000060b0"), 0x04f321), - (hex!("0122222222333333334444444455000003bf00000000000060c0"), 0x04f3c1), - (hex!("0122222222333333334444444455000003bf0000000000006a50"), 0x04f461), - (hex!("0122222222333333334444444455000003c00000000000003c10"), 0x04f501), - (hex!("0122222222333333334444444455000003c00000000000004030"), 0x04f5a1), - (hex!("0122222222333333334444444455000003c10000000000003c20"), 0x04f641), - (hex!("0122222222333333334444444455000003c20000000000003c30"), 0x04f6e1), - (hex!("0122222222333333334444444455000003c200000000000040b0"), 0x04f781), - (hex!("0122222222333333334444444455000003c30000000000003c40"), 0x04f821), - (hex!("0122222222333333334444444455000003c40000000000003c50"), 0x04f8c1), - (hex!("0122222222333333334444444455000003c40000000000005ba0"), 0x04f961), - (hex!("0122222222333333334444444455000003c50000000000003c60"), 0x04fa01), - (hex!("0122222222333333334444444455000003c60000000000003c70"), 0x04faa1), - (hex!("0122222222333333334444444455000003c70000000000003c80"), 0x04fb41), - (hex!("0122222222333333334444444455000003c70000000000004270"), 0x04fbe1), - (hex!("0122222222333333334444444455000003c80000000000003c90"), 0x04fc81), - (hex!("0122222222333333334444444455000003c80000000000006e70"), 0x04fd21), - (hex!("0122222222333333334444444455000003c90000000000003ca0"), 0x04fdc1), - (hex!("0122222222333333334444444455000003ca0000000000003cb0"), 0x04fe61), - (hex!("0122222222333333334444444455000003ca0000000000006e20"), 0x04ff01), - (hex!("0122222222333333334444444455000003ca0000000000007c20"), 0x04ffa1), - (hex!("0122222222333333334444444455000003cb0000000000003cc0"), 0x050041), - (hex!("0122222222333333334444444455000003cc0000000000003cd0"), 0x0500e1), - (hex!("0122222222333333334444444455000003cc0000000000006120"), 0x050181), - (hex!("0122222222333333334444444455000003cc0000000000007950"), 0x050221), - (hex!("0122222222333333334444444455000003cd0000000000003ce0"), 0x0502c1), - (hex!("0122222222333333334444444455000003ce0000000000003cf0"), 0x050361), - (hex!("0122222222333333334444444455000003cf0000000000003d00"), 0x050401), - (hex!("0122222222333333334444444455000003d00000000000003d10"), 0x0504a1), - (hex!("0122222222333333334444444455000003d10000000000003d20"), 0x050541), - (hex!("0122222222333333334444444455000003d10000000000005e50"), 0x0505e1), - (hex!("0122222222333333334444444455000003d10000000000007880"), 0x050681), - (hex!("0122222222333333334444444455000003d20000000000003d30"), 0x050721), - (hex!("0122222222333333334444444455000003d20000000000005d00"), 0x0507c1), - (hex!("0122222222333333334444444455000003d30000000000003d40"), 0x050861), - (hex!("0122222222333333334444444455000003d30000000000005d40"), 0x050901), - (hex!("0122222222333333334444444455000003d300000000000063f0"), 0x0509a1), - (hex!("0122222222333333334444444455000003d40000000000003d50"), 0x050a41), - (hex!("0122222222333333334444444455000003d40000000000005700"), 0x050ae1), - (hex!("0122222222333333334444444455000003d400000000000078f0"), 0x050b81), - (hex!("0122222222333333334444444455000003d50000000000003d60"), 0x050c21), - (hex!("0122222222333333334444444455000003d60000000000003d70"), 0x050cc1), - (hex!("0122222222333333334444444455000003d70000000000003d80"), 0x050d61), - (hex!("0122222222333333334444444455000003d80000000000003d90"), 0x050e01), - (hex!("0122222222333333334444444455000003d80000000000006690"), 0x050ea1), - (hex!("0122222222333333334444444455000003d90000000000003da0"), 0x050f41), - (hex!("0122222222333333334444444455000003d900000000000076d0"), 0x050fe1), - (hex!("0122222222333333334444444455000003da0000000000003db0"), 0x051081), - (hex!("0122222222333333334444444455000003db0000000000003dc0"), 0x051121), - (hex!("0122222222333333334444444455000003db0000000000004a30"), 0x0511c1), - (hex!("0122222222333333334444444455000003db0000000000005390"), 0x051261), - (hex!("0122222222333333334444444455000003dc0000000000003dd0"), 0x051301), - (hex!("0122222222333333334444444455000003dc0000000000006d60"), 0x0513a1), - (hex!("0122222222333333334444444455000003dd0000000000003de0"), 0x051441), - (hex!("0122222222333333334444444455000003de0000000000003df0"), 0x0514e1), - (hex!("0122222222333333334444444455000003df0000000000003e00"), 0x051581), - (hex!("0122222222333333334444444455000003df0000000000005240"), 0x051621), - (hex!("0122222222333333334444444455000003df0000000000005610"), 0x0516c1), - (hex!("0122222222333333334444444455000003e00000000000003e10"), 0x051761), - (hex!("0122222222333333334444444455000003e00000000000006500"), 0x051801), - (hex!("0122222222333333334444444455000003e10000000000003e20"), 0x0518a1), - (hex!("0122222222333333334444444455000003e10000000000006a10"), 0x051941), - (hex!("0122222222333333334444444455000003e10000000000007c10"), 0x0519e1), - (hex!("0122222222333333334444444455000003e20000000000003e30"), 0x051a81), - (hex!("0122222222333333334444444455000003e20000000000006310"), 0x051b21), - (hex!("0122222222333333334444444455000003e30000000000003e40"), 0x051bc1), - (hex!("0122222222333333334444444455000003e40000000000003e50"), 0x051c61), - (hex!("0122222222333333334444444455000003e40000000000006780"), 0x051d01), - (hex!("0122222222333333334444444455000003e40000000000007ce0"), 0x051da1), - (hex!("0122222222333333334444444455000003e50000000000003e60"), 0x051e41), - (hex!("0122222222333333334444444455000003e60000000000003e70"), 0x051ee1), - (hex!("0122222222333333334444444455000003e60000000000005040"), 0x051f81), - (hex!("0122222222333333334444444455000003e60000000000005bf0"), 0x052021), - (hex!("0122222222333333334444444455000003e70000000000003e80"), 0x0520c1), - (hex!("0122222222333333334444444455000003e70000000000003f50"), 0x052161), + (hex!("0100000000333333334444444455000000000000000000000010"), 0x004001), + (hex!("0100000000333333334444444455000000000000000000007cb0"), 0x0040a1), + (hex!("0100000000333333334444444455000000010000000000000020"), 0x004141), + (hex!("0100000000333333334444444455000000020000000000000030"), 0x0041e1), + (hex!("01000000003333333344444444550000000200000000000051a0"), 0x004281), + (hex!("0100000000333333334444444455000000030000000000000040"), 0x004321), + (hex!("0100000000333333334444444455000000030000000000006cf0"), 0x0043c1), + (hex!("0100000000333333334444444455000000030000000000007140"), 0x004461), + (hex!("0100000000333333334444444455000000040000000000000050"), 0x004501), + (hex!("01000000003333333344444444550000000400000000000047f0"), 0x0045a1), + (hex!("01000000003333333344444444550000000400000000000072b0"), 0x004641), + (hex!("0100000000333333334444444455000000050000000000000060"), 0x0046e1), + (hex!("0100000000333333334444444455000000050000000000005550"), 0x004781), + (hex!("0100000000333333334444444455000000060000000000000070"), 0x004821), + (hex!("01000000003333333344444444550000000600000000000044a0"), 0x0048c1), + (hex!("0100000000333333334444444455000000060000000000006870"), 0x004961), + (hex!("0100000000333333334444444455000000070000000000000080"), 0x004a01), + (hex!("0100000000333333334444444455000000080000000000000090"), 0x004aa1), + (hex!("0100000000333333334444444455000000080000000000004150"), 0x004b41), + (hex!("01000000003333333344444444550000000900000000000000a0"), 0x004be1), + (hex!("01000000003333333344444444550000000a00000000000000b0"), 0x004c81), + (hex!("01000000003333333344444444550000000a0000000000006680"), 0x004d21), + (hex!("01000000003333333344444444550000000b00000000000000c0"), 0x004dc1), + (hex!("01000000003333333344444444550000000b0000000000006230"), 0x004e61), + (hex!("01000000003333333344444444550000000c00000000000000d0"), 0x004f01), + (hex!("01000000003333333344444444550000000d00000000000000e0"), 0x004fa1), + (hex!("01000000003333333344444444550000000e00000000000000f0"), 0x005041), + (hex!("01000000003333333344444444550000000e0000000000006000"), 0x0050e1), + (hex!("01000000003333333344444444550000000f0000000000000100"), 0x005181), + (hex!("01000000003333333344444444550000000f00000000000053c0"), 0x005221), + (hex!("01000000003333333344444444550000000f0000000000006580"), 0x0052c1), + (hex!("0100000000333333334444444455000000100000000000000110"), 0x005361), + (hex!("01000000003333333344444444550000001000000000000046c0"), 0x005401), + (hex!("0100000000333333334444444455000000100000000000004e40"), 0x0054a1), + (hex!("0100000000333333334444444455000000110000000000000120"), 0x005541), + (hex!("0100000000333333334444444455000000120000000000000130"), 0x0055e1), + (hex!("01000000003333333344444444550000001200000000000066d0"), 0x005681), + (hex!("0100000000333333334444444455000000130000000000000140"), 0x005721), + (hex!("0100000000333333334444444455000000130000000000007710"), 0x0057c1), + (hex!("0100000000333333334444444455000000140000000000000150"), 0x005861), + (hex!("0100000000333333334444444455000000140000000000006c40"), 0x005901), + (hex!("0100000000333333334444444455000000150000000000000160"), 0x0059a1), + (hex!("0100000000333333334444444455000000150000000000005990"), 0x005a41), + (hex!("0100000000333333334444444455000000160000000000000170"), 0x005ae1), + (hex!("0100000000333333334444444455000000160000000000005530"), 0x005b81), + (hex!("0100000000333333334444444455000000170000000000000180"), 0x005c21), + (hex!("0100000000333333334444444455000000170000000000004290"), 0x005cc1), + (hex!("0100000000333333334444444455000000180000000000000190"), 0x005d61), + (hex!("01000000003333333344444444550000001800000000000051c0"), 0x005e01), + (hex!("01000000003333333344444444550000001900000000000001a0"), 0x005ea1), + (hex!("0100000000333333334444444455000000190000000000005420"), 0x005f41), + (hex!("0100000000333333334444444455000000190000000000005770"), 0x005fe1), + (hex!("01000000003333333344444444550000001900000000000079d0"), 0x006081), + (hex!("01000000003333333344444444550000001a00000000000001b0"), 0x006121), + (hex!("01000000003333333344444444550000001a0000000000006f70"), 0x0061c1), + (hex!("01000000003333333344444444550000001a0000000000007150"), 0x006261), + (hex!("01000000003333333344444444550000001b00000000000001c0"), 0x006301), + (hex!("01000000003333333344444444550000001b0000000000005070"), 0x0063a1), + (hex!("01000000003333333344444444550000001c00000000000001d0"), 0x006441), + (hex!("01000000003333333344444444550000001d00000000000001e0"), 0x0064e1), + (hex!("01000000003333333344444444550000001e00000000000001f0"), 0x006581), + (hex!("01000000003333333344444444550000001e0000000000005650"), 0x006621), + (hex!("01000000003333333344444444550000001f0000000000000200"), 0x0066c1), + (hex!("01000000003333333344444444550000001f0000000000006ca0"), 0x006761), + (hex!("0100000000333333334444444455000000200000000000000210"), 0x006801), + (hex!("0100000000333333334444444455000000200000000000005fc0"), 0x0068a1), + (hex!("0100000000333333334444444455000000210000000000000220"), 0x006941), + (hex!("0100000000333333334444444455000000210000000000006430"), 0x0069e1), + (hex!("0100000000333333334444444455000000220000000000000230"), 0x006a81), + (hex!("01000000003333333344444444550000002200000000000040e0"), 0x006b21), + (hex!("0100000000333333334444444455000000230000000000000240"), 0x006bc1), + (hex!("01000000003333333344444444550000002300000000000042d0"), 0x006c61), + (hex!("0100000000333333334444444455000000240000000000000250"), 0x006d01), + (hex!("0100000000333333334444444455000000250000000000000260"), 0x006da1), + (hex!("01000000003333333344444444550000002500000000000058c0"), 0x006e41), + (hex!("0100000000333333334444444455000000260000000000000270"), 0x006ee1), + (hex!("0100000000333333334444444455000000260000000000004020"), 0x006f81), + (hex!("0100000000333333334444444455000000270000000000000280"), 0x007021), + (hex!("0100000000333333334444444455000000280000000000000290"), 0x0070c1), + (hex!("0100000000333333334444444455000000280000000000007c00"), 0x007161), + (hex!("01000000003333333344444444550000002900000000000002a0"), 0x007201), + (hex!("01000000003333333344444444550000002a00000000000002b0"), 0x0072a1), + (hex!("01000000003333333344444444550000002b00000000000002c0"), 0x007341), + (hex!("01000000003333333344444444550000002c00000000000002d0"), 0x0073e1), + (hex!("01000000003333333344444444550000002c00000000000041b0"), 0x007481), + (hex!("01000000003333333344444444550000002c0000000000004c30"), 0x007521), + (hex!("01000000003333333344444444550000002d00000000000002e0"), 0x0075c1), + (hex!("01000000003333333344444444550000002d0000000000005e40"), 0x007661), + (hex!("01000000003333333344444444550000002d0000000000006990"), 0x007701), + (hex!("01000000003333333344444444550000002e00000000000002f0"), 0x0077a1), + (hex!("01000000003333333344444444550000002f0000000000000300"), 0x007841), + (hex!("01000000003333333344444444550000002f0000000000004a70"), 0x0078e1), + (hex!("01000000003333333344444444550000002f0000000000006b40"), 0x007981), + (hex!("0100000000333333334444444455000000300000000000000310"), 0x007a21), + (hex!("0100000000333333334444444455000000310000000000000320"), 0x007ac1), + (hex!("0100000000333333334444444455000000320000000000000330"), 0x007b61), + (hex!("01000000003333333344444444550000003200000000000041a0"), 0x007c01), + (hex!("0100000000333333334444444455000000320000000000007340"), 0x007ca1), + (hex!("0100000000333333334444444455000000320000000000007730"), 0x007d41), + (hex!("0100000000333333334444444455000000330000000000000340"), 0x007de1), + (hex!("01000000003333333344444444550000003300000000000055a0"), 0x007e81), + (hex!("0100000000333333334444444455000000340000000000000350"), 0x007f21), + (hex!("0100000000333333334444444455000000350000000000000360"), 0x007fc1), + (hex!("01000000003333333344444444550000003500000000000077a0"), 0x008061), + (hex!("0100000000333333334444444455000000360000000000000370"), 0x008101), + (hex!("0100000000333333334444444455000000370000000000000380"), 0x0081a1), + (hex!("0100000000333333334444444455000000380000000000000390"), 0x008241), + (hex!("01000000003333333344444444550000003900000000000003a0"), 0x0082e1), + (hex!("01000000003333333344444444550000003a00000000000003b0"), 0x008381), + (hex!("01000000003333333344444444550000003a00000000000071c0"), 0x008421), + (hex!("01000000003333333344444444550000003b00000000000003c0"), 0x0084c1), + (hex!("01000000003333333344444444550000003c00000000000003d0"), 0x008561), + (hex!("01000000003333333344444444550000003d00000000000003e0"), 0x008601), + (hex!("01000000003333333344444444550000003e00000000000003f0"), 0x0086a1), + (hex!("01000000003333333344444444550000003e00000000000062e0"), 0x008741), + (hex!("01000000003333333344444444550000003f0000000000000400"), 0x0087e1), + (hex!("0100000000333333334444444455000000400000000000000410"), 0x008881), + (hex!("0100000000333333334444444455000000400000000000004460"), 0x008921), + (hex!("0100000000333333334444444455000000400000000000005b90"), 0x0089c1), + (hex!("01000000003333333344444444550000004000000000000079b0"), 0x008a61), + (hex!("0100000000333333334444444455000000410000000000000420"), 0x008b01), + (hex!("0100000000333333334444444455000000420000000000000430"), 0x008ba1), + (hex!("0100000000333333334444444455000000420000000000005640"), 0x008c41), + (hex!("0100000000333333334444444455000000430000000000000440"), 0x008ce1), + (hex!("01000000003333333344444444550000004300000000000072a0"), 0x008d81), + (hex!("0100000000333333334444444455000000440000000000000450"), 0x008e21), + (hex!("0100000000333333334444444455000000450000000000000460"), 0x008ec1), + (hex!("0100000000333333334444444455000000450000000000005750"), 0x008f61), + (hex!("01000000003333333344444444550000004500000000000077b0"), 0x009001), + (hex!("0100000000333333334444444455000000460000000000000470"), 0x0090a1), + (hex!("0100000000333333334444444455000000470000000000000480"), 0x009141), + (hex!("0100000000333333334444444455000000480000000000000490"), 0x0091e1), + (hex!("01000000003333333344444444550000004800000000000069e0"), 0x009281), + (hex!("01000000003333333344444444550000004900000000000004a0"), 0x009321), + (hex!("0100000000333333334444444455000000490000000000007370"), 0x0093c1), + (hex!("01000000003333333344444444550000004a00000000000004b0"), 0x009461), + (hex!("01000000003333333344444444550000004a0000000000005cb0"), 0x009501), + (hex!("01000000003333333344444444550000004b00000000000004c0"), 0x0095a1), + (hex!("01000000003333333344444444550000004c00000000000004d0"), 0x009641), + (hex!("01000000003333333344444444550000004c0000000000004880"), 0x0096e1), + (hex!("01000000003333333344444444550000004c0000000000007a40"), 0x009781), + (hex!("01000000003333333344444444550000004d00000000000004e0"), 0x009821), + (hex!("01000000003333333344444444550000004d0000000000006390"), 0x0098c1), + (hex!("01000000003333333344444444550000004e00000000000004f0"), 0x009961), + (hex!("01000000003333333344444444550000004e0000000000004db0"), 0x009a01), + (hex!("01000000003333333344444444550000004f0000000000000500"), 0x009aa1), + (hex!("0100000000333333334444444455000000500000000000000510"), 0x009b41), + (hex!("0100000000333333334444444455000000510000000000000520"), 0x009be1), + (hex!("01000000003333333344444444550000005100000000000069c0"), 0x009c81), + (hex!("0100000000333333334444444455000000520000000000000530"), 0x009d21), + (hex!("0100000000333333334444444455000000520000000000006e60"), 0x009dc1), + (hex!("01000000003333333344444444550000005200000000000070c0"), 0x009e61), + (hex!("0100000000333333334444444455000000530000000000000540"), 0x009f01), + (hex!("0100000000333333334444444455000000530000000000005840"), 0x009fa1), + (hex!("0100000000333333334444444455000000540000000000000550"), 0x00a041), + (hex!("01000000003333333344444444550000005400000000000043e0"), 0x00a0e1), + (hex!("01000000003333333344444444550000005400000000000074e0"), 0x00a181), + (hex!("0100000000333333334444444455000000550000000000000560"), 0x00a221), + (hex!("0100000000333333334444444455000000550000000000003ee0"), 0x00a2c1), + (hex!("0100000000333333334444444455000000560000000000000570"), 0x00a361), + (hex!("0100000000333333334444444455000000570000000000000580"), 0x00a401), + (hex!("0100000000333333334444444455000000570000000000007030"), 0x00a4a1), + (hex!("0100000000333333334444444455000000580000000000000590"), 0x00a541), + (hex!("0100000000333333334444444455000000580000000000005340"), 0x00a5e1), + (hex!("01000000003333333344444444550000005800000000000059f0"), 0x00a681), + (hex!("0100000000333333334444444455000000580000000000006930"), 0x00a721), + (hex!("01000000003333333344444444550000005900000000000005a0"), 0x00a7c1), + (hex!("0100000000333333334444444455000000590000000000003f90"), 0x00a861), + (hex!("01000000003333333344444444550000005a00000000000005b0"), 0x00a901), + (hex!("01000000003333333344444444550000005b00000000000005c0"), 0x00a9a1), + (hex!("01000000003333333344444444550000005b00000000000062c0"), 0x00aa41), + (hex!("01000000003333333344444444550000005c00000000000005d0"), 0x00aae1), + (hex!("01000000003333333344444444550000005c0000000000005a70"), 0x00ab81), + (hex!("01000000003333333344444444550000005c0000000000005dd0"), 0x00ac21), + (hex!("01000000003333333344444444550000005d00000000000005e0"), 0x00acc1), + (hex!("01000000003333333344444444550000005d0000000000005730"), 0x00ad61), + (hex!("01000000003333333344444444550000005e00000000000005f0"), 0x00ae01), + (hex!("01000000003333333344444444550000005e0000000000004f40"), 0x00aea1), + (hex!("01000000003333333344444444550000005f0000000000000600"), 0x00af41), + (hex!("0100000000333333334444444455000000600000000000000610"), 0x00afe1), + (hex!("0100000000333333334444444455000000600000000000007c40"), 0x00b081), + (hex!("0100000000333333334444444455000000610000000000000620"), 0x00b121), + (hex!("0100000000333333334444444455000000610000000000007860"), 0x00b1c1), + (hex!("0100000000333333334444444455000000620000000000000630"), 0x00b261), + (hex!("0100000000333333334444444455000000620000000000005050"), 0x00b301), + (hex!("0100000000333333334444444455000000630000000000000640"), 0x00b3a1), + (hex!("0100000000333333334444444455000000640000000000000650"), 0x00b441), + (hex!("0100000000333333334444444455000000650000000000000660"), 0x00b4e1), + (hex!("0100000000333333334444444455000000650000000000005330"), 0x00b581), + (hex!("0100000000333333334444444455000000660000000000000670"), 0x00b621), + (hex!("0100000000333333334444444455000000660000000000004e20"), 0x00b6c1), + (hex!("0100000000333333334444444455000000660000000000005ee0"), 0x00b761), + (hex!("0100000000333333334444444455000000660000000000006360"), 0x00b801), + (hex!("0100000000333333334444444455000000670000000000000680"), 0x00b8a1), + (hex!("0100000000333333334444444455000000670000000000004040"), 0x00b941), + (hex!("0100000000333333334444444455000000680000000000000690"), 0x00b9e1), + (hex!("0100000000333333334444444455000000680000000000003f80"), 0x00ba81), + (hex!("01000000003333333344444444550000006800000000000041e0"), 0x00bb21), + (hex!("01000000003333333344444444550000006900000000000006a0"), 0x00bbc1), + (hex!("0100000000333333334444444455000000690000000000006080"), 0x00bc61), + (hex!("01000000003333333344444444550000006a00000000000006b0"), 0x00bd01), + (hex!("01000000003333333344444444550000006a00000000000042f0"), 0x00bda1), + (hex!("01000000003333333344444444550000006b00000000000006c0"), 0x00be41), + (hex!("01000000003333333344444444550000006b00000000000052f0"), 0x00bee1), + (hex!("01000000003333333344444444550000006b0000000000005980"), 0x00bf81), + (hex!("01000000003333333344444444550000006b0000000000006170"), 0x00c021), + (hex!("01000000003333333344444444550000006c00000000000006d0"), 0x00c0c1), + (hex!("01000000003333333344444444550000006d00000000000006e0"), 0x00c161), + (hex!("01000000003333333344444444550000006d0000000000006fb0"), 0x00c201), + (hex!("01000000003333333344444444550000006e00000000000006f0"), 0x00c2a1), + (hex!("01000000003333333344444444550000006e00000000000065b0"), 0x00c341), + (hex!("01000000003333333344444444550000006e0000000000007970"), 0x00c3e1), + (hex!("01000000003333333344444444550000006f0000000000000700"), 0x00c481), + (hex!("01000000003333333344444444550000006f0000000000005900"), 0x00c521), + (hex!("01000000003333333344444444550000006f0000000000006d90"), 0x00c5c1), + (hex!("0100000000333333334444444455000000700000000000000710"), 0x00c661), + (hex!("01000000003333333344444444550000007000000000000045c0"), 0x00c701), + (hex!("0100000000333333334444444455000000700000000000004d40"), 0x00c7a1), + (hex!("0100000000333333334444444455000000710000000000000720"), 0x00c841), + (hex!("0100000000333333334444444455000000710000000000004dc0"), 0x00c8e1), + (hex!("0100000000333333334444444455000000710000000000007550"), 0x00c981), + (hex!("0100000000333333334444444455000000720000000000000730"), 0x00ca21), + (hex!("0100000000333333334444444455000000720000000000003ec0"), 0x00cac1), + (hex!("01000000003333333344444444550000007200000000000045a0"), 0x00cb61), + (hex!("0100000000333333334444444455000000720000000000006770"), 0x00cc01), + (hex!("0100000000333333334444444455000000720000000000006bc0"), 0x00cca1), + (hex!("0100000000333333334444444455000000730000000000000740"), 0x00cd41), + (hex!("0100000000333333334444444455000000730000000000005250"), 0x00cde1), + (hex!("01000000003333333344444444550000007300000000000075f0"), 0x00ce81), + (hex!("0100000000333333334444444455000000740000000000000750"), 0x00cf21), + (hex!("0100000000333333334444444455000000740000000000003ff0"), 0x00cfc1), + (hex!("01000000003333333344444444550000007400000000000079e0"), 0x00d061), + (hex!("0100000000333333334444444455000000750000000000000760"), 0x00d101), + (hex!("0100000000333333334444444455000000750000000000004310"), 0x00d1a1), + (hex!("0100000000333333334444444455000000760000000000000770"), 0x00d241), + (hex!("0100000000333333334444444455000000770000000000000780"), 0x00d2e1), + (hex!("01000000003333333344444444550000007700000000000062f0"), 0x00d381), + (hex!("0100000000333333334444444455000000770000000000006940"), 0x00d421), + (hex!("0100000000333333334444444455000000780000000000000790"), 0x00d4c1), + (hex!("01000000003333333344444444550000007900000000000007a0"), 0x00d561), + (hex!("0100000000333333334444444455000000790000000000007af0"), 0x00d601), + (hex!("01000000003333333344444444550000007a00000000000007b0"), 0x00d6a1), + (hex!("01000000003333333344444444550000007b00000000000007c0"), 0x00d741), + (hex!("01000000003333333344444444550000007b00000000000067e0"), 0x00d7e1), + (hex!("01000000003333333344444444550000007b0000000000007890"), 0x00d881), + (hex!("01000000003333333344444444550000007c00000000000007d0"), 0x00d921), + (hex!("01000000003333333344444444550000007d00000000000007e0"), 0x00d9c1), + (hex!("01000000003333333344444444550000007e00000000000007f0"), 0x00da61), + (hex!("01000000003333333344444444550000007f0000000000000800"), 0x00db01), + (hex!("01000000003333333344444444550000007f0000000000005be0"), 0x00dba1), + (hex!("0100000000333333334444444455000000800000000000000810"), 0x00dc41), + (hex!("0100000000333333334444444455000000810000000000000820"), 0x00dce1), + (hex!("0100000000333333334444444455000000810000000000007190"), 0x00dd81), + (hex!("0100000000333333334444444455000000820000000000000830"), 0x00de21), + (hex!("0100000000333333334444444455000000820000000000004ab0"), 0x00dec1), + (hex!("0100000000333333334444444455000000830000000000000840"), 0x00df61), + (hex!("0100000000333333334444444455000000830000000000006720"), 0x00e001), + (hex!("0100000000333333334444444455000000840000000000000850"), 0x00e0a1), + (hex!("0100000000333333334444444455000000850000000000000860"), 0x00e141), + (hex!("01000000003333333344444444550000008500000000000054f0"), 0x00e1e1), + (hex!("0100000000333333334444444455000000850000000000007920"), 0x00e281), + (hex!("0100000000333333334444444455000000860000000000000870"), 0x00e321), + (hex!("01000000003333333344444444550000008600000000000060e0"), 0x00e3c1), + (hex!("0100000000333333334444444455000000860000000000006be0"), 0x00e461), + (hex!("0100000000333333334444444455000000870000000000000880"), 0x00e501), + (hex!("0100000000333333334444444455000000870000000000006820"), 0x00e5a1), + (hex!("0100000000333333334444444455000000880000000000000890"), 0x00e641), + (hex!("01000000003333333344444444550000008900000000000008a0"), 0x00e6e1), + (hex!("0100000000333333334444444455000000890000000000007c30"), 0x00e781), + (hex!("01000000003333333344444444550000008a00000000000008b0"), 0x00e821), + (hex!("01000000003333333344444444550000008b00000000000008c0"), 0x00e8c1), + (hex!("01000000003333333344444444550000008b0000000000005910"), 0x00e961), + (hex!("01000000003333333344444444550000008b0000000000006fe0"), 0x00ea01), + (hex!("01000000003333333344444444550000008c00000000000008d0"), 0x00eaa1), + (hex!("01000000003333333344444444550000008c0000000000006800"), 0x00eb41), + (hex!("01000000003333333344444444550000008d00000000000008e0"), 0x00ebe1), + (hex!("01000000003333333344444444550000008d0000000000005810"), 0x00ec81), + (hex!("01000000003333333344444444550000008d0000000000007c90"), 0x00ed21), + (hex!("01000000003333333344444444550000008e00000000000008f0"), 0x00edc1), + (hex!("01000000003333333344444444550000008e00000000000058f0"), 0x00ee61), + (hex!("01000000003333333344444444550000008f0000000000000900"), 0x00ef01), + (hex!("01000000003333333344444444550000008f0000000000005a30"), 0x00efa1), + (hex!("0100000000333333334444444455000000900000000000000910"), 0x00f041), + (hex!("0100000000333333334444444455000000900000000000006130"), 0x00f0e1), + (hex!("0100000000333333334444444455000000900000000000006550"), 0x00f181), + (hex!("0100000000333333334444444455000000910000000000000920"), 0x00f221), + (hex!("01000000003333333344444444550000009100000000000079f0"), 0x00f2c1), + (hex!("0100000000333333334444444455000000920000000000000930"), 0x00f361), + (hex!("0100000000333333334444444455000000920000000000005620"), 0x00f401), + (hex!("0100000000333333334444444455000000920000000000005e90"), 0x00f4a1), + (hex!("01000000003333333344444444550000009200000000000063d0"), 0x00f541), + (hex!("01000000003333333344444444550000009200000000000076c0"), 0x00f5e1), + (hex!("0100000000333333334444444455000000930000000000000940"), 0x00f681), + (hex!("01000000003333333344444444550000009300000000000044e0"), 0x00f721), + (hex!("0100000000333333334444444455000000940000000000000950"), 0x00f7c1), + (hex!("0100000000333333334444444455000000940000000000007a30"), 0x00f861), + (hex!("0100000000333333334444444455000000950000000000000960"), 0x00f901), + (hex!("0100000000333333334444444455000000950000000000007a70"), 0x00f9a1), + (hex!("0100000000333333334444444455000000960000000000000970"), 0x00fa41), + (hex!("0100000000333333334444444455000000970000000000000980"), 0x00fae1), + (hex!("0100000000333333334444444455000000970000000000007330"), 0x00fb81), + (hex!("0100000000333333334444444455000000980000000000000990"), 0x00fc21), + (hex!("0100000000333333334444444455000000980000000000005af0"), 0x00fcc1), + (hex!("0100000000333333334444444455000000980000000000007ae0"), 0x00fd61), + (hex!("01000000003333333344444444550000009900000000000009a0"), 0x00fe01), + (hex!("0100000000333333334444444455000000990000000000005160"), 0x00fea1), + (hex!("0100000000333333334444444455000000990000000000006850"), 0x00ff41), + (hex!("01000000003333333344444444550000009a00000000000009b0"), 0x00ffe1), + (hex!("01000000003333333344444444550000009b00000000000009c0"), 0x010081), + (hex!("01000000003333333344444444550000009b0000000000005010"), 0x010121), + (hex!("01000000003333333344444444550000009c00000000000009d0"), 0x0101c1), + (hex!("01000000003333333344444444550000009c00000000000042e0"), 0x010261), + (hex!("01000000003333333344444444550000009d00000000000009e0"), 0x010301), + (hex!("01000000003333333344444444550000009d00000000000057f0"), 0x0103a1), + (hex!("01000000003333333344444444550000009e00000000000009f0"), 0x010441), + (hex!("01000000003333333344444444550000009e0000000000004ef0"), 0x0104e1), + (hex!("01000000003333333344444444550000009f0000000000000a00"), 0x010581), + (hex!("01000000003333333344444444550000009f0000000000006110"), 0x010621), + (hex!("0100000000333333334444444455000000a00000000000000a10"), 0x0106c1), + (hex!("0100000000333333334444444455000000a10000000000000a20"), 0x010761), + (hex!("0100000000333333334444444455000000a100000000000040d0"), 0x010801), + (hex!("0100000000333333334444444455000000a10000000000007670"), 0x0108a1), + (hex!("0100000000333333334444444455000000a20000000000000a30"), 0x010941), + (hex!("0100000000333333334444444455000000a200000000000074d0"), 0x0109e1), + (hex!("0100000000333333334444444455000000a30000000000000a40"), 0x010a81), + (hex!("0100000000333333334444444455000000a30000000000004c90"), 0x010b21), + (hex!("0100000000333333334444444455000000a40000000000000a50"), 0x010bc1), + (hex!("0100000000333333334444444455000000a50000000000000a60"), 0x010c61), + (hex!("0100000000333333334444444455000000a60000000000000a70"), 0x010d01), + (hex!("0100000000333333334444444455000000a60000000000006d80"), 0x010da1), + (hex!("0100000000333333334444444455000000a60000000000007830"), 0x010e41), + (hex!("0100000000333333334444444455000000a70000000000000a80"), 0x010ee1), + (hex!("0100000000333333334444444455000000a700000000000064f0"), 0x010f81), + (hex!("0100000000333333334444444455000000a80000000000000a90"), 0x011021), + (hex!("0100000000333333334444444455000000a90000000000000aa0"), 0x0110c1), + (hex!("0100000000333333334444444455000000a90000000000005e30"), 0x011161), + (hex!("0100000000333333334444444455000000aa0000000000000ab0"), 0x011201), + (hex!("0100000000333333334444444455000000ab0000000000000ac0"), 0x0112a1), + (hex!("0100000000333333334444444455000000ac0000000000000ad0"), 0x011341), + (hex!("0100000000333333334444444455000000ac0000000000006d20"), 0x0113e1), + (hex!("0100000000333333334444444455000000ac0000000000007000"), 0x011481), + (hex!("0100000000333333334444444455000000ad0000000000000ae0"), 0x011521), + (hex!("0100000000333333334444444455000000ae0000000000000af0"), 0x0115c1), + (hex!("0100000000333333334444444455000000ae0000000000004a10"), 0x011661), + (hex!("0100000000333333334444444455000000af0000000000000b00"), 0x011701), + (hex!("0100000000333333334444444455000000af0000000000004e10"), 0x0117a1), + (hex!("0100000000333333334444444455000000b00000000000000b10"), 0x011841), + (hex!("0100000000333333334444444455000000b00000000000004280"), 0x0118e1), + (hex!("0100000000333333334444444455000000b000000000000077e0"), 0x011981), + (hex!("0100000000333333334444444455000000b10000000000000b20"), 0x011a21), + (hex!("0100000000333333334444444455000000b20000000000000b30"), 0x011ac1), + (hex!("0100000000333333334444444455000000b30000000000000b40"), 0x011b61), + (hex!("0100000000333333334444444455000000b30000000000004bc0"), 0x011c01), + (hex!("0100000000333333334444444455000000b40000000000000b50"), 0x011ca1), + (hex!("0100000000333333334444444455000000b50000000000000b60"), 0x011d41), + (hex!("0100000000333333334444444455000000b50000000000004fa0"), 0x011de1), + (hex!("0100000000333333334444444455000000b50000000000006a60"), 0x011e81), + (hex!("0100000000333333334444444455000000b60000000000000b70"), 0x011f21), + (hex!("0100000000333333334444444455000000b60000000000005630"), 0x011fc1), + (hex!("0100000000333333334444444455000000b70000000000000b80"), 0x012061), + (hex!("0100000000333333334444444455000000b80000000000000b90"), 0x012101), + (hex!("0100000000333333334444444455000000b80000000000006f80"), 0x0121a1), + (hex!("0100000000333333334444444455000000b90000000000000ba0"), 0x012241), + (hex!("0100000000333333334444444455000000ba0000000000000bb0"), 0x0122e1), + (hex!("0100000000333333334444444455000000bb0000000000000bc0"), 0x012381), + (hex!("0100000000333333334444444455000000bb00000000000047c0"), 0x012421), + (hex!("0100000000333333334444444455000000bb0000000000006060"), 0x0124c1), + (hex!("0100000000333333334444444455000000bc0000000000000bd0"), 0x012561), + (hex!("0100000000333333334444444455000000bd0000000000000be0"), 0x012601), + (hex!("0100000000333333334444444455000000bd0000000000004e80"), 0x0126a1), + (hex!("0100000000333333334444444455000000be0000000000000bf0"), 0x012741), + (hex!("0100000000333333334444444455000000bf0000000000000c00"), 0x0127e1), + (hex!("0100000000333333334444444455000000bf00000000000047a0"), 0x012881), + (hex!("0100000000333333334444444455000000bf0000000000006da0"), 0x012921), + (hex!("0100000000333333334444444455000000c00000000000000c10"), 0x0129c1), + (hex!("0100000000333333334444444455000000c10000000000000c20"), 0x012a61), + (hex!("0100000000333333334444444455000000c20000000000000c30"), 0x012b01), + (hex!("0100000000333333334444444455000000c20000000000004bd0"), 0x012ba1), + (hex!("0100000000333333334444444455000000c20000000000006ac0"), 0x012c41), + (hex!("0100000000333333334444444455000000c30000000000000c40"), 0x012ce1), + (hex!("0100000000333333334444444455000000c30000000000004660"), 0x012d81), + (hex!("0100000000333333334444444455000000c40000000000000c50"), 0x012e21), + (hex!("0100000000333333334444444455000000c50000000000000c60"), 0x012ec1), + (hex!("0100000000333333334444444455000000c60000000000000c70"), 0x012f61), + (hex!("0100000000333333334444444455000000c60000000000005880"), 0x013001), + (hex!("0100000000333333334444444455000000c60000000000006b70"), 0x0130a1), + (hex!("0100000000333333334444444455000000c70000000000000c80"), 0x013141), + (hex!("0100000000333333334444444455000000c80000000000000c90"), 0x0131e1), + (hex!("0100000000333333334444444455000000c80000000000005310"), 0x013281), + (hex!("0100000000333333334444444455000000c80000000000005db0"), 0x013321), + (hex!("0100000000333333334444444455000000c80000000000007040"), 0x0133c1), + (hex!("0100000000333333334444444455000000c80000000000007290"), 0x013461), + (hex!("0100000000333333334444444455000000c90000000000000ca0"), 0x013501), + (hex!("0100000000333333334444444455000000c90000000000004fe0"), 0x0135a1), + (hex!("0100000000333333334444444455000000ca0000000000000cb0"), 0x013641), + (hex!("0100000000333333334444444455000000ca0000000000006140"), 0x0136e1), + (hex!("0100000000333333334444444455000000ca0000000000007700"), 0x013781), + (hex!("0100000000333333334444444455000000cb0000000000000cc0"), 0x013821), + (hex!("0100000000333333334444444455000000cc0000000000000cd0"), 0x0138c1), + (hex!("0100000000333333334444444455000000cd0000000000000ce0"), 0x013961), + (hex!("0100000000333333334444444455000000cd0000000000003f20"), 0x013a01), + (hex!("0100000000333333334444444455000000cd00000000000040f0"), 0x013aa1), + (hex!("0100000000333333334444444455000000cd0000000000004ec0"), 0x013b41), + (hex!("0100000000333333334444444455000000ce0000000000000cf0"), 0x013be1), + (hex!("0100000000333333334444444455000000ce0000000000007200"), 0x013c81), + (hex!("0100000000333333334444444455000000cf0000000000000d00"), 0x013d21), + (hex!("0100000000333333334444444455000000cf00000000000046a0"), 0x013dc1), + (hex!("0100000000333333334444444455000000cf0000000000005960"), 0x013e61), + (hex!("0100000000333333334444444455000000d00000000000000d10"), 0x013f01), + (hex!("0100000000333333334444444455000000d00000000000005f30"), 0x013fa1), + (hex!("0100000000333333334444444455000000d10000000000000d20"), 0x014041), + (hex!("0100000000333333334444444455000000d10000000000007a00"), 0x0140e1), + (hex!("0100000000333333334444444455000000d20000000000000d30"), 0x014181), + (hex!("0100000000333333334444444455000000d30000000000000d40"), 0x014221), + (hex!("0100000000333333334444444455000000d40000000000000d50"), 0x0142c1), + (hex!("0100000000333333334444444455000000d50000000000000d60"), 0x014361), + (hex!("0100000000333333334444444455000000d50000000000004960"), 0x014401), + (hex!("0100000000333333334444444455000000d500000000000055d0"), 0x0144a1), + (hex!("0100000000333333334444444455000000d500000000000067d0"), 0x014541), + (hex!("0100000000333333334444444455000000d60000000000000d70"), 0x0145e1), + (hex!("0100000000333333334444444455000000d70000000000000d80"), 0x014681), + (hex!("0100000000333333334444444455000000d80000000000000d90"), 0x014721), + (hex!("0100000000333333334444444455000000d800000000000065f0"), 0x0147c1), + (hex!("0100000000333333334444444455000000d90000000000000da0"), 0x014861), + (hex!("0100000000333333334444444455000000d90000000000004980"), 0x014901), + (hex!("0100000000333333334444444455000000da0000000000000db0"), 0x0149a1), + (hex!("0100000000333333334444444455000000da00000000000048c0"), 0x014a41), + (hex!("0100000000333333334444444455000000da00000000000072c0"), 0x014ae1), + (hex!("0100000000333333334444444455000000da00000000000076b0"), 0x014b81), + (hex!("0100000000333333334444444455000000db0000000000000dc0"), 0x014c21), + (hex!("0100000000333333334444444455000000dc0000000000000dd0"), 0x014cc1), + (hex!("0100000000333333334444444455000000dc00000000000040a0"), 0x014d61), + (hex!("0100000000333333334444444455000000dc00000000000074c0"), 0x014e01), + (hex!("0100000000333333334444444455000000dd0000000000000de0"), 0x014ea1), + (hex!("0100000000333333334444444455000000dd0000000000004e50"), 0x014f41), + (hex!("0100000000333333334444444455000000dd0000000000007270"), 0x014fe1), + (hex!("0100000000333333334444444455000000de0000000000000df0"), 0x015081), + (hex!("0100000000333333334444444455000000de00000000000078d0"), 0x015121), + (hex!("0100000000333333334444444455000000df0000000000000e00"), 0x0151c1), + (hex!("0100000000333333334444444455000000df0000000000004d30"), 0x015261), + (hex!("0100000000333333334444444455000000df0000000000006c30"), 0x015301), + (hex!("0100000000333333334444444455000000e00000000000000e10"), 0x0153a1), + (hex!("0100000000333333334444444455000000e00000000000005d30"), 0x015441), + (hex!("0100000000333333334444444455000000e10000000000000e20"), 0x0154e1), + (hex!("0100000000333333334444444455000000e10000000000004610"), 0x015581), + (hex!("0100000000333333334444444455000000e100000000000051d0"), 0x015621), + (hex!("0100000000333333334444444455000000e10000000000005f10"), 0x0156c1), + (hex!("0100000000333333334444444455000000e20000000000000e30"), 0x015761), + (hex!("0100000000333333334444444455000000e20000000000007a90"), 0x015801), + (hex!("0100000000333333334444444455000000e30000000000000e40"), 0x0158a1), + (hex!("0100000000333333334444444455000000e30000000000005ae0"), 0x015941), + (hex!("0100000000333333334444444455000000e40000000000000e50"), 0x0159e1), + (hex!("0100000000333333334444444455000000e50000000000000e60"), 0x015a81), + (hex!("0100000000333333334444444455000000e50000000000004700"), 0x015b21), + (hex!("0100000000333333334444444455000000e500000000000065d0"), 0x015bc1), + (hex!("0100000000333333334444444455000000e60000000000000e70"), 0x015c61), + (hex!("0100000000333333334444444455000000e60000000000004fd0"), 0x015d01), + (hex!("0100000000333333334444444455000000e70000000000000e80"), 0x015da1), + (hex!("0100000000333333334444444455000000e70000000000005150"), 0x015e41), + (hex!("0100000000333333334444444455000000e70000000000005920"), 0x015ee1), + (hex!("0100000000333333334444444455000000e80000000000000e90"), 0x015f81), + (hex!("0100000000333333334444444455000000e80000000000004320"), 0x016021), + (hex!("0100000000333333334444444455000000e80000000000005ec0"), 0x0160c1), + (hex!("0100000000333333334444444455000000e90000000000000ea0"), 0x016161), + (hex!("0100000000333333334444444455000000e900000000000043b0"), 0x016201), + (hex!("0100000000333333334444444455000000ea0000000000000eb0"), 0x0162a1), + (hex!("0100000000333333334444444455000000ea0000000000003ea0"), 0x016341), + (hex!("0100000000333333334444444455000000ea0000000000004f50"), 0x0163e1), + (hex!("0100000000333333334444444455000000ea0000000000007520"), 0x016481), + (hex!("0100000000333333334444444455000000eb0000000000000ec0"), 0x016521), + (hex!("0100000000333333334444444455000000ec0000000000000ed0"), 0x0165c1), + (hex!("0100000000333333334444444455000000ec0000000000006670"), 0x016661), + (hex!("0100000000333333334444444455000000ed0000000000000ee0"), 0x016701), + (hex!("0100000000333333334444444455000000ee0000000000000ef0"), 0x0167a1), + (hex!("0100000000333333334444444455000000ee0000000000004d10"), 0x016841), + (hex!("0100000000333333334444444455000000ef0000000000000f00"), 0x0168e1), + (hex!("0100000000333333334444444455000000f00000000000000f10"), 0x016981), + (hex!("0100000000333333334444444455000000f00000000000007220"), 0x016a21), + (hex!("0100000000333333334444444455000000f00000000000007540"), 0x016ac1), + (hex!("0100000000333333334444444455000000f10000000000000f20"), 0x016b61), + (hex!("0100000000333333334444444455000000f100000000000066f0"), 0x016c01), + (hex!("0100000000333333334444444455000000f20000000000000f30"), 0x016ca1), + (hex!("0100000000333333334444444455000000f20000000000007810"), 0x016d41), + (hex!("0100000000333333334444444455000000f30000000000000f40"), 0x016de1), + (hex!("0100000000333333334444444455000000f30000000000007b70"), 0x016e81), + (hex!("0100000000333333334444444455000000f40000000000000f50"), 0x016f21), + (hex!("0100000000333333334444444455000000f400000000000059c0"), 0x016fc1), + (hex!("0100000000333333334444444455000000f50000000000000f60"), 0x017061), + (hex!("0100000000333333334444444455000000f50000000000003fb0"), 0x017101), + (hex!("0100000000333333334444444455000000f50000000000005740"), 0x0171a1), + (hex!("0100000000333333334444444455000000f500000000000064d0"), 0x017241), + (hex!("0100000000333333334444444455000000f50000000000006960"), 0x0172e1), + (hex!("0100000000333333334444444455000000f60000000000000f70"), 0x017381), + (hex!("0100000000333333334444444455000000f60000000000006d00"), 0x017421), + (hex!("0100000000333333334444444455000000f70000000000000f80"), 0x0174c1), + (hex!("0100000000333333334444444455000000f80000000000000f90"), 0x017561), + (hex!("0100000000333333334444444455000000f90000000000000fa0"), 0x017601), + (hex!("0100000000333333334444444455000000fa0000000000000fb0"), 0x0176a1), + (hex!("0100000000333333334444444455000000fa00000000000067b0"), 0x017741), + (hex!("0100000000333333334444444455000000fb0000000000000fc0"), 0x0177e1), + (hex!("0100000000333333334444444455000000fb0000000000004eb0"), 0x017881), + (hex!("0100000000333333334444444455000000fb0000000000006ef0"), 0x017921), + (hex!("0100000000333333334444444455000000fc0000000000000fd0"), 0x0179c1), + (hex!("0100000000333333334444444455000000fc0000000000004470"), 0x017a61), + (hex!("0100000000333333334444444455000000fc0000000000005940"), 0x017b01), + (hex!("0100000000333333334444444455000000fd0000000000000fe0"), 0x017ba1), + (hex!("0100000000333333334444444455000000fe0000000000000ff0"), 0x017c41), + (hex!("0100000000333333334444444455000000ff0000000000001000"), 0x017ce1), + (hex!("0100000000333333334444444455000000ff0000000000005690"), 0x017d81), + (hex!("0100000000333333334444444455000001000000000000001010"), 0x017e21), + (hex!("0100000000333333334444444455000001000000000000005210"), 0x017ec1), + (hex!("01000000003333333344444444550000010000000000000070a0"), 0x017f61), + (hex!("0100000000333333334444444455000001010000000000001020"), 0x018001), + (hex!("0100000000333333334444444455000001010000000000006b80"), 0x0180a1), + (hex!("0100000000333333334444444455000001020000000000001030"), 0x018141), + (hex!("0100000000333333334444444455000001030000000000001040"), 0x0181e1), + (hex!("0100000000333333334444444455000001030000000000004c80"), 0x018281), + (hex!("0100000000333333334444444455000001040000000000001050"), 0x018321), + (hex!("0100000000333333334444444455000001040000000000004850"), 0x0183c1), + (hex!("01000000003333333344444444550000010400000000000057b0"), 0x018461), + (hex!("0100000000333333334444444455000001050000000000001060"), 0x018501), + (hex!("01000000003333333344444444550000010500000000000048d0"), 0x0185a1), + (hex!("0100000000333333334444444455000001050000000000007870"), 0x018641), + (hex!("0100000000333333334444444455000001060000000000001070"), 0x0186e1), + (hex!("0100000000333333334444444455000001060000000000004f90"), 0x018781), + (hex!("0100000000333333334444444455000001060000000000006270"), 0x018821), + (hex!("0100000000333333334444444455000001070000000000001080"), 0x0188c1), + (hex!("01000000003333333344444444550000010700000000000063b0"), 0x018961), + (hex!("0100000000333333334444444455000001080000000000001090"), 0x018a01), + (hex!("01000000003333333344444444550000010900000000000010a0"), 0x018aa1), + (hex!("0100000000333333334444444455000001090000000000006f40"), 0x018b41), + (hex!("01000000003333333344444444550000010a00000000000010b0"), 0x018be1), + (hex!("01000000003333333344444444550000010a0000000000006640"), 0x018c81), + (hex!("01000000003333333344444444550000010b00000000000010c0"), 0x018d21), + (hex!("01000000003333333344444444550000010c00000000000010d0"), 0x018dc1), + (hex!("01000000003333333344444444550000010d00000000000010e0"), 0x018e61), + (hex!("01000000003333333344444444550000010e00000000000010f0"), 0x018f01), + (hex!("01000000003333333344444444550000010e0000000000005c40"), 0x018fa1), + (hex!("01000000003333333344444444550000010e0000000000007ba0"), 0x019041), + (hex!("01000000003333333344444444550000010f0000000000001100"), 0x0190e1), + (hex!("01000000003333333344444444550000010f0000000000005c30"), 0x019181), + (hex!("0100000000333333334444444455000001100000000000001110"), 0x019221), + (hex!("0100000000333333334444444455000001100000000000007640"), 0x0192c1), + (hex!("0100000000333333334444444455000001110000000000001120"), 0x019361), + (hex!("01000000003333333344444444550000011100000000000052c0"), 0x019401), + (hex!("0100000000333333334444444455000001110000000000005710"), 0x0194a1), + (hex!("0100000000333333334444444455000001110000000000006a00"), 0x019541), + (hex!("0100000000333333334444444455000001120000000000001130"), 0x0195e1), + (hex!("0100000000333333334444444455000001130000000000001140"), 0x019681), + (hex!("0100000000333333334444444455000001140000000000001150"), 0x019721), + (hex!("0100000000333333334444444455000001140000000000003fa0"), 0x0197c1), + (hex!("01000000003333333344444444550000011400000000000054b0"), 0x019861), + (hex!("0100000000333333334444444455000001140000000000006070"), 0x019901), + (hex!("0100000000333333334444444455000001150000000000001160"), 0x0199a1), + (hex!("0100000000333333334444444455000001150000000000005320"), 0x019a41), + (hex!("0100000000333333334444444455000001150000000000006600"), 0x019ae1), + (hex!("0100000000333333334444444455000001150000000000006df0"), 0x019b81), + (hex!("01000000003333333344444444550000011500000000000079c0"), 0x019c21), + (hex!("0100000000333333334444444455000001160000000000001170"), 0x019cc1), + (hex!("0100000000333333334444444455000001170000000000001180"), 0x019d61), + (hex!("0100000000333333334444444455000001170000000000004a60"), 0x019e01), + (hex!("01000000003333333344444444550000011700000000000063c0"), 0x019ea1), + (hex!("0100000000333333334444444455000001180000000000001190"), 0x019f41), + (hex!("0100000000333333334444444455000001180000000000004530"), 0x019fe1), + (hex!("01000000003333333344444444550000011800000000000077c0"), 0x01a081), + (hex!("01000000003333333344444444550000011900000000000011a0"), 0x01a121), + (hex!("01000000003333333344444444550000011a00000000000011b0"), 0x01a1c1), + (hex!("01000000003333333344444444550000011a00000000000041c0"), 0x01a261), + (hex!("01000000003333333344444444550000011a00000000000061e0"), 0x01a301), + (hex!("01000000003333333344444444550000011b00000000000011c0"), 0x01a3a1), + (hex!("01000000003333333344444444550000011c00000000000011d0"), 0x01a441), + (hex!("01000000003333333344444444550000011c0000000000005f90"), 0x01a4e1), + (hex!("01000000003333333344444444550000011d00000000000011e0"), 0x01a581), + (hex!("01000000003333333344444444550000011d0000000000004160"), 0x01a621), + (hex!("01000000003333333344444444550000011e00000000000011f0"), 0x01a6c1), + (hex!("01000000003333333344444444550000011e00000000000056d0"), 0x01a761), + (hex!("01000000003333333344444444550000011f0000000000001200"), 0x01a801), + (hex!("01000000003333333344444444550000011f0000000000004510"), 0x01a8a1), + (hex!("0100000000333333334444444455000001200000000000001210"), 0x01a941), + (hex!("0100000000333333334444444455000001210000000000001220"), 0x01a9e1), + (hex!("0100000000333333334444444455000001210000000000005140"), 0x01aa81), + (hex!("0100000000333333334444444455000001210000000000006710"), 0x01ab21), + (hex!("0100000000333333334444444455000001210000000000006f50"), 0x01abc1), + (hex!("0100000000333333334444444455000001220000000000001230"), 0x01ac61), + (hex!("0100000000333333334444444455000001220000000000005570"), 0x01ad01), + (hex!("0100000000333333334444444455000001220000000000007ac0"), 0x01ada1), + (hex!("0100000000333333334444444455000001230000000000001240"), 0x01ae41), + (hex!("0100000000333333334444444455000001240000000000001250"), 0x01aee1), + (hex!("0100000000333333334444444455000001240000000000006cd0"), 0x01af81), + (hex!("0100000000333333334444444455000001250000000000001260"), 0x01b021), + (hex!("01000000003333333344444444550000012500000000000046b0"), 0x01b0c1), + (hex!("0100000000333333334444444455000001250000000000005eb0"), 0x01b161), + (hex!("0100000000333333334444444455000001260000000000001270"), 0x01b201), + (hex!("0100000000333333334444444455000001260000000000004630"), 0x01b2a1), + (hex!("0100000000333333334444444455000001270000000000001280"), 0x01b341), + (hex!("0100000000333333334444444455000001270000000000004ff0"), 0x01b3e1), + (hex!("0100000000333333334444444455000001270000000000006ec0"), 0x01b481), + (hex!("0100000000333333334444444455000001280000000000001290"), 0x01b521), + (hex!("01000000003333333344444444550000012900000000000012a0"), 0x01b5c1), + (hex!("0100000000333333334444444455000001290000000000005f60"), 0x01b661), + (hex!("01000000003333333344444444550000012a00000000000012b0"), 0x01b701), + (hex!("01000000003333333344444444550000012a0000000000005480"), 0x01b7a1), + (hex!("01000000003333333344444444550000012b00000000000012c0"), 0x01b841), + (hex!("01000000003333333344444444550000012b00000000000065a0"), 0x01b8e1), + (hex!("01000000003333333344444444550000012b00000000000066c0"), 0x01b981), + (hex!("01000000003333333344444444550000012c00000000000012d0"), 0x01ba21), + (hex!("01000000003333333344444444550000012c00000000000064b0"), 0x01bac1), + (hex!("01000000003333333344444444550000012d00000000000012e0"), 0x01bb61), + (hex!("01000000003333333344444444550000012d00000000000049c0"), 0x01bc01), + (hex!("01000000003333333344444444550000012d0000000000004bf0"), 0x01bca1), + (hex!("01000000003333333344444444550000012e00000000000012f0"), 0x01bd41), + (hex!("01000000003333333344444444550000012e0000000000005ed0"), 0x01bde1), + (hex!("01000000003333333344444444550000012f0000000000001300"), 0x01be81), + (hex!("01000000003333333344444444550000012f00000000000049a0"), 0x01bf21), + (hex!("0100000000333333334444444455000001300000000000001310"), 0x01bfc1), + (hex!("0100000000333333334444444455000001300000000000007840"), 0x01c061), + (hex!("0100000000333333334444444455000001310000000000001320"), 0x01c101), + (hex!("0100000000333333334444444455000001310000000000005f70"), 0x01c1a1), + (hex!("0100000000333333334444444455000001320000000000001330"), 0x01c241), + (hex!("0100000000333333334444444455000001320000000000005a00"), 0x01c2e1), + (hex!("0100000000333333334444444455000001330000000000001340"), 0x01c381), + (hex!("0100000000333333334444444455000001330000000000006c70"), 0x01c421), + (hex!("0100000000333333334444444455000001340000000000001350"), 0x01c4c1), + (hex!("0100000000333333334444444455000001340000000000005c60"), 0x01c561), + (hex!("0100000000333333334444444455000001350000000000001360"), 0x01c601), + (hex!("0100000000333333334444444455000001350000000000004f10"), 0x01c6a1), + (hex!("0100000000333333334444444455000001360000000000001370"), 0x01c741), + (hex!("0100000000333333334444444455000001360000000000004c60"), 0x01c7e1), + (hex!("0100000000333333334444444455000001370000000000001380"), 0x01c881), + (hex!("0100000000333333334444444455000001380000000000001390"), 0x01c921), + (hex!("01000000003333333344444444550000013900000000000013a0"), 0x01c9c1), + (hex!("0100000000333333334444444455000001390000000000004ea0"), 0x01ca61), + (hex!("01000000003333333344444444550000013a00000000000013b0"), 0x01cb01), + (hex!("01000000003333333344444444550000013a0000000000007350"), 0x01cba1), + (hex!("01000000003333333344444444550000013b00000000000013c0"), 0x01cc41), + (hex!("01000000003333333344444444550000013c00000000000013d0"), 0x01cce1), + (hex!("01000000003333333344444444550000013c0000000000007050"), 0x01cd81), + (hex!("01000000003333333344444444550000013d00000000000013e0"), 0x01ce21), + (hex!("01000000003333333344444444550000013d0000000000006bd0"), 0x01cec1), + (hex!("01000000003333333344444444550000013e00000000000013f0"), 0x01cf61), + (hex!("01000000003333333344444444550000013e00000000000058e0"), 0x01d001), + (hex!("01000000003333333344444444550000013f0000000000001400"), 0x01d0a1), + (hex!("01000000003333333344444444550000013f0000000000004740"), 0x01d141), + (hex!("0100000000333333334444444455000001400000000000001410"), 0x01d1e1), + (hex!("0100000000333333334444444455000001400000000000003f10"), 0x01d281), + (hex!("0100000000333333334444444455000001400000000000006d40"), 0x01d321), + (hex!("01000000003333333344444444550000014000000000000072d0"), 0x01d3c1), + (hex!("0100000000333333334444444455000001410000000000001420"), 0x01d461), + (hex!("0100000000333333334444444455000001420000000000001430"), 0x01d501), + (hex!("0100000000333333334444444455000001430000000000001440"), 0x01d5a1), + (hex!("0100000000333333334444444455000001440000000000001450"), 0x01d641), + (hex!("0100000000333333334444444455000001450000000000001460"), 0x01d6e1), + (hex!("0100000000333333334444444455000001460000000000001470"), 0x01d781), + (hex!("01000000003333333344444444550000014600000000000055c0"), 0x01d821), + (hex!("0100000000333333334444444455000001470000000000001480"), 0x01d8c1), + (hex!("0100000000333333334444444455000001470000000000004570"), 0x01d961), + (hex!("0100000000333333334444444455000001470000000000004be0"), 0x01da01), + (hex!("0100000000333333334444444455000001480000000000001490"), 0x01daa1), + (hex!("0100000000333333334444444455000001480000000000005360"), 0x01db41), + (hex!("01000000003333333344444444550000014900000000000014a0"), 0x01dbe1), + (hex!("01000000003333333344444444550000014a00000000000014b0"), 0x01dc81), + (hex!("01000000003333333344444444550000014a00000000000053d0"), 0x01dd21), + (hex!("01000000003333333344444444550000014b00000000000014c0"), 0x01ddc1), + (hex!("01000000003333333344444444550000014b0000000000005950"), 0x01de61), + (hex!("01000000003333333344444444550000014c00000000000014d0"), 0x01df01), + (hex!("01000000003333333344444444550000014c0000000000004f60"), 0x01dfa1), + (hex!("01000000003333333344444444550000014d00000000000014e0"), 0x01e041), + (hex!("01000000003333333344444444550000014d0000000000004520"), 0x01e0e1), + (hex!("01000000003333333344444444550000014d0000000000005200"), 0x01e181), + (hex!("01000000003333333344444444550000014e00000000000014f0"), 0x01e221), + (hex!("01000000003333333344444444550000014e0000000000005bd0"), 0x01e2c1), + (hex!("01000000003333333344444444550000014f0000000000001500"), 0x01e361), + (hex!("01000000003333333344444444550000014f00000000000060d0"), 0x01e401), + (hex!("0100000000333333334444444455000001500000000000001510"), 0x01e4a1), + (hex!("01000000003333333344444444550000015000000000000075e0"), 0x01e541), + (hex!("0100000000333333334444444455000001510000000000001520"), 0x01e5e1), + (hex!("0100000000333333334444444455000001510000000000005c00"), 0x01e681), + (hex!("0100000000333333334444444455000001510000000000006af0"), 0x01e721), + (hex!("0100000000333333334444444455000001510000000000007b80"), 0x01e7c1), + (hex!("0100000000333333334444444455000001520000000000001530"), 0x01e861), + (hex!("0100000000333333334444444455000001520000000000004c70"), 0x01e901), + (hex!("0100000000333333334444444455000001530000000000001540"), 0x01e9a1), + (hex!("0100000000333333334444444455000001540000000000001550"), 0x01ea41), + (hex!("0100000000333333334444444455000001540000000000007cd0"), 0x01eae1), + (hex!("0100000000333333334444444455000001550000000000001560"), 0x01eb81), + (hex!("0100000000333333334444444455000001550000000000004ae0"), 0x01ec21), + (hex!("01000000003333333344444444550000015500000000000068c0"), 0x01ecc1), + (hex!("0100000000333333334444444455000001560000000000001570"), 0x01ed61), + (hex!("01000000003333333344444444550000015600000000000064a0"), 0x01ee01), + (hex!("0100000000333333334444444455000001570000000000001580"), 0x01eea1), + (hex!("0100000000333333334444444455000001580000000000001590"), 0x01ef41), + (hex!("0100000000333333334444444455000001580000000000006d30"), 0x01efe1), + (hex!("01000000003333333344444444550000015800000000000074f0"), 0x01f081), + (hex!("01000000003333333344444444550000015900000000000015a0"), 0x01f121), + (hex!("01000000003333333344444444550000015900000000000053a0"), 0x01f1c1), + (hex!("01000000003333333344444444550000015900000000000055e0"), 0x01f261), + (hex!("0100000000333333334444444455000001590000000000006210"), 0x01f301), + (hex!("01000000003333333344444444550000015900000000000067c0"), 0x01f3a1), + (hex!("01000000003333333344444444550000015a00000000000015b0"), 0x01f441), + (hex!("01000000003333333344444444550000015b00000000000015c0"), 0x01f4e1), + (hex!("01000000003333333344444444550000015c00000000000015d0"), 0x01f581), + (hex!("01000000003333333344444444550000015c0000000000004d80"), 0x01f621), + (hex!("01000000003333333344444444550000015c00000000000073f0"), 0x01f6c1), + (hex!("01000000003333333344444444550000015d00000000000015e0"), 0x01f761), + (hex!("01000000003333333344444444550000015e00000000000015f0"), 0x01f801), + (hex!("01000000003333333344444444550000015e0000000000004120"), 0x01f8a1), + (hex!("01000000003333333344444444550000015e0000000000004350"), 0x01f941), + (hex!("01000000003333333344444444550000015e0000000000007c50"), 0x01f9e1), + (hex!("01000000003333333344444444550000015f0000000000001600"), 0x01fa81), + (hex!("0100000000333333334444444455000001600000000000001610"), 0x01fb21), + (hex!("0100000000333333334444444455000001600000000000004840"), 0x01fbc1), + (hex!("0100000000333333334444444455000001600000000000004b10"), 0x01fc61), + (hex!("0100000000333333334444444455000001600000000000007060"), 0x01fd01), + (hex!("0100000000333333334444444455000001610000000000001620"), 0x01fda1), + (hex!("0100000000333333334444444455000001610000000000005300"), 0x01fe41), + (hex!("0100000000333333334444444455000001620000000000001630"), 0x01fee1), + (hex!("0100000000333333334444444455000001620000000000006530"), 0x01ff81), + (hex!("0100000000333333334444444455000001630000000000001640"), 0x020021), + (hex!("0100000000333333334444444455000001640000000000001650"), 0x0200c1), + (hex!("0100000000333333334444444455000001650000000000001660"), 0x020161), + (hex!("0100000000333333334444444455000001660000000000001670"), 0x020201), + (hex!("0100000000333333334444444455000001670000000000001680"), 0x0202a1), + (hex!("0100000000333333334444444455000001670000000000007310"), 0x020341), + (hex!("0100000000333333334444444455000001680000000000001690"), 0x0203e1), + (hex!("0100000000333333334444444455000001680000000000007b50"), 0x020481), + (hex!("01000000003333333344444444550000016900000000000016a0"), 0x020521), + (hex!("01000000003333333344444444550000016900000000000049d0"), 0x0205c1), + (hex!("01000000003333333344444444550000016a00000000000016b0"), 0x020661), + (hex!("01000000003333333344444444550000016a00000000000078b0"), 0x020701), + (hex!("01000000003333333344444444550000016b00000000000016c0"), 0x0207a1), + (hex!("01000000003333333344444444550000016b0000000000004100"), 0x020841), + (hex!("01000000003333333344444444550000016c00000000000016d0"), 0x0208e1), + (hex!("01000000003333333344444444550000016c0000000000006e00"), 0x020981), + (hex!("01000000003333333344444444550000016d00000000000016e0"), 0x020a21), + (hex!("01000000003333333344444444550000016e00000000000016f0"), 0x020ac1), + (hex!("01000000003333333344444444550000016e0000000000004ac0"), 0x020b61), + (hex!("01000000003333333344444444550000016e0000000000007820"), 0x020c01), + (hex!("01000000003333333344444444550000016f0000000000001700"), 0x020ca1), + (hex!("0100000000333333334444444455000001700000000000001710"), 0x020d41), + (hex!("0100000000333333334444444455000001700000000000005830"), 0x020de1), + (hex!("0100000000333333334444444455000001710000000000001720"), 0x020e81), + (hex!("01000000003333333344444444550000017100000000000072f0"), 0x020f21), + (hex!("0100000000333333334444444455000001720000000000001730"), 0x020fc1), + (hex!("0100000000333333334444444455000001720000000000004870"), 0x021061), + (hex!("01000000003333333344444444550000017200000000000070b0"), 0x021101), + (hex!("0100000000333333334444444455000001730000000000001740"), 0x0211a1), + (hex!("0100000000333333334444444455000001740000000000001750"), 0x021241), + (hex!("0100000000333333334444444455000001750000000000001760"), 0x0212e1), + (hex!("0100000000333333334444444455000001750000000000005670"), 0x021381), + (hex!("0100000000333333334444444455000001750000000000005870"), 0x021421), + (hex!("0100000000333333334444444455000001760000000000001770"), 0x0214c1), + (hex!("0100000000333333334444444455000001770000000000001780"), 0x021561), + (hex!("0100000000333333334444444455000001770000000000005000"), 0x021601), + (hex!("0100000000333333334444444455000001770000000000007090"), 0x0216a1), + (hex!("0100000000333333334444444455000001780000000000001790"), 0x021741), + (hex!("01000000003333333344444444550000017800000000000048a0"), 0x0217e1), + (hex!("0100000000333333334444444455000001780000000000006bf0"), 0x021881), + (hex!("01000000003333333344444444550000017900000000000017a0"), 0x021921), + (hex!("01000000003333333344444444550000017900000000000057d0"), 0x0219c1), + (hex!("0100000000333333334444444455000001790000000000006660"), 0x021a61), + (hex!("01000000003333333344444444550000017a00000000000017b0"), 0x021b01), + (hex!("01000000003333333344444444550000017a0000000000004970"), 0x021ba1), + (hex!("01000000003333333344444444550000017a0000000000005dc0"), 0x021c41), + (hex!("01000000003333333344444444550000017b00000000000017c0"), 0x021ce1), + (hex!("01000000003333333344444444550000017b0000000000004ee0"), 0x021d81), + (hex!("01000000003333333344444444550000017b00000000000054c0"), 0x021e21), + (hex!("01000000003333333344444444550000017c00000000000017d0"), 0x021ec1), + (hex!("01000000003333333344444444550000017c0000000000003fc0"), 0x021f61), + (hex!("01000000003333333344444444550000017c00000000000063e0"), 0x022001), + (hex!("01000000003333333344444444550000017c0000000000006520"), 0x0220a1), + (hex!("01000000003333333344444444550000017d00000000000017e0"), 0x022141), + (hex!("01000000003333333344444444550000017d0000000000006220"), 0x0221e1), + (hex!("01000000003333333344444444550000017d0000000000007120"), 0x022281), + (hex!("01000000003333333344444444550000017e00000000000017f0"), 0x022321), + (hex!("01000000003333333344444444550000017f0000000000001800"), 0x0223c1), + (hex!("0100000000333333334444444455000001800000000000001810"), 0x022461), + (hex!("0100000000333333334444444455000001810000000000001820"), 0x022501), + (hex!("01000000003333333344444444550000018100000000000041f0"), 0x0225a1), + (hex!("0100000000333333334444444455000001810000000000007590"), 0x022641), + (hex!("0100000000333333334444444455000001820000000000001830"), 0x0226e1), + (hex!("0100000000333333334444444455000001820000000000004ce0"), 0x022781), + (hex!("0100000000333333334444444455000001830000000000001840"), 0x022821), + (hex!("01000000003333333344444444550000018300000000000042c0"), 0x0228c1), + (hex!("0100000000333333334444444455000001840000000000001850"), 0x022961), + (hex!("0100000000333333334444444455000001840000000000004f70"), 0x022a01), + (hex!("0100000000333333334444444455000001850000000000001860"), 0x022aa1), + (hex!("0100000000333333334444444455000001850000000000006470"), 0x022b41), + (hex!("0100000000333333334444444455000001850000000000007500"), 0x022be1), + (hex!("0100000000333333334444444455000001860000000000001870"), 0x022c81), + (hex!("0100000000333333334444444455000001860000000000004770"), 0x022d21), + (hex!("0100000000333333334444444455000001870000000000001880"), 0x022dc1), + (hex!("0100000000333333334444444455000001870000000000006a30"), 0x022e61), + (hex!("0100000000333333334444444455000001880000000000001890"), 0x022f01), + (hex!("0100000000333333334444444455000001880000000000007410"), 0x022fa1), + (hex!("01000000003333333344444444550000018900000000000018a0"), 0x023041), + (hex!("01000000003333333344444444550000018900000000000044d0"), 0x0230e1), + (hex!("0100000000333333334444444455000001890000000000005ac0"), 0x023181), + (hex!("01000000003333333344444444550000018a00000000000018b0"), 0x023221), + (hex!("01000000003333333344444444550000018a0000000000006260"), 0x0232c1), + (hex!("01000000003333333344444444550000018a0000000000006d70"), 0x023361), + (hex!("01000000003333333344444444550000018b00000000000018c0"), 0x023401), + (hex!("01000000003333333344444444550000018b0000000000004aa0"), 0x0234a1), + (hex!("01000000003333333344444444550000018b0000000000006fd0"), 0x023541), + (hex!("01000000003333333344444444550000018c00000000000018d0"), 0x0235e1), + (hex!("01000000003333333344444444550000018c00000000000051b0"), 0x023681), + (hex!("01000000003333333344444444550000018c0000000000006650"), 0x023721), + (hex!("01000000003333333344444444550000018d00000000000018e0"), 0x0237c1), + (hex!("01000000003333333344444444550000018e00000000000018f0"), 0x023861), + (hex!("01000000003333333344444444550000018e00000000000041d0"), 0x023901), + (hex!("01000000003333333344444444550000018f0000000000001900"), 0x0239a1), + (hex!("01000000003333333344444444550000018f0000000000007600"), 0x023a41), + (hex!("0100000000333333334444444455000001900000000000001910"), 0x023ae1), + (hex!("0100000000333333334444444455000001900000000000005410"), 0x023b81), + (hex!("0100000000333333334444444455000001900000000000006760"), 0x023c21), + (hex!("0100000000333333334444444455000001910000000000001920"), 0x023cc1), + (hex!("0100000000333333334444444455000001920000000000001930"), 0x023d61), + (hex!("0100000000333333334444444455000001920000000000004ca0"), 0x023e01), + (hex!("0100000000333333334444444455000001920000000000005d80"), 0x023ea1), + (hex!("0100000000333333334444444455000001920000000000005fd0"), 0x023f41), + (hex!("01000000003333333344444444550000019200000000000070d0"), 0x023fe1), + (hex!("0100000000333333334444444455000001930000000000001940"), 0x024081), + (hex!("0100000000333333334444444455000001930000000000004010"), 0x024121), + (hex!("0100000000333333334444444455000001930000000000007ca0"), 0x0241c1), + (hex!("0100000000333333334444444455000001940000000000001950"), 0x024261), + (hex!("0100000000333333334444444455000001950000000000001960"), 0x024301), + (hex!("0100000000333333334444444455000001950000000000005380"), 0x0243a1), + (hex!("0100000000333333334444444455000001960000000000001970"), 0x024441), + (hex!("0100000000333333334444444455000001960000000000006de0"), 0x0244e1), + (hex!("0100000000333333334444444455000001970000000000001980"), 0x024581), + (hex!("01000000003333333344444444550000019700000000000048f0"), 0x024621), + (hex!("0100000000333333334444444455000001980000000000001990"), 0x0246c1), + (hex!("0100000000333333334444444455000001980000000000006510"), 0x024761), + (hex!("01000000003333333344444444550000019900000000000019a0"), 0x024801), + (hex!("0100000000333333334444444455000001990000000000007570"), 0x0248a1), + (hex!("0100000000333333334444444455000001990000000000007580"), 0x024941), + (hex!("01000000003333333344444444550000019a00000000000019b0"), 0x0249e1), + (hex!("01000000003333333344444444550000019a0000000000004050"), 0x024a81), + (hex!("01000000003333333344444444550000019a0000000000004ba0"), 0x024b21), + (hex!("01000000003333333344444444550000019a0000000000005540"), 0x024bc1), + (hex!("01000000003333333344444444550000019a00000000000061c0"), 0x024c61), + (hex!("01000000003333333344444444550000019a0000000000007c60"), 0x024d01), + (hex!("01000000003333333344444444550000019b00000000000019c0"), 0x024da1), + (hex!("01000000003333333344444444550000019b0000000000006240"), 0x024e41), + (hex!("01000000003333333344444444550000019c00000000000019d0"), 0x024ee1), + (hex!("01000000003333333344444444550000019d00000000000019e0"), 0x024f81), + (hex!("01000000003333333344444444550000019d0000000000004640"), 0x025021), + (hex!("01000000003333333344444444550000019d00000000000052a0"), 0x0250c1), + (hex!("01000000003333333344444444550000019d00000000000052b0"), 0x025161), + (hex!("01000000003333333344444444550000019e00000000000019f0"), 0x025201), + (hex!("01000000003333333344444444550000019f0000000000001a00"), 0x0252a1), + (hex!("01000000003333333344444444550000019f0000000000006b20"), 0x025341), + (hex!("0100000000333333334444444455000001a00000000000001a10"), 0x0253e1), + (hex!("0100000000333333334444444455000001a10000000000001a20"), 0x025481), + (hex!("0100000000333333334444444455000001a10000000000005460"), 0x025521), + (hex!("0100000000333333334444444455000001a10000000000005d20"), 0x0255c1), + (hex!("0100000000333333334444444455000001a100000000000068f0"), 0x025661), + (hex!("0100000000333333334444444455000001a20000000000001a30"), 0x025701), + (hex!("0100000000333333334444444455000001a20000000000007170"), 0x0257a1), + (hex!("0100000000333333334444444455000001a30000000000001a40"), 0x025841), + (hex!("0100000000333333334444444455000001a40000000000001a50"), 0x0258e1), + (hex!("0100000000333333334444444455000001a50000000000001a60"), 0x025981), + (hex!("0100000000333333334444444455000001a60000000000001a70"), 0x025a21), + (hex!("0100000000333333334444444455000001a70000000000001a80"), 0x025ac1), + (hex!("0100000000333333334444444455000001a70000000000005a90"), 0x025b61), + (hex!("0100000000333333334444444455000001a70000000000006440"), 0x025c01), + (hex!("0100000000333333334444444455000001a80000000000001a90"), 0x025ca1), + (hex!("0100000000333333334444444455000001a80000000000004800"), 0x025d41), + (hex!("0100000000333333334444444455000001a90000000000001aa0"), 0x025de1), + (hex!("0100000000333333334444444455000001aa0000000000001ab0"), 0x025e81), + (hex!("0100000000333333334444444455000001aa0000000000005b60"), 0x025f21), + (hex!("0100000000333333334444444455000001ab0000000000001ac0"), 0x025fc1), + (hex!("0100000000333333334444444455000001ab0000000000006700"), 0x026061), + (hex!("0100000000333333334444444455000001ab00000000000071d0"), 0x026101), + (hex!("0100000000333333334444444455000001ac0000000000001ad0"), 0x0261a1), + (hex!("0100000000333333334444444455000001ac0000000000007380"), 0x026241), + (hex!("0100000000333333334444444455000001ad0000000000001ae0"), 0x0262e1), + (hex!("0100000000333333334444444455000001ad0000000000006350"), 0x026381), + (hex!("0100000000333333334444444455000001ae0000000000001af0"), 0x026421), + (hex!("0100000000333333334444444455000001af0000000000001b00"), 0x0264c1), + (hex!("0100000000333333334444444455000001af0000000000007390"), 0x026561), + (hex!("0100000000333333334444444455000001b00000000000001b10"), 0x026601), + (hex!("0100000000333333334444444455000001b10000000000001b20"), 0x0266a1), + (hex!("0100000000333333334444444455000001b10000000000005cc0"), 0x026741), + (hex!("0100000000333333334444444455000001b20000000000001b30"), 0x0267e1), + (hex!("0100000000333333334444444455000001b20000000000004fb0"), 0x026881), + (hex!("0100000000333333334444444455000001b30000000000001b40"), 0x026921), + (hex!("0100000000333333334444444455000001b40000000000001b50"), 0x0269c1), + (hex!("0100000000333333334444444455000001b50000000000001b60"), 0x026a61), + (hex!("0100000000333333334444444455000001b60000000000001b70"), 0x026b01), + (hex!("0100000000333333334444444455000001b600000000000048e0"), 0x026ba1), + (hex!("0100000000333333334444444455000001b70000000000001b80"), 0x026c41), + (hex!("0100000000333333334444444455000001b70000000000005ca0"), 0x026ce1), + (hex!("0100000000333333334444444455000001b70000000000007900"), 0x026d81), + (hex!("0100000000333333334444444455000001b80000000000001b90"), 0x026e21), + (hex!("0100000000333333334444444455000001b80000000000004d90"), 0x026ec1), + (hex!("0100000000333333334444444455000001b90000000000001ba0"), 0x026f61), + (hex!("0100000000333333334444444455000001b90000000000003f40"), 0x027001), + (hex!("0100000000333333334444444455000001ba0000000000001bb0"), 0x0270a1), + (hex!("0100000000333333334444444455000001ba00000000000042a0"), 0x027141), + (hex!("0100000000333333334444444455000001ba00000000000067f0"), 0x0271e1), + (hex!("0100000000333333334444444455000001ba00000000000073a0"), 0x027281), + (hex!("0100000000333333334444444455000001bb0000000000001bc0"), 0x027321), + (hex!("0100000000333333334444444455000001bb0000000000004a00"), 0x0273c1), + (hex!("0100000000333333334444444455000001bb0000000000005e00"), 0x027461), + (hex!("0100000000333333334444444455000001bc0000000000001bd0"), 0x027501), + (hex!("0100000000333333334444444455000001bc0000000000004230"), 0x0275a1), + (hex!("0100000000333333334444444455000001bc0000000000005860"), 0x027641), + (hex!("0100000000333333334444444455000001bd0000000000001be0"), 0x0276e1), + (hex!("0100000000333333334444444455000001bd0000000000007c70"), 0x027781), + (hex!("0100000000333333334444444455000001be0000000000001bf0"), 0x027821), + (hex!("0100000000333333334444444455000001be0000000000007770"), 0x0278c1), + (hex!("0100000000333333334444444455000001be0000000000007cf0"), 0x027961), + (hex!("0100000000333333334444444455000001bf0000000000001c00"), 0x027a01), + (hex!("0100000000333333334444444455000001bf0000000000006490"), 0x027aa1), + (hex!("0100000000333333334444444455000001c00000000000001c10"), 0x027b41), + (hex!("0100000000333333334444444455000001c10000000000001c20"), 0x027be1), + (hex!("0100000000333333334444444455000001c10000000000004600"), 0x027c81), + (hex!("0100000000333333334444444455000001c20000000000001c30"), 0x027d21), + (hex!("0100000000333333334444444455000001c20000000000006e30"), 0x027dc1), + (hex!("0100000000333333334444444455000001c30000000000001c40"), 0x027e61), + (hex!("0100000000333333334444444455000001c40000000000001c50"), 0x027f01), + (hex!("0100000000333333334444444455000001c50000000000001c60"), 0x027fa1), + (hex!("0100000000333333334444444455000001c60000000000001c70"), 0x028041), + (hex!("0100000000333333334444444455000001c60000000000004240"), 0x0280e1), + (hex!("0100000000333333334444444455000001c60000000000005bb0"), 0x028181), + (hex!("0100000000333333334444444455000001c70000000000001c80"), 0x028221), + (hex!("0100000000333333334444444455000001c80000000000001c90"), 0x0282c1), + (hex!("0100000000333333334444444455000001c90000000000001ca0"), 0x028361), + (hex!("0100000000333333334444444455000001c90000000000006730"), 0x028401), + (hex!("0100000000333333334444444455000001ca0000000000001cb0"), 0x0284a1), + (hex!("0100000000333333334444444455000001ca00000000000070f0"), 0x028541), + (hex!("0100000000333333334444444455000001cb0000000000001cc0"), 0x0285e1), + (hex!("0100000000333333334444444455000001cb00000000000071a0"), 0x028681), + (hex!("0100000000333333334444444455000001cc0000000000001cd0"), 0x028721), + (hex!("0100000000333333334444444455000001cc0000000000005280"), 0x0287c1), + (hex!("0100000000333333334444444455000001cc0000000000005d90"), 0x028861), + (hex!("0100000000333333334444444455000001cd0000000000001ce0"), 0x028901), + (hex!("0100000000333333334444444455000001cd00000000000069b0"), 0x0289a1), + (hex!("0100000000333333334444444455000001ce0000000000001cf0"), 0x028a41), + (hex!("0100000000333333334444444455000001ce0000000000004540"), 0x028ae1), + (hex!("0100000000333333334444444455000001cf0000000000001d00"), 0x028b81), + (hex!("0100000000333333334444444455000001cf00000000000076a0"), 0x028c21), + (hex!("0100000000333333334444444455000001d00000000000001d10"), 0x028cc1), + (hex!("0100000000333333334444444455000001d000000000000060a0"), 0x028d61), + (hex!("0100000000333333334444444455000001d10000000000001d20"), 0x028e01), + (hex!("0100000000333333334444444455000001d20000000000001d30"), 0x028ea1), + (hex!("0100000000333333334444444455000001d30000000000001d40"), 0x028f41), + (hex!("0100000000333333334444444455000001d30000000000004000"), 0x028fe1), + (hex!("0100000000333333334444444455000001d30000000000004140"), 0x029081), + (hex!("0100000000333333334444444455000001d30000000000006790"), 0x029121), + (hex!("0100000000333333334444444455000001d40000000000001d50"), 0x0291c1), + (hex!("0100000000333333334444444455000001d50000000000001d60"), 0x029261), + (hex!("0100000000333333334444444455000001d60000000000001d70"), 0x029301), + (hex!("0100000000333333334444444455000001d60000000000004b50"), 0x0293a1), + (hex!("0100000000333333334444444455000001d60000000000007430"), 0x029441), + (hex!("0100000000333333334444444455000001d70000000000001d80"), 0x0294e1), + (hex!("0100000000333333334444444455000001d70000000000006920"), 0x029581), + (hex!("0100000000333333334444444455000001d80000000000001d90"), 0x029621), + (hex!("0100000000333333334444444455000001d80000000000005b30"), 0x0296c1), + (hex!("0100000000333333334444444455000001d90000000000001da0"), 0x029761), + (hex!("0100000000333333334444444455000001da0000000000001db0"), 0x029801), + (hex!("0100000000333333334444444455000001da0000000000004af0"), 0x0298a1), + (hex!("0100000000333333334444444455000001da0000000000007240"), 0x029941), + (hex!("0100000000333333334444444455000001da0000000000007470"), 0x0299e1), + (hex!("0100000000333333334444444455000001db0000000000001dc0"), 0x029a81), + (hex!("0100000000333333334444444455000001db00000000000045d0"), 0x029b21), + (hex!("0100000000333333334444444455000001dc0000000000001dd0"), 0x029bc1), + (hex!("0100000000333333334444444455000001dd0000000000001de0"), 0x029c61), + (hex!("0100000000333333334444444455000001dd0000000000004bb0"), 0x029d01), + (hex!("0100000000333333334444444455000001dd0000000000004cd0"), 0x029da1), + (hex!("0100000000333333334444444455000001dd0000000000006100"), 0x029e41), + (hex!("0100000000333333334444444455000001dd0000000000007bb0"), 0x029ee1), + (hex!("0100000000333333334444444455000001de0000000000001df0"), 0x029f81), + (hex!("0100000000333333334444444455000001de0000000000004260"), 0x02a021), + (hex!("0100000000333333334444444455000001de0000000000006040"), 0x02a0c1), + (hex!("0100000000333333334444444455000001df0000000000001e00"), 0x02a161), + (hex!("0100000000333333334444444455000001df0000000000005fa0"), 0x02a201), + (hex!("0100000000333333334444444455000001df0000000000006a70"), 0x02a2a1), + (hex!("0100000000333333334444444455000001df0000000000006dc0"), 0x02a341), + (hex!("0100000000333333334444444455000001e00000000000001e10"), 0x02a3e1), + (hex!("0100000000333333334444444455000001e00000000000007010"), 0x02a481), + (hex!("0100000000333333334444444455000001e10000000000001e20"), 0x02a521), + (hex!("0100000000333333334444444455000001e10000000000005720"), 0x02a5c1), + (hex!("0100000000333333334444444455000001e10000000000006830"), 0x02a661), + (hex!("0100000000333333334444444455000001e20000000000001e30"), 0x02a701), + (hex!("0100000000333333334444444455000001e20000000000005100"), 0x02a7a1), + (hex!("0100000000333333334444444455000001e30000000000001e40"), 0x02a841), + (hex!("0100000000333333334444444455000001e40000000000001e50"), 0x02a8e1), + (hex!("0100000000333333334444444455000001e40000000000003f30"), 0x02a981), + (hex!("0100000000333333334444444455000001e40000000000005220"), 0x02aa21), + (hex!("0100000000333333334444444455000001e50000000000001e60"), 0x02aac1), + (hex!("0100000000333333334444444455000001e50000000000006f60"), 0x02ab61), + (hex!("0100000000333333334444444455000001e60000000000001e70"), 0x02ac01), + (hex!("0100000000333333334444444455000001e60000000000006c80"), 0x02aca1), + (hex!("0100000000333333334444444455000001e70000000000001e80"), 0x02ad41), + (hex!("0100000000333333334444444455000001e80000000000001e90"), 0x02ade1), + (hex!("0100000000333333334444444455000001e80000000000004e30"), 0x02ae81), + (hex!("0100000000333333334444444455000001e90000000000001ea0"), 0x02af21), + (hex!("0100000000333333334444444455000001e90000000000005470"), 0x02afc1), + (hex!("0100000000333333334444444455000001ea0000000000001eb0"), 0x02b061), + (hex!("0100000000333333334444444455000001ea0000000000007980"), 0x02b101), + (hex!("0100000000333333334444444455000001eb0000000000001ec0"), 0x02b1a1), + (hex!("0100000000333333334444444455000001eb0000000000004390"), 0x02b241), + (hex!("0100000000333333334444444455000001eb0000000000005970"), 0x02b2e1), + (hex!("0100000000333333334444444455000001ec0000000000001ed0"), 0x02b381), + (hex!("0100000000333333334444444455000001ec0000000000005d50"), 0x02b421), + (hex!("0100000000333333334444444455000001ec00000000000076e0"), 0x02b4c1), + (hex!("0100000000333333334444444455000001ed0000000000001ee0"), 0x02b561), + (hex!("0100000000333333334444444455000001ed0000000000006190"), 0x02b601), + (hex!("0100000000333333334444444455000001ee0000000000001ef0"), 0x02b6a1), + (hex!("0100000000333333334444444455000001ee0000000000004900"), 0x02b741), + (hex!("0100000000333333334444444455000001ef0000000000001f00"), 0x02b7e1), + (hex!("0100000000333333334444444455000001ef0000000000006c60"), 0x02b881), + (hex!("0100000000333333334444444455000001f00000000000001f10"), 0x02b921), + (hex!("0100000000333333334444444455000001f00000000000006950"), 0x02b9c1), + (hex!("0100000000333333334444444455000001f10000000000001f20"), 0x02ba61), + (hex!("0100000000333333334444444455000001f10000000000006400"), 0x02bb01), + (hex!("0100000000333333334444444455000001f20000000000001f30"), 0x02bba1), + (hex!("0100000000333333334444444455000001f20000000000006f00"), 0x02bc41), + (hex!("0100000000333333334444444455000001f20000000000007b10"), 0x02bce1), + (hex!("0100000000333333334444444455000001f30000000000001f40"), 0x02bd81), + (hex!("0100000000333333334444444455000001f40000000000001f50"), 0x02be21), + (hex!("0100000000333333334444444455000001f50000000000001f60"), 0x02bec1), + (hex!("0100000000333333334444444455000001f500000000000044f0"), 0x02bf61), + (hex!("0100000000333333334444444455000001f60000000000001f70"), 0x02c001), + (hex!("0100000000333333334444444455000001f70000000000001f80"), 0x02c0a1), + (hex!("0100000000333333334444444455000001f70000000000004ad0"), 0x02c141), + (hex!("0100000000333333334444444455000001f80000000000001f90"), 0x02c1e1), + (hex!("0100000000333333334444444455000001f90000000000001fa0"), 0x02c281), + (hex!("0100000000333333334444444455000001f90000000000003f60"), 0x02c321), + (hex!("0100000000333333334444444455000001f90000000000004a80"), 0x02c3c1), + (hex!("0100000000333333334444444455000001fa0000000000001fb0"), 0x02c461), + (hex!("0100000000333333334444444455000001fa0000000000006f90"), 0x02c501), + (hex!("0100000000333333334444444455000001fb0000000000001fc0"), 0x02c5a1), + (hex!("0100000000333333334444444455000001fc0000000000001fd0"), 0x02c641), + (hex!("0100000000333333334444444455000001fc0000000000004a90"), 0x02c6e1), + (hex!("0100000000333333334444444455000001fd0000000000001fe0"), 0x02c781), + (hex!("0100000000333333334444444455000001fd0000000000005f50"), 0x02c821), + (hex!("0100000000333333334444444455000001fe0000000000001ff0"), 0x02c8c1), + (hex!("0100000000333333334444444455000001ff0000000000002000"), 0x02c961), + (hex!("0100000000333333334444444455000002000000000000002010"), 0x02ca01), + (hex!("0100000000333333334444444455000002000000000000005f00"), 0x02caa1), + (hex!("0100000000333333334444444455000002000000000000006840"), 0x02cb41), + (hex!("0100000000333333334444444455000002010000000000002020"), 0x02cbe1), + (hex!("0100000000333333334444444455000002020000000000002030"), 0x02cc81), + (hex!("0100000000333333334444444455000002030000000000002040"), 0x02cd21), + (hex!("0100000000333333334444444455000002040000000000002050"), 0x02cdc1), + (hex!("01000000003333333344444444550000020400000000000051f0"), 0x02ce61), + (hex!("0100000000333333334444444455000002050000000000002060"), 0x02cf01), + (hex!("0100000000333333334444444455000002060000000000002070"), 0x02cfa1), + (hex!("0100000000333333334444444455000002060000000000005c80"), 0x02d041), + (hex!("01000000003333333344444444550000020600000000000061d0"), 0x02d0e1), + (hex!("01000000003333333344444444550000020600000000000078c0"), 0x02d181), + (hex!("0100000000333333334444444455000002070000000000002080"), 0x02d221), + (hex!("0100000000333333334444444455000002070000000000006ba0"), 0x02d2c1), + (hex!("0100000000333333334444444455000002080000000000002090"), 0x02d361), + (hex!("01000000003333333344444444550000020900000000000020a0"), 0x02d401), + (hex!("01000000003333333344444444550000020900000000000067a0"), 0x02d4a1), + (hex!("01000000003333333344444444550000020a00000000000020b0"), 0x02d541), + (hex!("01000000003333333344444444550000020a0000000000004950"), 0x02d5e1), + (hex!("01000000003333333344444444550000020a0000000000004de0"), 0x02d681), + (hex!("01000000003333333344444444550000020b00000000000020c0"), 0x02d721), + (hex!("01000000003333333344444444550000020b0000000000004b00"), 0x02d7c1), + (hex!("01000000003333333344444444550000020c00000000000020d0"), 0x02d861), + (hex!("01000000003333333344444444550000020d00000000000020e0"), 0x02d901), + (hex!("01000000003333333344444444550000020e00000000000020f0"), 0x02d9a1), + (hex!("01000000003333333344444444550000020f0000000000002100"), 0x02da41), + (hex!("0100000000333333334444444455000002100000000000002110"), 0x02dae1), + (hex!("0100000000333333334444444455000002110000000000002120"), 0x02db81), + (hex!("0100000000333333334444444455000002110000000000004490"), 0x02dc21), + (hex!("0100000000333333334444444455000002120000000000002130"), 0x02dcc1), + (hex!("0100000000333333334444444455000002130000000000002140"), 0x02dd61), + (hex!("01000000003333333344444444550000021300000000000046d0"), 0x02de01), + (hex!("01000000003333333344444444550000021300000000000046e0"), 0x02dea1), + (hex!("0100000000333333334444444455000002130000000000004b70"), 0x02df41), + (hex!("0100000000333333334444444455000002140000000000002150"), 0x02dfe1), + (hex!("0100000000333333334444444455000002140000000000006c50"), 0x02e081), + (hex!("0100000000333333334444444455000002150000000000002160"), 0x02e121), + (hex!("01000000003333333344444444550000021500000000000043c0"), 0x02e1c1), + (hex!("0100000000333333334444444455000002160000000000002170"), 0x02e261), + (hex!("01000000003333333344444444550000021600000000000055b0"), 0x02e301), + (hex!("0100000000333333334444444455000002160000000000006150"), 0x02e3a1), + (hex!("0100000000333333334444444455000002170000000000002180"), 0x02e441), + (hex!("01000000003333333344444444550000021700000000000053b0"), 0x02e4e1), + (hex!("0100000000333333334444444455000002170000000000007460"), 0x02e581), + (hex!("0100000000333333334444444455000002180000000000002190"), 0x02e621), + (hex!("01000000003333333344444444550000021900000000000021a0"), 0x02e6c1), + (hex!("01000000003333333344444444550000021a00000000000021b0"), 0x02e761), + (hex!("01000000003333333344444444550000021a0000000000007650"), 0x02e801), + (hex!("01000000003333333344444444550000021b00000000000021c0"), 0x02e8a1), + (hex!("01000000003333333344444444550000021b0000000000004b20"), 0x02e941), + (hex!("01000000003333333344444444550000021c00000000000021d0"), 0x02e9e1), + (hex!("01000000003333333344444444550000021c0000000000007610"), 0x02ea81), + (hex!("01000000003333333344444444550000021d00000000000021e0"), 0x02eb21), + (hex!("01000000003333333344444444550000021d0000000000005f40"), 0x02ebc1), + (hex!("01000000003333333344444444550000021e00000000000021f0"), 0x02ec61), + (hex!("01000000003333333344444444550000021e0000000000005a50"), 0x02ed01), + (hex!("01000000003333333344444444550000021e0000000000005ff0"), 0x02eda1), + (hex!("01000000003333333344444444550000021f0000000000002200"), 0x02ee41), + (hex!("01000000003333333344444444550000021f00000000000043a0"), 0x02eee1), + (hex!("01000000003333333344444444550000021f0000000000004cb0"), 0x02ef81), + (hex!("01000000003333333344444444550000021f0000000000004e00"), 0x02f021), + (hex!("0100000000333333334444444455000002200000000000002210"), 0x02f0c1), + (hex!("0100000000333333334444444455000002210000000000002220"), 0x02f161), + (hex!("0100000000333333334444444455000002210000000000006290"), 0x02f201), + (hex!("0100000000333333334444444455000002210000000000007230"), 0x02f2a1), + (hex!("0100000000333333334444444455000002220000000000002230"), 0x02f341), + (hex!("0100000000333333334444444455000002220000000000006ea0"), 0x02f3e1), + (hex!("0100000000333333334444444455000002230000000000002240"), 0x02f481), + (hex!("0100000000333333334444444455000002230000000000004710"), 0x02f521), + (hex!("0100000000333333334444444455000002240000000000002250"), 0x02f5c1), + (hex!("0100000000333333334444444455000002250000000000002260"), 0x02f661), + (hex!("0100000000333333334444444455000002260000000000002270"), 0x02f701), + (hex!("0100000000333333334444444455000002260000000000005b40"), 0x02f7a1), + (hex!("0100000000333333334444444455000002260000000000006300"), 0x02f841), + (hex!("0100000000333333334444444455000002270000000000002280"), 0x02f8e1), + (hex!("0100000000333333334444444455000002270000000000005b80"), 0x02f981), + (hex!("0100000000333333334444444455000002280000000000002290"), 0x02fa21), + (hex!("0100000000333333334444444455000002280000000000003ed0"), 0x02fac1), + (hex!("0100000000333333334444444455000002280000000000004550"), 0x02fb61), + (hex!("01000000003333333344444444550000022800000000000077d0"), 0x02fc01), + (hex!("01000000003333333344444444550000022900000000000022a0"), 0x02fca1), + (hex!("0100000000333333334444444455000002290000000000006480"), 0x02fd41), + (hex!("01000000003333333344444444550000022a00000000000022b0"), 0x02fde1), + (hex!("01000000003333333344444444550000022a0000000000005450"), 0x02fe81), + (hex!("01000000003333333344444444550000022b00000000000022c0"), 0x02ff21), + (hex!("01000000003333333344444444550000022b0000000000006dd0"), 0x02ffc1), + (hex!("01000000003333333344444444550000022c00000000000022d0"), 0x030061), + (hex!("01000000003333333344444444550000022c0000000000006890"), 0x030101), + (hex!("01000000003333333344444444550000022d00000000000022e0"), 0x0301a1), + (hex!("01000000003333333344444444550000022e00000000000022f0"), 0x030241), + (hex!("01000000003333333344444444550000022e0000000000004f20"), 0x0302e1), + (hex!("01000000003333333344444444550000022f0000000000002300"), 0x030381), + (hex!("01000000003333333344444444550000022f0000000000005260"), 0x030421), + (hex!("01000000003333333344444444550000022f00000000000053f0"), 0x0304c1), + (hex!("0100000000333333334444444455000002300000000000002310"), 0x030561), + (hex!("01000000003333333344444444550000023000000000000050e0"), 0x030601), + (hex!("0100000000333333334444444455000002310000000000002320"), 0x0306a1), + (hex!("0100000000333333334444444455000002310000000000007800"), 0x030741), + (hex!("0100000000333333334444444455000002320000000000002330"), 0x0307e1), + (hex!("0100000000333333334444444455000002330000000000002340"), 0x030881), + (hex!("0100000000333333334444444455000002330000000000004d70"), 0x030921), + (hex!("0100000000333333334444444455000002330000000000005cf0"), 0x0309c1), + (hex!("0100000000333333334444444455000002340000000000002350"), 0x030a61), + (hex!("0100000000333333334444444455000002350000000000002360"), 0x030b01), + (hex!("0100000000333333334444444455000002350000000000006970"), 0x030ba1), + (hex!("0100000000333333334444444455000002360000000000002370"), 0x030c41), + (hex!("0100000000333333334444444455000002360000000000005270"), 0x030ce1), + (hex!("0100000000333333334444444455000002370000000000002380"), 0x030d81), + (hex!("0100000000333333334444444455000002370000000000005d70"), 0x030e21), + (hex!("0100000000333333334444444455000002380000000000002390"), 0x030ec1), + (hex!("01000000003333333344444444550000023800000000000069a0"), 0x030f61), + (hex!("01000000003333333344444444550000023900000000000023a0"), 0x031001), + (hex!("01000000003333333344444444550000023900000000000052e0"), 0x0310a1), + (hex!("0100000000333333334444444455000002390000000000005a10"), 0x031141), + (hex!("0100000000333333334444444455000002390000000000007440"), 0x0311e1), + (hex!("01000000003333333344444444550000023a00000000000023b0"), 0x031281), + (hex!("01000000003333333344444444550000023a0000000000003f00"), 0x031321), + (hex!("01000000003333333344444444550000023a0000000000004430"), 0x0313c1), + (hex!("01000000003333333344444444550000023a0000000000007070"), 0x031461), + (hex!("01000000003333333344444444550000023a00000000000074a0"), 0x031501), + (hex!("01000000003333333344444444550000023b00000000000023c0"), 0x0315a1), + (hex!("01000000003333333344444444550000023b0000000000004730"), 0x031641), + (hex!("01000000003333333344444444550000023b00000000000068b0"), 0x0316e1), + (hex!("01000000003333333344444444550000023c00000000000023d0"), 0x031781), + (hex!("01000000003333333344444444550000023c0000000000004680"), 0x031821), + (hex!("01000000003333333344444444550000023d00000000000023e0"), 0x0318c1), + (hex!("01000000003333333344444444550000023d00000000000059a0"), 0x031961), + (hex!("01000000003333333344444444550000023e00000000000023f0"), 0x031a01), + (hex!("01000000003333333344444444550000023f0000000000002400"), 0x031aa1), + (hex!("0100000000333333334444444455000002400000000000002410"), 0x031b41), + (hex!("0100000000333333334444444455000002400000000000004920"), 0x031be1), + (hex!("01000000003333333344444444550000024000000000000066e0"), 0x031c81), + (hex!("01000000003333333344444444550000024000000000000076f0"), 0x031d21), + (hex!("01000000003333333344444444550000024000000000000078e0"), 0x031dc1), + (hex!("0100000000333333334444444455000002410000000000002420"), 0x031e61), + (hex!("0100000000333333334444444455000002420000000000002430"), 0x031f01), + (hex!("0100000000333333334444444455000002420000000000006590"), 0x031fa1), + (hex!("0100000000333333334444444455000002430000000000002440"), 0x032041), + (hex!("0100000000333333334444444455000002430000000000004d00"), 0x0320e1), + (hex!("0100000000333333334444444455000002440000000000002450"), 0x032181), + (hex!("0100000000333333334444444455000002440000000000005f80"), 0x032221), + (hex!("0100000000333333334444444455000002450000000000002460"), 0x0322c1), + (hex!("0100000000333333334444444455000002450000000000004940"), 0x032361), + (hex!("0100000000333333334444444455000002460000000000002470"), 0x032401), + (hex!("0100000000333333334444444455000002470000000000002480"), 0x0324a1), + (hex!("0100000000333333334444444455000002470000000000004dd0"), 0x032541), + (hex!("0100000000333333334444444455000002470000000000005930"), 0x0325e1), + (hex!("01000000003333333344444444550000024700000000000061b0"), 0x032681), + (hex!("0100000000333333334444444455000002470000000000007740"), 0x032721), + (hex!("0100000000333333334444444455000002480000000000002490"), 0x0327c1), + (hex!("0100000000333333334444444455000002480000000000004890"), 0x032861), + (hex!("01000000003333333344444444550000024900000000000024a0"), 0x032901), + (hex!("01000000003333333344444444550000024a00000000000024b0"), 0x0329a1), + (hex!("01000000003333333344444444550000024b00000000000024c0"), 0x032a41), + (hex!("01000000003333333344444444550000024c00000000000024d0"), 0x032ae1), + (hex!("01000000003333333344444444550000024d00000000000024e0"), 0x032b81), + (hex!("01000000003333333344444444550000024d0000000000004070"), 0x032c21), + (hex!("01000000003333333344444444550000024e00000000000024f0"), 0x032cc1), + (hex!("01000000003333333344444444550000024e00000000000066a0"), 0x032d61), + (hex!("01000000003333333344444444550000024e0000000000006ab0"), 0x032e01), + (hex!("01000000003333333344444444550000024f0000000000002500"), 0x032ea1), + (hex!("0100000000333333334444444455000002500000000000002510"), 0x032f41), + (hex!("0100000000333333334444444455000002510000000000002520"), 0x032fe1), + (hex!("0100000000333333334444444455000002510000000000007320"), 0x033081), + (hex!("0100000000333333334444444455000002520000000000002530"), 0x033121), + (hex!("0100000000333333334444444455000002520000000000006410"), 0x0331c1), + (hex!("0100000000333333334444444455000002530000000000002540"), 0x033261), + (hex!("0100000000333333334444444455000002530000000000005110"), 0x033301), + (hex!("0100000000333333334444444455000002540000000000002550"), 0x0333a1), + (hex!("01000000003333333344444444550000025400000000000040c0"), 0x033441), + (hex!("0100000000333333334444444455000002540000000000006a40"), 0x0334e1), + (hex!("0100000000333333334444444455000002550000000000002560"), 0x033581), + (hex!("0100000000333333334444444455000002550000000000005190"), 0x033621), + (hex!("0100000000333333334444444455000002560000000000002570"), 0x0336c1), + (hex!("01000000003333333344444444550000025600000000000061f0"), 0x033761), + (hex!("0100000000333333334444444455000002570000000000002580"), 0x033801), + (hex!("0100000000333333334444444455000002580000000000002590"), 0x0338a1), + (hex!("01000000003333333344444444550000025800000000000043d0"), 0x033941), + (hex!("01000000003333333344444444550000025900000000000025a0"), 0x0339e1), + (hex!("0100000000333333334444444455000002590000000000006bb0"), 0x033a81), + (hex!("01000000003333333344444444550000025a00000000000025b0"), 0x033b21), + (hex!("01000000003333333344444444550000025a0000000000005fb0"), 0x033bc1), + (hex!("01000000003333333344444444550000025a00000000000064c0"), 0x033c61), + (hex!("01000000003333333344444444550000025b00000000000025c0"), 0x033d01), + (hex!("01000000003333333344444444550000025b0000000000005c10"), 0x033da1), + (hex!("01000000003333333344444444550000025c00000000000025d0"), 0x033e41), + (hex!("01000000003333333344444444550000025c0000000000007d00"), 0x033ee1), + (hex!("01000000003333333344444444550000025d00000000000025e0"), 0x033f81), + (hex!("01000000003333333344444444550000025e00000000000025f0"), 0x034021), + (hex!("01000000003333333344444444550000025e00000000000045e0"), 0x0340c1), + (hex!("01000000003333333344444444550000025e0000000000006ee0"), 0x034161), + (hex!("01000000003333333344444444550000025f0000000000002600"), 0x034201), + (hex!("01000000003333333344444444550000025f00000000000050b0"), 0x0342a1), + (hex!("01000000003333333344444444550000025f0000000000007690"), 0x034341), + (hex!("0100000000333333334444444455000002600000000000002610"), 0x0343e1), + (hex!("0100000000333333334444444455000002600000000000007b60"), 0x034481), + (hex!("0100000000333333334444444455000002610000000000002620"), 0x034521), + (hex!("0100000000333333334444444455000002620000000000002630"), 0x0345c1), + (hex!("0100000000333333334444444455000002630000000000002640"), 0x034661), + (hex!("0100000000333333334444444455000002640000000000002650"), 0x034701), + (hex!("0100000000333333334444444455000002650000000000002660"), 0x0347a1), + (hex!("0100000000333333334444444455000002650000000000006180"), 0x034841), + (hex!("0100000000333333334444444455000002660000000000002670"), 0x0348e1), + (hex!("0100000000333333334444444455000002660000000000005430"), 0x034981), + (hex!("0100000000333333334444444455000002660000000000007a60"), 0x034a21), + (hex!("0100000000333333334444444455000002670000000000002680"), 0x034ac1), + (hex!("01000000003333333344444444550000026700000000000077f0"), 0x034b61), + (hex!("0100000000333333334444444455000002680000000000002690"), 0x034c01), + (hex!("01000000003333333344444444550000026900000000000026a0"), 0x034ca1), + (hex!("01000000003333333344444444550000026a00000000000026b0"), 0x034d41), + (hex!("01000000003333333344444444550000026a0000000000007530"), 0x034de1), + (hex!("01000000003333333344444444550000026b00000000000026c0"), 0x034e81), + (hex!("01000000003333333344444444550000026b00000000000058b0"), 0x034f21), + (hex!("01000000003333333344444444550000026b00000000000066b0"), 0x034fc1), + (hex!("01000000003333333344444444550000026b0000000000006b10"), 0x035061), + (hex!("01000000003333333344444444550000026c00000000000026d0"), 0x035101), + (hex!("01000000003333333344444444550000026d00000000000026e0"), 0x0351a1), + (hex!("01000000003333333344444444550000026d0000000000004210"), 0x035241), + (hex!("01000000003333333344444444550000026d0000000000005490"), 0x0352e1), + (hex!("01000000003333333344444444550000026d0000000000005e60"), 0x035381), + (hex!("01000000003333333344444444550000026d00000000000068e0"), 0x035421), + (hex!("01000000003333333344444444550000026d0000000000007020"), 0x0354c1), + (hex!("01000000003333333344444444550000026d0000000000007300"), 0x035561), + (hex!("01000000003333333344444444550000026e00000000000026f0"), 0x035601), + (hex!("01000000003333333344444444550000026f0000000000002700"), 0x0356a1), + (hex!("01000000003333333344444444550000026f0000000000004910"), 0x035741), + (hex!("0100000000333333334444444455000002700000000000002710"), 0x0357e1), + (hex!("0100000000333333334444444455000002710000000000002720"), 0x035881), + (hex!("01000000003333333344444444550000027100000000000050c0"), 0x035921), + (hex!("0100000000333333334444444455000002720000000000002730"), 0x0359c1), + (hex!("0100000000333333334444444455000002730000000000002740"), 0x035a61), + (hex!("0100000000333333334444444455000002740000000000002750"), 0x035b01), + (hex!("0100000000333333334444444455000002740000000000007490"), 0x035ba1), + (hex!("0100000000333333334444444455000002750000000000002760"), 0x035c41), + (hex!("0100000000333333334444444455000002760000000000002770"), 0x035ce1), + (hex!("0100000000333333334444444455000002760000000000004790"), 0x035d81), + (hex!("0100000000333333334444444455000002770000000000002780"), 0x035e21), + (hex!("01000000003333333344444444550000027700000000000050a0"), 0x035ec1), + (hex!("0100000000333333334444444455000002780000000000002790"), 0x035f61), + (hex!("0100000000333333334444444455000002780000000000004330"), 0x036001), + (hex!("0100000000333333334444444455000002780000000000006b00"), 0x0360a1), + (hex!("01000000003333333344444444550000027900000000000027a0"), 0x036141), + (hex!("01000000003333333344444444550000027a00000000000027b0"), 0x0361e1), + (hex!("01000000003333333344444444550000027b00000000000027c0"), 0x036281), + (hex!("01000000003333333344444444550000027b0000000000004930"), 0x036321), + (hex!("01000000003333333344444444550000027b0000000000006250"), 0x0363c1), + (hex!("01000000003333333344444444550000027c00000000000027d0"), 0x036461), + (hex!("01000000003333333344444444550000027d00000000000027e0"), 0x036501), + (hex!("01000000003333333344444444550000027d0000000000005ce0"), 0x0365a1), + (hex!("01000000003333333344444444550000027d0000000000005fe0"), 0x036641), + (hex!("01000000003333333344444444550000027e00000000000027f0"), 0x0366e1), + (hex!("01000000003333333344444444550000027f0000000000002800"), 0x036781), + (hex!("01000000003333333344444444550000027f0000000000003e90"), 0x036821), + (hex!("01000000003333333344444444550000027f0000000000007910"), 0x0368c1), + (hex!("0100000000333333334444444455000002800000000000002810"), 0x036961), + (hex!("0100000000333333334444444455000002800000000000004990"), 0x036a01), + (hex!("0100000000333333334444444455000002800000000000006160"), 0x036aa1), + (hex!("0100000000333333334444444455000002800000000000006740"), 0x036b41), + (hex!("0100000000333333334444444455000002810000000000002820"), 0x036be1), + (hex!("0100000000333333334444444455000002820000000000002830"), 0x036c81), + (hex!("0100000000333333334444444455000002820000000000005170"), 0x036d21), + (hex!("0100000000333333334444444455000002830000000000002840"), 0x036dc1), + (hex!("0100000000333333334444444455000002840000000000002850"), 0x036e61), + (hex!("0100000000333333334444444455000002840000000000004810"), 0x036f01), + (hex!("0100000000333333334444444455000002840000000000006aa0"), 0x036fa1), + (hex!("0100000000333333334444444455000002850000000000002860"), 0x037041), + (hex!("0100000000333333334444444455000002860000000000002870"), 0x0370e1), + (hex!("0100000000333333334444444455000002860000000000005080"), 0x037181), + (hex!("0100000000333333334444444455000002870000000000002880"), 0x037221), + (hex!("0100000000333333334444444455000002870000000000004e60"), 0x0372c1), + (hex!("0100000000333333334444444455000002880000000000002890"), 0x037361), + (hex!("0100000000333333334444444455000002880000000000005060"), 0x037401), + (hex!("0100000000333333334444444455000002880000000000006f20"), 0x0374a1), + (hex!("01000000003333333344444444550000028900000000000028a0"), 0x037541), + (hex!("01000000003333333344444444550000028900000000000047e0"), 0x0375e1), + (hex!("01000000003333333344444444550000028a00000000000028b0"), 0x037681), + (hex!("01000000003333333344444444550000028a0000000000005ab0"), 0x037721), + (hex!("01000000003333333344444444550000028a0000000000007130"), 0x0377c1), + (hex!("01000000003333333344444444550000028a0000000000007660"), 0x037861), + (hex!("01000000003333333344444444550000028b00000000000028c0"), 0x037901), + (hex!("01000000003333333344444444550000028b00000000000054e0"), 0x0379a1), + (hex!("01000000003333333344444444550000028c00000000000028d0"), 0x037a41), + (hex!("01000000003333333344444444550000028c00000000000046f0"), 0x037ae1), + (hex!("01000000003333333344444444550000028c00000000000061a0"), 0x037b81), + (hex!("01000000003333333344444444550000028d00000000000028e0"), 0x037c21), + (hex!("01000000003333333344444444550000028e00000000000028f0"), 0x037cc1), + (hex!("01000000003333333344444444550000028e0000000000004130"), 0x037d61), + (hex!("01000000003333333344444444550000028f0000000000002900"), 0x037e01), + (hex!("01000000003333333344444444550000028f0000000000007510"), 0x037ea1), + (hex!("0100000000333333334444444455000002900000000000002910"), 0x037f41), + (hex!("0100000000333333334444444455000002900000000000004a40"), 0x037fe1), + (hex!("0100000000333333334444444455000002910000000000002920"), 0x038081), + (hex!("0100000000333333334444444455000002920000000000002930"), 0x038121), + (hex!("0100000000333333334444444455000002920000000000004e90"), 0x0381c1), + (hex!("0100000000333333334444444455000002930000000000002940"), 0x038261), + (hex!("0100000000333333334444444455000002930000000000006880"), 0x038301), + (hex!("0100000000333333334444444455000002940000000000002950"), 0x0383a1), + (hex!("0100000000333333334444444455000002940000000000007bc0"), 0x038441), + (hex!("0100000000333333334444444455000002950000000000002960"), 0x0384e1), + (hex!("0100000000333333334444444455000002960000000000002970"), 0x038581), + (hex!("01000000003333333344444444550000029600000000000059d0"), 0x038621), + (hex!("0100000000333333334444444455000002970000000000002980"), 0x0386c1), + (hex!("0100000000333333334444444455000002970000000000004a50"), 0x038761), + (hex!("0100000000333333334444444455000002970000000000005f20"), 0x038801), + (hex!("01000000003333333344444444550000029700000000000068d0"), 0x0388a1), + (hex!("0100000000333333334444444455000002980000000000002990"), 0x038941), + (hex!("0100000000333333334444444455000002980000000000004370"), 0x0389e1), + (hex!("0100000000333333334444444455000002980000000000004420"), 0x038a81), + (hex!("01000000003333333344444444550000029900000000000029a0"), 0x038b21), + (hex!("01000000003333333344444444550000029a00000000000029b0"), 0x038bc1), + (hex!("01000000003333333344444444550000029a0000000000006010"), 0x038c61), + (hex!("01000000003333333344444444550000029a0000000000006980"), 0x038d01), + (hex!("01000000003333333344444444550000029b00000000000029c0"), 0x038da1), + (hex!("01000000003333333344444444550000029c00000000000029d0"), 0x038e41), + (hex!("01000000003333333344444444550000029c0000000000007480"), 0x038ee1), + (hex!("01000000003333333344444444550000029d00000000000029e0"), 0x038f81), + (hex!("01000000003333333344444444550000029d0000000000005030"), 0x039021), + (hex!("01000000003333333344444444550000029d0000000000007780"), 0x0390c1), + (hex!("01000000003333333344444444550000029d0000000000007a50"), 0x039161), + (hex!("01000000003333333344444444550000029e00000000000029f0"), 0x039201), + (hex!("01000000003333333344444444550000029e00000000000074b0"), 0x0392a1), + (hex!("01000000003333333344444444550000029f0000000000002a00"), 0x039341), + (hex!("0100000000333333334444444455000002a00000000000002a10"), 0x0393e1), + (hex!("0100000000333333334444444455000002a10000000000002a20"), 0x039481), + (hex!("0100000000333333334444444455000002a20000000000002a30"), 0x039521), + (hex!("0100000000333333334444444455000002a20000000000004c50"), 0x0395c1), + (hex!("0100000000333333334444444455000002a20000000000006f10"), 0x039661), + (hex!("0100000000333333334444444455000002a30000000000002a40"), 0x039701), + (hex!("0100000000333333334444444455000002a40000000000002a50"), 0x0397a1), + (hex!("0100000000333333334444444455000002a40000000000005d60"), 0x039841), + (hex!("0100000000333333334444444455000002a50000000000002a60"), 0x0398e1), + (hex!("0100000000333333334444444455000002a50000000000005440"), 0x039981), + (hex!("0100000000333333334444444455000002a50000000000005890"), 0x039a21), + (hex!("0100000000333333334444444455000002a60000000000002a70"), 0x039ac1), + (hex!("0100000000333333334444444455000002a70000000000002a80"), 0x039b61), + (hex!("0100000000333333334444444455000002a700000000000054a0"), 0x039c01), + (hex!("0100000000333333334444444455000002a70000000000007280"), 0x039ca1), + (hex!("0100000000333333334444444455000002a80000000000002a90"), 0x039d41), + (hex!("0100000000333333334444444455000002a90000000000002aa0"), 0x039de1), + (hex!("0100000000333333334444444455000002aa0000000000002ab0"), 0x039e81), + (hex!("0100000000333333334444444455000002ab0000000000002ac0"), 0x039f21), + (hex!("0100000000333333334444444455000002ab0000000000006c90"), 0x039fc1), + (hex!("0100000000333333334444444455000002ac0000000000002ad0"), 0x03a061), + (hex!("0100000000333333334444444455000002ac0000000000006db0"), 0x03a101), + (hex!("0100000000333333334444444455000002ad0000000000002ae0"), 0x03a1a1), + (hex!("0100000000333333334444444455000002ad00000000000065e0"), 0x03a241), + (hex!("0100000000333333334444444455000002ad0000000000007b40"), 0x03a2e1), + (hex!("0100000000333333334444444455000002ae0000000000002af0"), 0x03a381), + (hex!("0100000000333333334444444455000002ae0000000000004d20"), 0x03a421), + (hex!("0100000000333333334444444455000002ae0000000000006f30"), 0x03a4c1), + (hex!("0100000000333333334444444455000002af0000000000002b00"), 0x03a561), + (hex!("0100000000333333334444444455000002b00000000000002b10"), 0x03a601), + (hex!("0100000000333333334444444455000002b00000000000004560"), 0x03a6a1), + (hex!("0100000000333333334444444455000002b00000000000005800"), 0x03a741), + (hex!("0100000000333333334444444455000002b00000000000005a60"), 0x03a7e1), + (hex!("0100000000333333334444444455000002b10000000000002b20"), 0x03a881), + (hex!("0100000000333333334444444455000002b10000000000007b30"), 0x03a921), + (hex!("0100000000333333334444444455000002b20000000000002b30"), 0x03a9c1), + (hex!("0100000000333333334444444455000002b20000000000004440"), 0x03aa61), + (hex!("0100000000333333334444444455000002b20000000000004f80"), 0x03ab01), + (hex!("0100000000333333334444444455000002b20000000000005020"), 0x03aba1), + (hex!("0100000000333333334444444455000002b30000000000002b40"), 0x03ac41), + (hex!("0100000000333333334444444455000002b40000000000002b50"), 0x03ace1), + (hex!("0100000000333333334444444455000002b50000000000002b60"), 0x03ad81), + (hex!("0100000000333333334444444455000002b500000000000059e0"), 0x03ae21), + (hex!("0100000000333333334444444455000002b60000000000002b70"), 0x03aec1), + (hex!("0100000000333333334444444455000002b70000000000002b80"), 0x03af61), + (hex!("0100000000333333334444444455000002b80000000000002b90"), 0x03b001), + (hex!("0100000000333333334444444455000002b80000000000004590"), 0x03b0a1), + (hex!("0100000000333333334444444455000002b800000000000047d0"), 0x03b141), + (hex!("0100000000333333334444444455000002b80000000000006030"), 0x03b1e1), + (hex!("0100000000333333334444444455000002b80000000000006a20"), 0x03b281), + (hex!("0100000000333333334444444455000002b80000000000006a90"), 0x03b321), + (hex!("0100000000333333334444444455000002b90000000000002ba0"), 0x03b3c1), + (hex!("0100000000333333334444444455000002ba0000000000002bb0"), 0x03b461), + (hex!("0100000000333333334444444455000002ba0000000000006e80"), 0x03b501), + (hex!("0100000000333333334444444455000002bb0000000000002bc0"), 0x03b5a1), + (hex!("0100000000333333334444444455000002bc0000000000002bd0"), 0x03b641), + (hex!("0100000000333333334444444455000002bc0000000000004b30"), 0x03b6e1), + (hex!("0100000000333333334444444455000002bd0000000000002be0"), 0x03b781), + (hex!("0100000000333333334444444455000002bd0000000000005e10"), 0x03b821), + (hex!("0100000000333333334444444455000002be0000000000002bf0"), 0x03b8c1), + (hex!("0100000000333333334444444455000002bf0000000000002c00"), 0x03b961), + (hex!("0100000000333333334444444455000002c00000000000002c10"), 0x03ba01), + (hex!("0100000000333333334444444455000002c10000000000002c20"), 0x03baa1), + (hex!("0100000000333333334444444455000002c10000000000003ef0"), 0x03bb41), + (hex!("0100000000333333334444444455000002c20000000000002c30"), 0x03bbe1), + (hex!("0100000000333333334444444455000002c200000000000056e0"), 0x03bc81), + (hex!("0100000000333333334444444455000002c30000000000002c40"), 0x03bd21), + (hex!("0100000000333333334444444455000002c30000000000004b60"), 0x03bdc1), + (hex!("0100000000333333334444444455000002c40000000000002c50"), 0x03be61), + (hex!("0100000000333333334444444455000002c400000000000045f0"), 0x03bf01), + (hex!("0100000000333333334444444455000002c40000000000005290"), 0x03bfa1), + (hex!("0100000000333333334444444455000002c50000000000002c60"), 0x03c041), + (hex!("0100000000333333334444444455000002c60000000000002c70"), 0x03c0e1), + (hex!("0100000000333333334444444455000002c60000000000006ae0"), 0x03c181), + (hex!("0100000000333333334444444455000002c70000000000002c80"), 0x03c221), + (hex!("0100000000333333334444444455000002c70000000000005680"), 0x03c2c1), + (hex!("0100000000333333334444444455000002c70000000000006e10"), 0x03c361), + (hex!("0100000000333333334444444455000002c80000000000002c90"), 0x03c401), + (hex!("0100000000333333334444444455000002c90000000000002ca0"), 0x03c4a1), + (hex!("0100000000333333334444444455000002ca0000000000002cb0"), 0x03c541), + (hex!("0100000000333333334444444455000002cb0000000000002cc0"), 0x03c5e1), + (hex!("0100000000333333334444444455000002cc0000000000002cd0"), 0x03c681), + (hex!("0100000000333333334444444455000002cc0000000000005b50"), 0x03c721), + (hex!("0100000000333333334444444455000002cd0000000000002ce0"), 0x03c7c1), + (hex!("0100000000333333334444444455000002ce0000000000002cf0"), 0x03c861), + (hex!("0100000000333333334444444455000002ce00000000000043f0"), 0x03c901), + (hex!("0100000000333333334444444455000002ce0000000000006420"), 0x03c9a1), + (hex!("0100000000333333334444444455000002cf0000000000002d00"), 0x03ca41), + (hex!("0100000000333333334444444455000002d00000000000002d10"), 0x03cae1), + (hex!("0100000000333333334444444455000002d10000000000002d20"), 0x03cb81), + (hex!("0100000000333333334444444455000002d10000000000005370"), 0x03cc21), + (hex!("0100000000333333334444444455000002d20000000000002d30"), 0x03ccc1), + (hex!("0100000000333333334444444455000002d20000000000005ef0"), 0x03cd61), + (hex!("0100000000333333334444444455000002d20000000000006570"), 0x03ce01), + (hex!("0100000000333333334444444455000002d30000000000002d40"), 0x03cea1), + (hex!("0100000000333333334444444455000002d30000000000007360"), 0x03cf41), + (hex!("0100000000333333334444444455000002d40000000000002d50"), 0x03cfe1), + (hex!("0100000000333333334444444455000002d400000000000079a0"), 0x03d081), + (hex!("0100000000333333334444444455000002d50000000000002d60"), 0x03d121), + (hex!("0100000000333333334444444455000002d50000000000004250"), 0x03d1c1), + (hex!("0100000000333333334444444455000002d50000000000006050"), 0x03d261), + (hex!("0100000000333333334444444455000002d60000000000002d70"), 0x03d301), + (hex!("0100000000333333334444444455000002d60000000000007080"), 0x03d3a1), + (hex!("0100000000333333334444444455000002d70000000000002d80"), 0x03d441), + (hex!("0100000000333333334444444455000002d80000000000002d90"), 0x03d4e1), + (hex!("0100000000333333334444444455000002d80000000000007110"), 0x03d581), + (hex!("0100000000333333334444444455000002d800000000000073c0"), 0x03d621), + (hex!("0100000000333333334444444455000002d800000000000075a0"), 0x03d6c1), + (hex!("0100000000333333334444444455000002d90000000000002da0"), 0x03d761), + (hex!("0100000000333333334444444455000002d90000000000004860"), 0x03d801), + (hex!("0100000000333333334444444455000002d90000000000006b60"), 0x03d8a1), + (hex!("0100000000333333334444444455000002da0000000000002db0"), 0x03d941), + (hex!("0100000000333333334444444455000002da0000000000006630"), 0x03d9e1), + (hex!("0100000000333333334444444455000002db0000000000002dc0"), 0x03da81), + (hex!("0100000000333333334444444455000002dc0000000000002dd0"), 0x03db21), + (hex!("0100000000333333334444444455000002dc0000000000004830"), 0x03dbc1), + (hex!("0100000000333333334444444455000002dd0000000000002de0"), 0x03dc61), + (hex!("0100000000333333334444444455000002de0000000000002df0"), 0x03dd01), + (hex!("0100000000333333334444444455000002de0000000000004f00"), 0x03dda1), + (hex!("0100000000333333334444444455000002df0000000000002e00"), 0x03de41), + (hex!("0100000000333333334444444455000002e00000000000002e10"), 0x03dee1), + (hex!("0100000000333333334444444455000002e10000000000002e20"), 0x03df81), + (hex!("0100000000333333334444444455000002e10000000000006e90"), 0x03e021), + (hex!("0100000000333333334444444455000002e20000000000002e30"), 0x03e0c1), + (hex!("0100000000333333334444444455000002e200000000000053e0"), 0x03e161), + (hex!("0100000000333333334444444455000002e30000000000002e40"), 0x03e201), + (hex!("0100000000333333334444444455000002e30000000000006020"), 0x03e2a1), + (hex!("0100000000333333334444444455000002e30000000000006540"), 0x03e341), + (hex!("0100000000333333334444444455000002e40000000000002e50"), 0x03e3e1), + (hex!("0100000000333333334444444455000002e50000000000002e60"), 0x03e481), + (hex!("0100000000333333334444444455000002e50000000000005180"), 0x03e521), + (hex!("0100000000333333334444444455000002e50000000000007bf0"), 0x03e5c1), + (hex!("0100000000333333334444444455000002e60000000000002e70"), 0x03e661), + (hex!("0100000000333333334444444455000002e60000000000005350"), 0x03e701), + (hex!("0100000000333333334444444455000002e60000000000007960"), 0x03e7a1), + (hex!("0100000000333333334444444455000002e70000000000002e80"), 0x03e841), + (hex!("0100000000333333334444444455000002e80000000000002e90"), 0x03e8e1), + (hex!("0100000000333333334444444455000002e90000000000002ea0"), 0x03e981), + (hex!("0100000000333333334444444455000002ea0000000000002eb0"), 0x03ea21), + (hex!("0100000000333333334444444455000002eb0000000000002ec0"), 0x03eac1), + (hex!("0100000000333333334444444455000002ec0000000000002ed0"), 0x03eb61), + (hex!("0100000000333333334444444455000002ec0000000000006c10"), 0x03ec01), + (hex!("0100000000333333334444444455000002ed0000000000002ee0"), 0x03eca1), + (hex!("0100000000333333334444444455000002ed0000000000005590"), 0x03ed41), + (hex!("0100000000333333334444444455000002ed0000000000005cd0"), 0x03ede1), + (hex!("0100000000333333334444444455000002ed0000000000006910"), 0x03ee81), + (hex!("0100000000333333334444444455000002ee0000000000002ef0"), 0x03ef21), + (hex!("0100000000333333334444444455000002ef0000000000002f00"), 0x03efc1), + (hex!("0100000000333333334444444455000002ef0000000000004ed0"), 0x03f061), + (hex!("0100000000333333334444444455000002f00000000000002f10"), 0x03f101), + (hex!("0100000000333333334444444455000002f00000000000004cf0"), 0x03f1a1), + (hex!("0100000000333333334444444455000002f00000000000005d10"), 0x03f241), + (hex!("0100000000333333334444444455000002f00000000000006860"), 0x03f2e1), + (hex!("0100000000333333334444444455000002f00000000000006b50"), 0x03f381), + (hex!("0100000000333333334444444455000002f00000000000007100"), 0x03f421), + (hex!("0100000000333333334444444455000002f00000000000007aa0"), 0x03f4c1), + (hex!("0100000000333333334444444455000002f10000000000002f20"), 0x03f561), + (hex!("0100000000333333334444444455000002f20000000000002f30"), 0x03f601), + (hex!("0100000000333333334444444455000002f200000000000044b0"), 0x03f6a1), + (hex!("0100000000333333334444444455000002f30000000000002f40"), 0x03f741), + (hex!("0100000000333333334444444455000002f300000000000075b0"), 0x03f7e1), + (hex!("0100000000333333334444444455000002f40000000000002f50"), 0x03f881), + (hex!("0100000000333333334444444455000002f400000000000060f0"), 0x03f921), + (hex!("0100000000333333334444444455000002f50000000000002f60"), 0x03f9c1), + (hex!("0100000000333333334444444455000002f50000000000007210"), 0x03fa61), + (hex!("0100000000333333334444444455000002f60000000000002f70"), 0x03fb01), + (hex!("0100000000333333334444444455000002f60000000000006610"), 0x03fba1), + (hex!("0100000000333333334444444455000002f70000000000002f80"), 0x03fc41), + (hex!("0100000000333333334444444455000002f70000000000007560"), 0x03fce1), + (hex!("0100000000333333334444444455000002f80000000000002f90"), 0x03fd81), + (hex!("0100000000333333334444444455000002f80000000000006320"), 0x03fe21), + (hex!("0100000000333333334444444455000002f90000000000002fa0"), 0x03fec1), + (hex!("0100000000333333334444444455000002f90000000000006e50"), 0x03ff61), + (hex!("0100000000333333334444444455000002fa0000000000002fb0"), 0x040001), + (hex!("0100000000333333334444444455000002fb0000000000002fc0"), 0x0400a1), + (hex!("0100000000333333334444444455000002fb0000000000004780"), 0x040141), + (hex!("0100000000333333334444444455000002fc0000000000002fd0"), 0x0401e1), + (hex!("0100000000333333334444444455000002fd0000000000002fe0"), 0x040281), + (hex!("0100000000333333334444444455000002fd0000000000005600"), 0x040321), + (hex!("0100000000333333334444444455000002fd0000000000006c00"), 0x0403c1), + (hex!("0100000000333333334444444455000002fe0000000000002ff0"), 0x040461), + (hex!("0100000000333333334444444455000002ff0000000000003000"), 0x040501), + (hex!("0100000000333333334444444455000003000000000000003010"), 0x0405a1), + (hex!("0100000000333333334444444455000003000000000000004080"), 0x040641), + (hex!("0100000000333333334444444455000003010000000000003020"), 0x0406e1), + (hex!("0100000000333333334444444455000003010000000000006340"), 0x040781), + (hex!("0100000000333333334444444455000003020000000000003030"), 0x040821), + (hex!("0100000000333333334444444455000003020000000000005b00"), 0x0408c1), + (hex!("0100000000333333334444444455000003020000000000007b20"), 0x040961), + (hex!("0100000000333333334444444455000003030000000000003040"), 0x040a01), + (hex!("01000000003333333344444444550000030300000000000056b0"), 0x040aa1), + (hex!("0100000000333333334444444455000003030000000000006280"), 0x040b41), + (hex!("0100000000333333334444444455000003030000000000007ad0"), 0x040be1), + (hex!("0100000000333333334444444455000003040000000000003050"), 0x040c81), + (hex!("0100000000333333334444444455000003040000000000005c50"), 0x040d21), + (hex!("0100000000333333334444444455000003050000000000003060"), 0x040dc1), + (hex!("01000000003333333344444444550000030500000000000072e0"), 0x040e61), + (hex!("0100000000333333334444444455000003060000000000003070"), 0x040f01), + (hex!("0100000000333333334444444455000003060000000000004360"), 0x040fa1), + (hex!("0100000000333333334444444455000003060000000000004380"), 0x041041), + (hex!("0100000000333333334444444455000003060000000000004820"), 0x0410e1), + (hex!("0100000000333333334444444455000003060000000000006d10"), 0x041181), + (hex!("0100000000333333334444444455000003070000000000003080"), 0x041221), + (hex!("0100000000333333334444444455000003070000000000004450"), 0x0412c1), + (hex!("0100000000333333334444444455000003080000000000003090"), 0x041361), + (hex!("0100000000333333334444444455000003080000000000005ad0"), 0x041401), + (hex!("01000000003333333344444444550000030900000000000030a0"), 0x0414a1), + (hex!("01000000003333333344444444550000030a00000000000030b0"), 0x041541), + (hex!("01000000003333333344444444550000030a0000000000007760"), 0x0415e1), + (hex!("01000000003333333344444444550000030b00000000000030c0"), 0x041681), + (hex!("01000000003333333344444444550000030b0000000000007a80"), 0x041721), + (hex!("01000000003333333344444444550000030c00000000000030d0"), 0x0417c1), + (hex!("01000000003333333344444444550000030d00000000000030e0"), 0x041861), + (hex!("01000000003333333344444444550000030d0000000000003eb0"), 0x041901), + (hex!("01000000003333333344444444550000030e00000000000030f0"), 0x0419a1), + (hex!("01000000003333333344444444550000030f0000000000003100"), 0x041a41), + (hex!("01000000003333333344444444550000030f0000000000004690"), 0x041ae1), + (hex!("01000000003333333344444444550000030f0000000000006900"), 0x041b81), + (hex!("0100000000333333334444444455000003100000000000003110"), 0x041c21), + (hex!("01000000003333333344444444550000031000000000000058a0"), 0x041cc1), + (hex!("0100000000333333334444444455000003110000000000003120"), 0x041d61), + (hex!("0100000000333333334444444455000003110000000000004200"), 0x041e01), + (hex!("0100000000333333334444444455000003120000000000003130"), 0x041ea1), + (hex!("0100000000333333334444444455000003130000000000003140"), 0x041f41), + (hex!("0100000000333333334444444455000003130000000000004d50"), 0x041fe1), + (hex!("0100000000333333334444444455000003130000000000005400"), 0x042081), + (hex!("0100000000333333334444444455000003130000000000005520"), 0x042121), + (hex!("0100000000333333334444444455000003140000000000003150"), 0x0421c1), + (hex!("0100000000333333334444444455000003140000000000006450"), 0x042261), + (hex!("0100000000333333334444444455000003150000000000003160"), 0x042301), + (hex!("01000000003333333344444444550000031500000000000062d0"), 0x0423a1), + (hex!("0100000000333333334444444455000003160000000000003170"), 0x042441), + (hex!("0100000000333333334444444455000003160000000000004c40"), 0x0424e1), + (hex!("0100000000333333334444444455000003160000000000007c80"), 0x042581), + (hex!("0100000000333333334444444455000003170000000000003180"), 0x042621), + (hex!("0100000000333333334444444455000003170000000000004400"), 0x0426c1), + (hex!("0100000000333333334444444455000003170000000000005090"), 0x042761), + (hex!("0100000000333333334444444455000003170000000000006cb0"), 0x042801), + (hex!("0100000000333333334444444455000003180000000000003190"), 0x0428a1), + (hex!("0100000000333333334444444455000003180000000000006560"), 0x042941), + (hex!("01000000003333333344444444550000031900000000000031a0"), 0x0429e1), + (hex!("01000000003333333344444444550000031900000000000052d0"), 0x042a81), + (hex!("01000000003333333344444444550000031900000000000057e0"), 0x042b21), + (hex!("01000000003333333344444444550000031a00000000000031b0"), 0x042bc1), + (hex!("01000000003333333344444444550000031a00000000000071e0"), 0x042c61), + (hex!("01000000003333333344444444550000031b00000000000031c0"), 0x042d01), + (hex!("01000000003333333344444444550000031c00000000000031d0"), 0x042da1), + (hex!("01000000003333333344444444550000031c0000000000004480"), 0x042e41), + (hex!("01000000003333333344444444550000031c0000000000005790"), 0x042ee1), + (hex!("01000000003333333344444444550000031c0000000000007be0"), 0x042f81), + (hex!("01000000003333333344444444550000031d00000000000031e0"), 0x043021), + (hex!("01000000003333333344444444550000031d0000000000005560"), 0x0430c1), + (hex!("01000000003333333344444444550000031e00000000000031f0"), 0x043161), + (hex!("01000000003333333344444444550000031f0000000000003200"), 0x043201), + (hex!("01000000003333333344444444550000031f0000000000004190"), 0x0432a1), + (hex!("0100000000333333334444444455000003200000000000003210"), 0x043341), + (hex!("0100000000333333334444444455000003210000000000003220"), 0x0433e1), + (hex!("0100000000333333334444444455000003220000000000003230"), 0x043481), + (hex!("0100000000333333334444444455000003230000000000003240"), 0x043521), + (hex!("01000000003333333344444444550000032300000000000069d0"), 0x0435c1), + (hex!("0100000000333333334444444455000003240000000000003250"), 0x043661), + (hex!("0100000000333333334444444455000003250000000000003260"), 0x043701), + (hex!("01000000003333333344444444550000032500000000000042b0"), 0x0437a1), + (hex!("01000000003333333344444444550000032500000000000064e0"), 0x043841), + (hex!("0100000000333333334444444455000003260000000000003270"), 0x0438e1), + (hex!("0100000000333333334444444455000003270000000000003280"), 0x043981), + (hex!("0100000000333333334444444455000003270000000000005b20"), 0x043a21), + (hex!("0100000000333333334444444455000003270000000000006330"), 0x043ac1), + (hex!("0100000000333333334444444455000003270000000000006810"), 0x043b61), + (hex!("0100000000333333334444444455000003280000000000003290"), 0x043c01), + (hex!("01000000003333333344444444550000032900000000000032a0"), 0x043ca1), + (hex!("01000000003333333344444444550000032900000000000056f0"), 0x043d41), + (hex!("0100000000333333334444444455000003290000000000005e20"), 0x043de1), + (hex!("0100000000333333334444444455000003290000000000005e70"), 0x043e81), + (hex!("01000000003333333344444444550000032a00000000000032b0"), 0x043f21), + (hex!("01000000003333333344444444550000032b00000000000032c0"), 0x043fc1), + (hex!("01000000003333333344444444550000032b0000000000005500"), 0x044061), + (hex!("01000000003333333344444444550000032b0000000000005a20"), 0x044101), + (hex!("01000000003333333344444444550000032c00000000000032d0"), 0x0441a1), + (hex!("01000000003333333344444444550000032c0000000000004060"), 0x044241), + (hex!("01000000003333333344444444550000032c0000000000004760"), 0x0442e1), + (hex!("01000000003333333344444444550000032d00000000000032e0"), 0x044381), + (hex!("01000000003333333344444444550000032d00000000000068a0"), 0x044421), + (hex!("01000000003333333344444444550000032e00000000000032f0"), 0x0444c1), + (hex!("01000000003333333344444444550000032f0000000000003300"), 0x044561), + (hex!("0100000000333333334444444455000003300000000000003310"), 0x044601), + (hex!("0100000000333333334444444455000003300000000000006e40"), 0x0446a1), + (hex!("0100000000333333334444444455000003310000000000003320"), 0x044741), + (hex!("0100000000333333334444444455000003310000000000004620"), 0x0447e1), + (hex!("0100000000333333334444444455000003320000000000003330"), 0x044881), + (hex!("0100000000333333334444444455000003330000000000003340"), 0x044921), + (hex!("0100000000333333334444444455000003330000000000004b80"), 0x0449c1), + (hex!("0100000000333333334444444455000003340000000000003350"), 0x044a61), + (hex!("0100000000333333334444444455000003350000000000003360"), 0x044b01), + (hex!("0100000000333333334444444455000003360000000000003370"), 0x044ba1), + (hex!("0100000000333333334444444455000003370000000000003380"), 0x044c41), + (hex!("0100000000333333334444444455000003380000000000003390"), 0x044ce1), + (hex!("01000000003333333344444444550000033900000000000033a0"), 0x044d81), + (hex!("0100000000333333334444444455000003390000000000006b90"), 0x044e21), + (hex!("01000000003333333344444444550000033a00000000000033b0"), 0x044ec1), + (hex!("01000000003333333344444444550000033a0000000000007420"), 0x044f61), + (hex!("01000000003333333344444444550000033b00000000000033c0"), 0x045001), + (hex!("01000000003333333344444444550000033b0000000000007620"), 0x0450a1), + (hex!("01000000003333333344444444550000033c00000000000033d0"), 0x045141), + (hex!("01000000003333333344444444550000033c0000000000006b30"), 0x0451e1), + (hex!("01000000003333333344444444550000033d00000000000033e0"), 0x045281), + (hex!("01000000003333333344444444550000033e00000000000033f0"), 0x045321), + (hex!("01000000003333333344444444550000033e00000000000048b0"), 0x0453c1), + (hex!("01000000003333333344444444550000033e0000000000004e70"), 0x045461), + (hex!("01000000003333333344444444550000033f0000000000003400"), 0x045501), + (hex!("01000000003333333344444444550000033f0000000000006380"), 0x0455a1), + (hex!("0100000000333333334444444455000003400000000000003410"), 0x045641), + (hex!("0100000000333333334444444455000003410000000000003420"), 0x0456e1), + (hex!("0100000000333333334444444455000003410000000000006090"), 0x045781), + (hex!("0100000000333333334444444455000003420000000000003430"), 0x045821), + (hex!("01000000003333333344444444550000034200000000000073d0"), 0x0458c1), + (hex!("0100000000333333334444444455000003430000000000003440"), 0x045961), + (hex!("0100000000333333334444444455000003430000000000006370"), 0x045a01), + (hex!("01000000003333333344444444550000034300000000000075c0"), 0x045aa1), + (hex!("0100000000333333334444444455000003440000000000003450"), 0x045b41), + (hex!("0100000000333333334444444455000003450000000000003460"), 0x045be1), + (hex!("0100000000333333334444444455000003460000000000003470"), 0x045c81), + (hex!("01000000003333333344444444550000034600000000000055f0"), 0x045d21), + (hex!("0100000000333333334444444455000003470000000000003480"), 0x045dc1), + (hex!("0100000000333333334444444455000003470000000000003fe0"), 0x045e61), + (hex!("0100000000333333334444444455000003480000000000003490"), 0x045f01), + (hex!("0100000000333333334444444455000003480000000000007990"), 0x045fa1), + (hex!("01000000003333333344444444550000034900000000000034a0"), 0x046041), + (hex!("0100000000333333334444444455000003490000000000004410"), 0x0460e1), + (hex!("01000000003333333344444444550000034a00000000000034b0"), 0x046181), + (hex!("01000000003333333344444444550000034a00000000000062a0"), 0x046221), + (hex!("01000000003333333344444444550000034a0000000000007260"), 0x0462c1), + (hex!("01000000003333333344444444550000034b00000000000034c0"), 0x046361), + (hex!("01000000003333333344444444550000034b0000000000005760"), 0x046401), + (hex!("01000000003333333344444444550000034b0000000000006200"), 0x0464a1), + (hex!("01000000003333333344444444550000034c00000000000034d0"), 0x046541), + (hex!("01000000003333333344444444550000034d00000000000034e0"), 0x0465e1), + (hex!("01000000003333333344444444550000034e00000000000034f0"), 0x046681), + (hex!("01000000003333333344444444550000034e0000000000007790"), 0x046721), + (hex!("01000000003333333344444444550000034f0000000000003500"), 0x0467c1), + (hex!("0100000000333333334444444455000003500000000000003510"), 0x046861), + (hex!("0100000000333333334444444455000003510000000000003520"), 0x046901), + (hex!("0100000000333333334444444455000003520000000000003530"), 0x0469a1), + (hex!("01000000003333333344444444550000035200000000000056a0"), 0x046a41), + (hex!("0100000000333333334444444455000003530000000000003540"), 0x046ae1), + (hex!("0100000000333333334444444455000003540000000000003550"), 0x046b81), + (hex!("01000000003333333344444444550000035400000000000047b0"), 0x046c21), + (hex!("0100000000333333334444444455000003550000000000003560"), 0x046cc1), + (hex!("0100000000333333334444444455000003550000000000004500"), 0x046d61), + (hex!("0100000000333333334444444455000003560000000000003570"), 0x046e01), + (hex!("0100000000333333334444444455000003560000000000004fc0"), 0x046ea1), + (hex!("0100000000333333334444444455000003560000000000007160"), 0x046f41), + (hex!("0100000000333333334444444455000003560000000000007400"), 0x046fe1), + (hex!("0100000000333333334444444455000003570000000000003580"), 0x047081), + (hex!("0100000000333333334444444455000003580000000000003590"), 0x047121), + (hex!("0100000000333333334444444455000003580000000000005a80"), 0x0471c1), + (hex!("01000000003333333344444444550000035900000000000035a0"), 0x047261), + (hex!("01000000003333333344444444550000035900000000000073b0"), 0x047301), + (hex!("01000000003333333344444444550000035a00000000000035b0"), 0x0473a1), + (hex!("01000000003333333344444444550000035a0000000000004c20"), 0x047441), + (hex!("01000000003333333344444444550000035b00000000000035c0"), 0x0474e1), + (hex!("01000000003333333344444444550000035b0000000000005120"), 0x047581), + (hex!("01000000003333333344444444550000035c00000000000035d0"), 0x047621), + (hex!("01000000003333333344444444550000035c0000000000004300"), 0x0476c1), + (hex!("01000000003333333344444444550000035c0000000000005a40"), 0x047761), + (hex!("01000000003333333344444444550000035c0000000000006620"), 0x047801), + (hex!("01000000003333333344444444550000035c0000000000006ed0"), 0x0478a1), + (hex!("01000000003333333344444444550000035d00000000000035e0"), 0x047941), + (hex!("01000000003333333344444444550000035d0000000000005df0"), 0x0479e1), + (hex!("01000000003333333344444444550000035e00000000000035f0"), 0x047a81), + (hex!("01000000003333333344444444550000035f0000000000003600"), 0x047b21), + (hex!("01000000003333333344444444550000035f00000000000058d0"), 0x047bc1), + (hex!("0100000000333333334444444455000003600000000000003610"), 0x047c61), + (hex!("0100000000333333334444444455000003600000000000007b90"), 0x047d01), + (hex!("0100000000333333334444444455000003610000000000003620"), 0x047da1), + (hex!("0100000000333333334444444455000003610000000000006ad0"), 0x047e41), + (hex!("0100000000333333334444444455000003620000000000003630"), 0x047ee1), + (hex!("01000000003333333344444444550000036200000000000063a0"), 0x047f81), + (hex!("0100000000333333334444444455000003630000000000003640"), 0x048021), + (hex!("0100000000333333334444444455000003630000000000007250"), 0x0480c1), + (hex!("0100000000333333334444444455000003640000000000003650"), 0x048161), + (hex!("0100000000333333334444444455000003640000000000005510"), 0x048201), + (hex!("0100000000333333334444444455000003640000000000007850"), 0x0482a1), + (hex!("0100000000333333334444444455000003650000000000003660"), 0x048341), + (hex!("0100000000333333334444444455000003660000000000003670"), 0x0483e1), + (hex!("0100000000333333334444444455000003660000000000004650"), 0x048481), + (hex!("01000000003333333344444444550000036600000000000050d0"), 0x048521), + (hex!("0100000000333333334444444455000003660000000000006eb0"), 0x0485c1), + (hex!("0100000000333333334444444455000003670000000000003680"), 0x048661), + (hex!("01000000003333333344444444550000036700000000000071f0"), 0x048701), + (hex!("0100000000333333334444444455000003680000000000003690"), 0x0487a1), + (hex!("01000000003333333344444444550000036900000000000036a0"), 0x048841), + (hex!("0100000000333333334444444455000003690000000000005c70"), 0x0488e1), + (hex!("01000000003333333344444444550000036a00000000000036b0"), 0x048981), + (hex!("01000000003333333344444444550000036a00000000000071b0"), 0x048a21), + (hex!("01000000003333333344444444550000036b00000000000036c0"), 0x048ac1), + (hex!("01000000003333333344444444550000036b0000000000004670"), 0x048b61), + (hex!("01000000003333333344444444550000036c00000000000036d0"), 0x048c01), + (hex!("01000000003333333344444444550000036c0000000000004750"), 0x048ca1), + (hex!("01000000003333333344444444550000036c0000000000006fa0"), 0x048d41), + (hex!("01000000003333333344444444550000036d00000000000036e0"), 0x048de1), + (hex!("01000000003333333344444444550000036d0000000000003f70"), 0x048e81), + (hex!("01000000003333333344444444550000036d0000000000004b90"), 0x048f21), + (hex!("01000000003333333344444444550000036d00000000000057a0"), 0x048fc1), + (hex!("01000000003333333344444444550000036e00000000000036f0"), 0x049061), + (hex!("01000000003333333344444444550000036e00000000000075d0"), 0x049101), + (hex!("01000000003333333344444444550000036f0000000000003700"), 0x0491a1), + (hex!("0100000000333333334444444455000003700000000000003710"), 0x049241), + (hex!("0100000000333333334444444455000003700000000000005aa0"), 0x0492e1), + (hex!("0100000000333333334444444455000003710000000000003720"), 0x049381), + (hex!("0100000000333333334444444455000003710000000000005130"), 0x049421), + (hex!("0100000000333333334444444455000003710000000000006fc0"), 0x0494c1), + (hex!("0100000000333333334444444455000003710000000000007b00"), 0x049561), + (hex!("0100000000333333334444444455000003720000000000003730"), 0x049601), + (hex!("01000000003333333344444444550000037200000000000054d0"), 0x0496a1), + (hex!("0100000000333333334444444455000003730000000000003740"), 0x049741), + (hex!("0100000000333333334444444455000003730000000000004220"), 0x0497e1), + (hex!("0100000000333333334444444455000003740000000000003750"), 0x049881), + (hex!("0100000000333333334444444455000003740000000000004720"), 0x049921), + (hex!("0100000000333333334444444455000003750000000000003760"), 0x0499c1), + (hex!("0100000000333333334444444455000003750000000000004110"), 0x049a61), + (hex!("0100000000333333334444444455000003760000000000003770"), 0x049b01), + (hex!("0100000000333333334444444455000003770000000000003780"), 0x049ba1), + (hex!("0100000000333333334444444455000003780000000000003790"), 0x049c41), + (hex!("0100000000333333334444444455000003780000000000004b40"), 0x049ce1), + (hex!("0100000000333333334444444455000003780000000000005660"), 0x049d81), + (hex!("0100000000333333334444444455000003780000000000005ea0"), 0x049e21), + (hex!("01000000003333333344444444550000037900000000000037a0"), 0x049ec1), + (hex!("01000000003333333344444444550000037a00000000000037b0"), 0x049f61), + (hex!("01000000003333333344444444550000037b00000000000037c0"), 0x04a001), + (hex!("01000000003333333344444444550000037c00000000000037d0"), 0x04a0a1), + (hex!("01000000003333333344444444550000037c0000000000004340"), 0x04a141), + (hex!("01000000003333333344444444550000037c0000000000005230"), 0x04a1e1), + (hex!("01000000003333333344444444550000037d00000000000037e0"), 0x04a281), + (hex!("01000000003333333344444444550000037d00000000000051e0"), 0x04a321), + (hex!("01000000003333333344444444550000037e00000000000037f0"), 0x04a3c1), + (hex!("01000000003333333344444444550000037e0000000000004090"), 0x04a461), + (hex!("01000000003333333344444444550000037e0000000000005c20"), 0x04a501), + (hex!("01000000003333333344444444550000037f0000000000003800"), 0x04a5a1), + (hex!("0100000000333333334444444455000003800000000000003810"), 0x04a641), + (hex!("0100000000333333334444444455000003800000000000007630"), 0x04a6e1), + (hex!("0100000000333333334444444455000003810000000000003820"), 0x04a781), + (hex!("0100000000333333334444444455000003820000000000003830"), 0x04a821), + (hex!("0100000000333333334444444455000003820000000000004170"), 0x04a8c1), + (hex!("0100000000333333334444444455000003830000000000003840"), 0x04a961), + (hex!("0100000000333333334444444455000003840000000000003850"), 0x04aa01), + (hex!("0100000000333333334444444455000003850000000000003860"), 0x04aaa1), + (hex!("0100000000333333334444444455000003850000000000004180"), 0x04ab41), + (hex!("0100000000333333334444444455000003850000000000005c90"), 0x04abe1), + (hex!("0100000000333333334444444455000003850000000000005da0"), 0x04ac81), + (hex!("0100000000333333334444444455000003850000000000006ff0"), 0x04ad21), + (hex!("0100000000333333334444444455000003860000000000003870"), 0x04adc1), + (hex!("01000000003333333344444444550000038600000000000065c0"), 0x04ae61), + (hex!("0100000000333333334444444455000003870000000000003880"), 0x04af01), + (hex!("0100000000333333334444444455000003870000000000007cc0"), 0x04afa1), + (hex!("0100000000333333334444444455000003880000000000003890"), 0x04b041), + (hex!("01000000003333333344444444550000038900000000000038a0"), 0x04b0e1), + (hex!("01000000003333333344444444550000038a00000000000038b0"), 0x04b181), + (hex!("01000000003333333344444444550000038a00000000000073e0"), 0x04b221), + (hex!("01000000003333333344444444550000038b00000000000038c0"), 0x04b2c1), + (hex!("01000000003333333344444444550000038c00000000000038d0"), 0x04b361), + (hex!("01000000003333333344444444550000038d00000000000038e0"), 0x04b401), + (hex!("01000000003333333344444444550000038d00000000000069f0"), 0x04b4a1), + (hex!("01000000003333333344444444550000038d0000000000007680"), 0x04b541), + (hex!("01000000003333333344444444550000038e00000000000038f0"), 0x04b5e1), + (hex!("01000000003333333344444444550000038f0000000000003900"), 0x04b681), + (hex!("01000000003333333344444444550000038f00000000000045b0"), 0x04b721), + (hex!("01000000003333333344444444550000038f0000000000007180"), 0x04b7c1), + (hex!("0100000000333333334444444455000003900000000000003910"), 0x04b861), + (hex!("0100000000333333334444444455000003910000000000003920"), 0x04b901), + (hex!("0100000000333333334444444455000003910000000000004a20"), 0x04b9a1), + (hex!("0100000000333333334444444455000003920000000000003930"), 0x04ba41), + (hex!("01000000003333333344444444550000039200000000000059b0"), 0x04bae1), + (hex!("0100000000333333334444444455000003930000000000003940"), 0x04bb81), + (hex!("0100000000333333334444444455000003930000000000006cc0"), 0x04bc21), + (hex!("0100000000333333334444444455000003940000000000003950"), 0x04bcc1), + (hex!("01000000003333333344444444550000039400000000000056c0"), 0x04bd61), + (hex!("0100000000333333334444444455000003950000000000003960"), 0x04be01), + (hex!("0100000000333333334444444455000003950000000000004cc0"), 0x04bea1), + (hex!("0100000000333333334444444455000003950000000000007720"), 0x04bf41), + (hex!("0100000000333333334444444455000003960000000000003970"), 0x04bfe1), + (hex!("0100000000333333334444444455000003960000000000004da0"), 0x04c081), + (hex!("0100000000333333334444444455000003960000000000004df0"), 0x04c121), + (hex!("0100000000333333334444444455000003960000000000004f30"), 0x04c1c1), + (hex!("01000000003333333344444444550000039600000000000050f0"), 0x04c261), + (hex!("0100000000333333334444444455000003960000000000007940"), 0x04c301), + (hex!("0100000000333333334444444455000003970000000000003980"), 0x04c3a1), + (hex!("0100000000333333334444444455000003970000000000005850"), 0x04c441), + (hex!("0100000000333333334444444455000003970000000000007bd0"), 0x04c4e1), + (hex!("0100000000333333334444444455000003980000000000003990"), 0x04c581), + (hex!("0100000000333333334444444455000003980000000000004c00"), 0x04c621), + (hex!("0100000000333333334444444455000003980000000000005580"), 0x04c6c1), + (hex!("01000000003333333344444444550000039900000000000039a0"), 0x04c761), + (hex!("0100000000333333334444444455000003990000000000005820"), 0x04c801), + (hex!("01000000003333333344444444550000039a00000000000039b0"), 0x04c8a1), + (hex!("01000000003333333344444444550000039b00000000000039c0"), 0x04c941), + (hex!("01000000003333333344444444550000039b0000000000004c10"), 0x04c9e1), + (hex!("01000000003333333344444444550000039b0000000000006460"), 0x04ca81), + (hex!("01000000003333333344444444550000039c00000000000039d0"), 0x04cb21), + (hex!("01000000003333333344444444550000039d00000000000039e0"), 0x04cbc1), + (hex!("01000000003333333344444444550000039d00000000000044c0"), 0x04cc61), + (hex!("01000000003333333344444444550000039d00000000000049e0"), 0x04cd01), + (hex!("01000000003333333344444444550000039e00000000000039f0"), 0x04cda1), + (hex!("01000000003333333344444444550000039f0000000000003a00"), 0x04ce41), + (hex!("0100000000333333334444444455000003a00000000000003a10"), 0x04cee1), + (hex!("0100000000333333334444444455000003a10000000000003a20"), 0x04cf81), + (hex!("0100000000333333334444444455000003a10000000000006a80"), 0x04d021), + (hex!("0100000000333333334444444455000003a20000000000003a30"), 0x04d0c1), + (hex!("0100000000333333334444444455000003a200000000000062b0"), 0x04d161), + (hex!("0100000000333333334444444455000003a30000000000003a40"), 0x04d201), + (hex!("0100000000333333334444444455000003a30000000000006ce0"), 0x04d2a1), + (hex!("0100000000333333334444444455000003a40000000000003a50"), 0x04d341), + (hex!("0100000000333333334444444455000003a50000000000003a60"), 0x04d3e1), + (hex!("0100000000333333334444444455000003a60000000000003a70"), 0x04d481), + (hex!("0100000000333333334444444455000003a60000000000007750"), 0x04d521), + (hex!("0100000000333333334444444455000003a70000000000003a80"), 0x04d5c1), + (hex!("0100000000333333334444444455000003a70000000000005b10"), 0x04d661), + (hex!("0100000000333333334444444455000003a80000000000003a90"), 0x04d701), + (hex!("0100000000333333334444444455000003a80000000000006c20"), 0x04d7a1), + (hex!("0100000000333333334444444455000003a90000000000003aa0"), 0x04d841), + (hex!("0100000000333333334444444455000003a90000000000005b70"), 0x04d8e1), + (hex!("0100000000333333334444444455000003a900000000000070e0"), 0x04d981), + (hex!("0100000000333333334444444455000003aa0000000000003ab0"), 0x04da21), + (hex!("0100000000333333334444444455000003aa00000000000049f0"), 0x04dac1), + (hex!("0100000000333333334444444455000003aa0000000000004d60"), 0x04db61), + (hex!("0100000000333333334444444455000003ab0000000000003ac0"), 0x04dc01), + (hex!("0100000000333333334444444455000003ac0000000000003ad0"), 0x04dca1), + (hex!("0100000000333333334444444455000003ac0000000000004580"), 0x04dd41), + (hex!("0100000000333333334444444455000003ad0000000000003ae0"), 0x04dde1), + (hex!("0100000000333333334444444455000003ae0000000000003af0"), 0x04de81), + (hex!("0100000000333333334444444455000003af0000000000003b00"), 0x04df21), + (hex!("0100000000333333334444444455000003b00000000000003b10"), 0x04dfc1), + (hex!("0100000000333333334444444455000003b10000000000003b20"), 0x04e061), + (hex!("0100000000333333334444444455000003b10000000000003fd0"), 0x04e101), + (hex!("0100000000333333334444444455000003b20000000000003b30"), 0x04e1a1), + (hex!("0100000000333333334444444455000003b30000000000003b40"), 0x04e241), + (hex!("0100000000333333334444444455000003b40000000000003b50"), 0x04e2e1), + (hex!("0100000000333333334444444455000003b40000000000007450"), 0x04e381), + (hex!("0100000000333333334444444455000003b50000000000003b60"), 0x04e421), + (hex!("0100000000333333334444444455000003b60000000000003b70"), 0x04e4c1), + (hex!("0100000000333333334444444455000003b70000000000003b80"), 0x04e561), + (hex!("0100000000333333334444444455000003b70000000000006d50"), 0x04e601), + (hex!("0100000000333333334444444455000003b80000000000003b90"), 0x04e6a1), + (hex!("0100000000333333334444444455000003b800000000000057c0"), 0x04e741), + (hex!("0100000000333333334444444455000003b800000000000078a0"), 0x04e7e1), + (hex!("0100000000333333334444444455000003b90000000000003ba0"), 0x04e881), + (hex!("0100000000333333334444444455000003b90000000000006750"), 0x04e921), + (hex!("0100000000333333334444444455000003ba0000000000003bb0"), 0x04e9c1), + (hex!("0100000000333333334444444455000003ba0000000000007a10"), 0x04ea61), + (hex!("0100000000333333334444444455000003ba0000000000007a20"), 0x04eb01), + (hex!("0100000000333333334444444455000003bb0000000000003bc0"), 0x04eba1), + (hex!("0100000000333333334444444455000003bb0000000000005bc0"), 0x04ec41), + (hex!("0100000000333333334444444455000003bc0000000000003bd0"), 0x04ece1), + (hex!("0100000000333333334444444455000003bc0000000000005e80"), 0x04ed81), + (hex!("0100000000333333334444444455000003bc0000000000007ab0"), 0x04ee21), + (hex!("0100000000333333334444444455000003bd0000000000003be0"), 0x04eec1), + (hex!("0100000000333333334444444455000003bd00000000000049b0"), 0x04ef61), + (hex!("0100000000333333334444444455000003be0000000000003bf0"), 0x04f001), + (hex!("0100000000333333334444444455000003be0000000000005780"), 0x04f0a1), + (hex!("0100000000333333334444444455000003be0000000000007930"), 0x04f141), + (hex!("0100000000333333334444444455000003bf0000000000003c00"), 0x04f1e1), + (hex!("0100000000333333334444444455000003bf0000000000005de0"), 0x04f281), + (hex!("0100000000333333334444444455000003bf00000000000060b0"), 0x04f321), + (hex!("0100000000333333334444444455000003bf00000000000060c0"), 0x04f3c1), + (hex!("0100000000333333334444444455000003bf0000000000006a50"), 0x04f461), + (hex!("0100000000333333334444444455000003c00000000000003c10"), 0x04f501), + (hex!("0100000000333333334444444455000003c00000000000004030"), 0x04f5a1), + (hex!("0100000000333333334444444455000003c10000000000003c20"), 0x04f641), + (hex!("0100000000333333334444444455000003c20000000000003c30"), 0x04f6e1), + (hex!("0100000000333333334444444455000003c200000000000040b0"), 0x04f781), + (hex!("0100000000333333334444444455000003c30000000000003c40"), 0x04f821), + (hex!("0100000000333333334444444455000003c40000000000003c50"), 0x04f8c1), + (hex!("0100000000333333334444444455000003c40000000000005ba0"), 0x04f961), + (hex!("0100000000333333334444444455000003c50000000000003c60"), 0x04fa01), + (hex!("0100000000333333334444444455000003c60000000000003c70"), 0x04faa1), + (hex!("0100000000333333334444444455000003c70000000000003c80"), 0x04fb41), + (hex!("0100000000333333334444444455000003c70000000000004270"), 0x04fbe1), + (hex!("0100000000333333334444444455000003c80000000000003c90"), 0x04fc81), + (hex!("0100000000333333334444444455000003c80000000000006e70"), 0x04fd21), + (hex!("0100000000333333334444444455000003c90000000000003ca0"), 0x04fdc1), + (hex!("0100000000333333334444444455000003ca0000000000003cb0"), 0x04fe61), + (hex!("0100000000333333334444444455000003ca0000000000006e20"), 0x04ff01), + (hex!("0100000000333333334444444455000003ca0000000000007c20"), 0x04ffa1), + (hex!("0100000000333333334444444455000003cb0000000000003cc0"), 0x050041), + (hex!("0100000000333333334444444455000003cc0000000000003cd0"), 0x0500e1), + (hex!("0100000000333333334444444455000003cc0000000000006120"), 0x050181), + (hex!("0100000000333333334444444455000003cc0000000000007950"), 0x050221), + (hex!("0100000000333333334444444455000003cd0000000000003ce0"), 0x0502c1), + (hex!("0100000000333333334444444455000003ce0000000000003cf0"), 0x050361), + (hex!("0100000000333333334444444455000003cf0000000000003d00"), 0x050401), + (hex!("0100000000333333334444444455000003d00000000000003d10"), 0x0504a1), + (hex!("0100000000333333334444444455000003d10000000000003d20"), 0x050541), + (hex!("0100000000333333334444444455000003d10000000000005e50"), 0x0505e1), + (hex!("0100000000333333334444444455000003d10000000000007880"), 0x050681), + (hex!("0100000000333333334444444455000003d20000000000003d30"), 0x050721), + (hex!("0100000000333333334444444455000003d20000000000005d00"), 0x0507c1), + (hex!("0100000000333333334444444455000003d30000000000003d40"), 0x050861), + (hex!("0100000000333333334444444455000003d30000000000005d40"), 0x050901), + (hex!("0100000000333333334444444455000003d300000000000063f0"), 0x0509a1), + (hex!("0100000000333333334444444455000003d40000000000003d50"), 0x050a41), + (hex!("0100000000333333334444444455000003d40000000000005700"), 0x050ae1), + (hex!("0100000000333333334444444455000003d400000000000078f0"), 0x050b81), + (hex!("0100000000333333334444444455000003d50000000000003d60"), 0x050c21), + (hex!("0100000000333333334444444455000003d60000000000003d70"), 0x050cc1), + (hex!("0100000000333333334444444455000003d70000000000003d80"), 0x050d61), + (hex!("0100000000333333334444444455000003d80000000000003d90"), 0x050e01), + (hex!("0100000000333333334444444455000003d80000000000006690"), 0x050ea1), + (hex!("0100000000333333334444444455000003d90000000000003da0"), 0x050f41), + (hex!("0100000000333333334444444455000003d900000000000076d0"), 0x050fe1), + (hex!("0100000000333333334444444455000003da0000000000003db0"), 0x051081), + (hex!("0100000000333333334444444455000003db0000000000003dc0"), 0x051121), + (hex!("0100000000333333334444444455000003db0000000000004a30"), 0x0511c1), + (hex!("0100000000333333334444444455000003db0000000000005390"), 0x051261), + (hex!("0100000000333333334444444455000003dc0000000000003dd0"), 0x051301), + (hex!("0100000000333333334444444455000003dc0000000000006d60"), 0x0513a1), + (hex!("0100000000333333334444444455000003dd0000000000003de0"), 0x051441), + (hex!("0100000000333333334444444455000003de0000000000003df0"), 0x0514e1), + (hex!("0100000000333333334444444455000003df0000000000003e00"), 0x051581), + (hex!("0100000000333333334444444455000003df0000000000005240"), 0x051621), + (hex!("0100000000333333334444444455000003df0000000000005610"), 0x0516c1), + (hex!("0100000000333333334444444455000003e00000000000003e10"), 0x051761), + (hex!("0100000000333333334444444455000003e00000000000006500"), 0x051801), + (hex!("0100000000333333334444444455000003e10000000000003e20"), 0x0518a1), + (hex!("0100000000333333334444444455000003e10000000000006a10"), 0x051941), + (hex!("0100000000333333334444444455000003e10000000000007c10"), 0x0519e1), + (hex!("0100000000333333334444444455000003e20000000000003e30"), 0x051a81), + (hex!("0100000000333333334444444455000003e20000000000006310"), 0x051b21), + (hex!("0100000000333333334444444455000003e30000000000003e40"), 0x051bc1), + (hex!("0100000000333333334444444455000003e40000000000003e50"), 0x051c61), + (hex!("0100000000333333334444444455000003e40000000000006780"), 0x051d01), + (hex!("0100000000333333334444444455000003e40000000000007ce0"), 0x051da1), + (hex!("0100000000333333334444444455000003e50000000000003e60"), 0x051e41), + (hex!("0100000000333333334444444455000003e60000000000003e70"), 0x051ee1), + (hex!("0100000000333333334444444455000003e60000000000005040"), 0x051f81), + (hex!("0100000000333333334444444455000003e60000000000005bf0"), 0x052021), + (hex!("0100000000333333334444444455000003e70000000000003e80"), 0x0520c1), + (hex!("0100000000333333334444444455000003e70000000000003f50"), 0x052161), ]; diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 75ffe09696..38fd426746 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -406,4 +406,123 @@ mod tests { METADATA_OLD_FORMAT_VERSION, METADATA_FORMAT_VERSION ); } + + #[test] + fn test_metadata_bincode_serde() { + let original_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + // Any version will do here, so use the default + crate::DEFAULT_PG_VERSION, + ); + let metadata_bytes = original_metadata + .to_bytes() + .expect("Cannot create bytes array from metadata"); + + let metadata_bincode_be_bytes = original_metadata + .ser() + .expect("Cannot serialize the metadata"); + + // 8 bytes for the length of the vector + assert_eq!(metadata_bincode_be_bytes.len(), 8 + metadata_bytes.len()); + + let expected_bincode_bytes = { + let mut temp = vec![]; + let len_bytes = metadata_bytes.len().to_be_bytes(); + temp.extend_from_slice(&len_bytes); + temp.extend_from_slice(&metadata_bytes); + temp + }; + assert_eq!(metadata_bincode_be_bytes, expected_bincode_bytes); + + let deserialized_metadata = TimelineMetadata::des(&metadata_bincode_be_bytes).unwrap(); + // Deserialized metadata has the metadata header, which is different from the serialized one. + // Reference: TimelineMetaData::to_bytes() + let expected_metadata = { + let mut temp_metadata = original_metadata; + let body_bytes = temp_metadata + .body + .ser() + .expect("Cannot serialize the metadata body"); + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_FORMAT_VERSION, + checksum: crc32c::crc32c(&body_bytes), + }; + temp_metadata.hdr = hdr; + temp_metadata + }; + assert_eq!(deserialized_metadata, expected_metadata); + } + + #[test] + fn test_metadata_bincode_serde_ensure_roundtrip() { + let original_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + // Any version will do here, so use the default + crate::DEFAULT_PG_VERSION, + ); + let expected_bytes = vec![ + /* bincode length encoding bytes */ + 0, 0, 0, 0, 0, 0, 2, 0, // 8 bytes for the length of the serialized vector + /* TimelineMetadataHeader */ + 4, 37, 101, 34, 0, 70, 0, 4, // checksum, size, format_version (4 + 2 + 2) + /* TimelineMetadataBodyV2 */ + 0, 0, 0, 0, 0, 0, 2, 0, // disk_consistent_lsn (8 bytes) + 1, 0, 0, 0, 0, 0, 0, 1, 0, // prev_record_lsn (9 bytes) + 1, 17, 34, 51, 68, 85, 102, 119, 136, 17, 34, 51, 68, 85, 102, 119, + 136, // ancestor_timeline (17 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // ancestor_lsn (8 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // latest_gc_cutoff_lsn (8 bytes) + 0, 0, 0, 0, 0, 0, 0, 0, // initdb_lsn (8 bytes) + 0, 0, 0, 15, // pg_version (4 bytes) + /* padding bytes */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + ]; + let metadata_ser_bytes = original_metadata.ser().unwrap(); + assert_eq!(metadata_ser_bytes, expected_bytes); + + let expected_metadata = { + let mut temp_metadata = original_metadata; + let body_bytes = temp_metadata + .body + .ser() + .expect("Cannot serialize the metadata body"); + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_FORMAT_VERSION, + checksum: crc32c::crc32c(&body_bytes), + }; + temp_metadata.hdr = hdr; + temp_metadata + }; + let des_metadata = TimelineMetadata::des(&metadata_ser_bytes).unwrap(); + assert_eq!(des_metadata, expected_metadata); + } } diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 33fdc76f8d..a766cca0c5 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -2,14 +2,18 @@ //! page server. use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; +use pageserver_api::shard::TenantShardId; use rand::{distributions::Alphanumeric, Rng}; -use std::collections::{hash_map, HashMap}; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap}; +use std::ops::Deref; use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::fs; +use utils::timeout::{timeout_cancellable, TimeoutCancellableError}; use anyhow::Context; use once_cell::sync::Lazy; -use tokio::sync::RwLock; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::*; @@ -23,9 +27,11 @@ use crate::control_plane_client::{ ControlPlaneClient, ControlPlaneGenerationsApi, RetryForeverError, }; use crate::deletion_queue::DeletionQueueClient; +use crate::metrics::TENANT_MANAGER as METRICS; use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; use crate::tenant::delete::DeleteTenantFlow; +use crate::tenant::span::debug_assert_current_span_has_tenant_id; use crate::tenant::{create_tenant_files, AttachedTenantConf, SpawnMode, Tenant, TenantState}; use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME, TEMP_FILE_SUFFIX}; @@ -47,10 +53,22 @@ use super::TenantSharedResources; /// that way we avoid having to carefully switch a tenant's ingestion etc on and off during /// its lifetime, and we can preserve some important safety invariants like `Tenant` always /// having a properly acquired generation (Secondary doesn't need a generation) -#[derive(Clone)] pub(crate) enum TenantSlot { Attached(Arc), Secondary, + /// In this state, other administrative operations acting on the TenantId should + /// block, or return a retry indicator equivalent to HTTP 503. + InProgress(utils::completion::Barrier), +} + +impl std::fmt::Debug for TenantSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Attached(tenant) => write!(f, "Attached({})", tenant.current_state()), + Self::Secondary => write!(f, "Secondary"), + Self::InProgress(_) => write!(f, "InProgress"), + } + } } impl TenantSlot { @@ -59,14 +77,7 @@ impl TenantSlot { match self { Self::Attached(t) => Some(t), Self::Secondary => None, - } - } - - /// Consume self and return the `Tenant` that was in this slot if attached, else None - fn into_attached(self) -> Option> { - match self { - Self::Attached(t) => Some(t), - Self::Secondary => None, + Self::InProgress(_) => None, } } } @@ -77,11 +88,38 @@ pub(crate) enum TenantsMap { /// [`init_tenant_mgr`] is not done yet. Initializing, /// [`init_tenant_mgr`] is done, all on-disk tenants have been loaded. - /// New tenants can be added using [`tenant_map_insert`]. - Open(HashMap), + /// New tenants can be added using [`tenant_map_acquire_slot`]. + Open(BTreeMap), /// The pageserver has entered shutdown mode via [`shutdown_all_tenants`]. /// Existing tenants are still accessible, but no new tenants can be created. - ShuttingDown(HashMap), + ShuttingDown(BTreeMap), +} + +/// Helper for mapping shard-unaware functions to a sharding-aware map +/// TODO(sharding): all users of this must be made shard-aware. +fn exactly_one_or_none<'a>( + map: &'a BTreeMap, + tenant_id: &TenantId, +) -> Option<(&'a TenantShardId, &'a TenantSlot)> { + let mut slots = map.range(TenantShardId::tenant_range(*tenant_id)); + + // Retrieve the first two slots in the range: if both are populated, we must panic because the caller + // needs a shard-naive view of the world in which only one slot can exist for a TenantId at a time. + let slot_a = slots.next(); + let slot_b = slots.next(); + match (slot_a, slot_b) { + (None, None) => None, + (Some(slot), None) => { + // Exactly one matching slot + Some(slot) + } + (Some(_slot_a), Some(_slot_b)) => { + // Multiple shards for this tenant: cannot handle this yet. + // TODO(sharding): callers of get() should be shard-aware. + todo!("Attaching multiple shards in teh same tenant to the same pageserver") + } + (None, Some(_)) => unreachable!(), + } } impl TenantsMap { @@ -92,26 +130,28 @@ impl TenantsMap { match self { TenantsMap::Initializing => None, TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => { - m.get(tenant_id).and_then(TenantSlot::get_attached) + // TODO(sharding): callers of get() should be shard-aware. + exactly_one_or_none(m, tenant_id).and_then(|(_, slot)| slot.get_attached()) } } } - /// Get the contents of the map at this tenant ID, even if it is in secondary state. - pub(crate) fn get_slot(&self, tenant_id: &TenantId) -> Option<&TenantSlot> { - match self { - TenantsMap::Initializing => None, - TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.get(tenant_id), - } - } - pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option> { + pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option { match self { TenantsMap::Initializing => None, TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => { - m.remove(tenant_id).and_then(TenantSlot::into_attached) + let key = exactly_one_or_none(m, tenant_id).map(|(k, _)| *k); + key.and_then(|key| m.remove(&key)) } } } + + pub(crate) fn len(&self) -> usize { + match self { + TenantsMap::Initializing => 0, + TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m.len(), + } + } } /// This is "safe" in that that it won't leave behind a partially deleted directory @@ -147,7 +187,8 @@ async fn safe_rename_tenant_dir(path: impl AsRef) -> std::io::Result> = Lazy::new(|| RwLock::new(TenantsMap::Initializing)); +static TENANTS: Lazy> = + Lazy::new(|| std::sync::RwLock::new(TenantsMap::Initializing)); /// Create a directory, including parents. This does no fsyncs and makes /// no guarantees about the persistence of the resulting metadata: for @@ -192,6 +233,22 @@ async fn unsafe_create_dir_all(path: &Utf8PathBuf) -> std::io::Result<()> { Ok(()) } +/// The TenantManager is responsible for storing and mutating the collection of all tenants +/// that this pageserver process has state for. Every Tenant and SecondaryTenant instance +/// lives inside the TenantManager. +/// +/// The most important role of the TenantManager is to prevent conflicts: e.g. trying to attach +/// the same tenant twice concurrently, or trying to configure the same tenant into secondary +/// and attached modes concurrently. +pub struct TenantManager { + conf: &'static PageServerConf, + // TODO: currently this is a &'static pointing to TENANTs. When we finish refactoring + // out of that static variable, the TenantManager can own this. + // See https://github.com/neondatabase/neon/issues/5796 + tenants: &'static std::sync::RwLock, + resources: TenantSharedResources, +} + fn emergency_generations( tenant_confs: &HashMap>, ) -> HashMap { @@ -358,8 +415,8 @@ pub async fn init_tenant_mgr( resources: TenantSharedResources, init_order: InitializationOrder, cancel: CancellationToken, -) -> anyhow::Result<()> { - let mut tenants = HashMap::new(); +) -> anyhow::Result { + let mut tenants = BTreeMap::new(); let ctx = RequestContext::todo_child(TaskKind::Startup, DownloadBehavior::Warn); @@ -380,7 +437,7 @@ pub async fn init_tenant_mgr( warn!(%tenant_id, "Marking tenant broken, failed to {e:#}"); tenants.insert( - tenant_id, + TenantShardId::unsharded(tenant_id), TenantSlot::Attached(Tenant::create_broken_tenant( conf, tenant_id, @@ -403,7 +460,7 @@ pub async fn init_tenant_mgr( // tenants, because they do no remote writes and hence require no // generation number info!(%tenant_id, "Loaded tenant in secondary mode"); - tenants.insert(tenant_id, TenantSlot::Secondary); + tenants.insert(TenantShardId::unsharded(tenant_id), TenantSlot::Secondary); } LocationMode::Attached(_) => { // TODO: augment re-attach API to enable the control plane to @@ -446,7 +503,10 @@ pub async fn init_tenant_mgr( &ctx, ) { Ok(tenant) => { - tenants.insert(tenant.tenant_id(), TenantSlot::Attached(tenant)); + tenants.insert( + TenantShardId::unsharded(tenant.tenant_id()), + TenantSlot::Attached(tenant), + ); } Err(e) => { error!(%tenant_id, "Failed to start tenant: {e:#}"); @@ -456,10 +516,16 @@ pub async fn init_tenant_mgr( info!("Processed {} local tenants at startup", tenants.len()); - let mut tenants_map = TENANTS.write().await; + let mut tenants_map = TENANTS.write().unwrap(); assert!(matches!(&*tenants_map, &TenantsMap::Initializing)); + METRICS.tenant_slots.set(tenants.len() as u64); *tenants_map = TenantsMap::Open(tenants); - Ok(()) + + Ok(TenantManager { + conf, + tenants: &TENANTS, + resources, + }) } /// Wrapper for Tenant::spawn that checks invariants before running, and inserts @@ -472,7 +538,7 @@ pub(crate) fn tenant_spawn( resources: TenantSharedResources, location_conf: AttachedTenantConf, init_order: Option, - tenants: &'static tokio::sync::RwLock, + tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { @@ -533,26 +599,70 @@ pub(crate) async fn shutdown_all_tenants() { shutdown_all_tenants0(&TENANTS).await } -async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { +async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { use utils::completion; - // Prevent new tenants from being created. - let tenants_to_shut_down = { - let mut m = tenants.write().await; + let mut join_set = JoinSet::new(); + + // Atomically, 1. create the shutdown tasks and 2. prevent creation of new tenants. + let (total_in_progress, total_attached) = { + let mut m = tenants.write().unwrap(); match &mut *m { TenantsMap::Initializing => { - *m = TenantsMap::ShuttingDown(HashMap::default()); + *m = TenantsMap::ShuttingDown(BTreeMap::default()); info!("tenants map is empty"); return; } TenantsMap::Open(tenants) => { - let tenants_clone = tenants.clone(); - *m = TenantsMap::ShuttingDown(std::mem::take(tenants)); - tenants_clone + let mut shutdown_state = BTreeMap::new(); + let mut total_in_progress = 0; + let mut total_attached = 0; + + for (tenant_shard_id, v) in std::mem::take(tenants).into_iter() { + match v { + TenantSlot::Attached(t) => { + shutdown_state.insert(tenant_shard_id, TenantSlot::Attached(t.clone())); + join_set.spawn( + async move { + let freeze_and_flush = true; + + let res = { + let (_guard, shutdown_progress) = completion::channel(); + t.shutdown(shutdown_progress, freeze_and_flush).await + }; + + if let Err(other_progress) = res { + // join the another shutdown in progress + other_progress.wait().await; + } + + // we cannot afford per tenant logging here, because if s3 is degraded, we are + // going to log too many lines + debug!("tenant successfully stopped"); + } + .instrument(info_span!("shutdown", tenant_id=%tenant_shard_id.tenant_id, shard=%tenant_shard_id.shard_slug())), + ); + + total_attached += 1; + } + TenantSlot::Secondary => { + shutdown_state.insert(tenant_shard_id, TenantSlot::Secondary); + } + TenantSlot::InProgress(notify) => { + // InProgress tenants are not visible in TenantsMap::ShuttingDown: we will + // wait for their notifications to fire in this function. + join_set.spawn(async move { + notify.wait().await; + }); + + total_in_progress += 1; + } + } + } + *m = TenantsMap::ShuttingDown(shutdown_state); + (total_in_progress, total_attached) } TenantsMap::ShuttingDown(_) => { - // TODO: it is possible that detach and shutdown happen at the same time. as a - // result, during shutdown we do not wait for detach. error!("already shutting down, this function isn't supposed to be called more than once"); return; } @@ -560,39 +670,11 @@ async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { }; let started_at = std::time::Instant::now(); - let mut join_set = JoinSet::new(); - for (tenant_id, tenant) in tenants_to_shut_down { - join_set.spawn( - async move { - let freeze_and_flush = true; - let res = { - let (_guard, shutdown_progress) = completion::channel(); - match tenant { - TenantSlot::Attached(t) => { - t.shutdown(shutdown_progress, freeze_and_flush).await - } - TenantSlot::Secondary => { - // TODO: once secondary mode downloads are implemented, - // ensure they have all stopped before we reach this point. - Ok(()) - } - } - }; - - if let Err(other_progress) = res { - // join the another shutdown in progress - other_progress.wait().await; - } - - // we cannot afford per tenant logging here, because if s3 is degraded, we are - // going to log too many lines - - debug!("tenant successfully stopped"); - } - .instrument(info_span!("shutdown", %tenant_id)), - ); - } + info!( + "Waiting for {} InProgress tenants and {} Attached tenants to shut down", + total_in_progress, total_attached + ); let total = join_set.len(); let mut panicked = 0; @@ -606,7 +688,7 @@ async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { match joined { Ok(()) => {} Err(join_error) if join_error.is_cancelled() => { - unreachable!("we are not cancelling any of the futures"); + unreachable!("we are not cancelling any of the tasks"); } Err(join_error) if join_error.is_panic() => { // cannot really do anything, as this panic is likely a bug @@ -644,47 +726,39 @@ async fn shutdown_all_tenants0(tenants: &tokio::sync::RwLock) { pub(crate) async fn create_tenant( conf: &'static PageServerConf, tenant_conf: TenantConfOpt, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, generation: Generation, resources: TenantSharedResources, ctx: &RequestContext, ) -> Result, TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let location_conf = LocationConf::attached_single(tenant_conf, generation); + let location_conf = LocationConf::attached_single(tenant_conf, generation); - // We're holding the tenants lock in write mode while doing local IO. - // If this section ever becomes contentious, introduce a new `TenantState::Creating` - // and do the work in that state. - super::create_tenant_files(conf, &location_conf, &tenant_id).await?; + let slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; + // TODO(sharding): make local paths shard-aware + let tenant_path = + super::create_tenant_files(conf, &location_conf, &tenant_shard_id.tenant_id).await?; - // TODO: tenant directory remains on disk if we bail out from here on. - // See https://github.com/neondatabase/neon/issues/4233 + let created_tenant = tenant_spawn( + conf, + tenant_shard_id.tenant_id, + &tenant_path, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Create, + ctx, + )?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 - let tenant_path = conf.tenant_path(&tenant_id); + let created_tenant_id = created_tenant.tenant_id(); + debug_assert_eq!(created_tenant_id, tenant_shard_id.tenant_id); - let created_tenant = tenant_spawn( - conf, - tenant_id, - &tenant_path, - resources, - AttachedTenantConf::try_from(location_conf)?, - None, - &TENANTS, - SpawnMode::Create, - ctx, - )?; - // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. - // See https://github.com/neondatabase/neon/issues/4233 + slot_guard.upsert(TenantSlot::Attached(created_tenant.clone()))?; - let crated_tenant_id = created_tenant.tenant_id(); - anyhow::ensure!( - tenant_id == crated_tenant_id, - "loaded created tenant has unexpected tenant id \ - (expect {tenant_id} != actual {crated_tenant_id})", - ); - Ok(created_tenant) - }) - .await + Ok(created_tenant) } #[derive(Debug, thiserror::Error)] @@ -701,7 +775,7 @@ pub(crate) async fn set_new_tenant_config( tenant_id: TenantId, ) -> Result<(), SetNewTenantConfigError> { info!("configuring tenant {tenant_id}"); - let tenant = get_tenant(tenant_id, true).await?; + let tenant = get_tenant(tenant_id, true)?; // This is a legacy API that only operates on attached tenants: the preferred // API to use is the location_config/ endpoint, which lets the caller provide @@ -715,142 +789,195 @@ pub(crate) async fn set_new_tenant_config( Ok(()) } -#[instrument(skip_all, fields(%tenant_id))] -pub(crate) async fn upsert_location( - conf: &'static PageServerConf, - tenant_id: TenantId, - new_location_config: LocationConf, - broker_client: storage_broker::BrokerClientChannel, - remote_storage: Option, - deletion_queue_client: DeletionQueueClient, - ctx: &RequestContext, -) -> Result<(), anyhow::Error> { - info!("configuring tenant location {tenant_id} to state {new_location_config:?}"); +impl TenantManager { + /// Gets the attached tenant from the in-memory data, erroring if it's absent, in secondary mode, or is not fitting to the query. + /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. + /// + /// This method is cancel-safe. + pub(crate) fn get_attached_tenant_shard( + &self, + tenant_shard_id: TenantShardId, + active_only: bool, + ) -> Result, GetTenantError> { + let locked = self.tenants.read().unwrap(); - let mut existing_tenant = match get_tenant(tenant_id, false).await { - Ok(t) => Some(t), - Err(GetTenantError::NotFound(_)) => None, - Err(e) => anyhow::bail!(e), - }; + let peek_slot = tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read)?; - // If we need to shut down a Tenant, do that first - let shutdown_tenant = match (&new_location_config.mode, &existing_tenant) { - (LocationMode::Secondary(_), Some(t)) => Some(t), - (LocationMode::Attached(attach_conf), Some(t)) => { - if attach_conf.generation != t.generation { - Some(t) - } else { - None + match peek_slot { + Some(TenantSlot::Attached(tenant)) => match tenant.current_state() { + TenantState::Broken { + reason, + backtrace: _, + } if active_only => Err(GetTenantError::Broken(reason)), + TenantState::Active => Ok(Arc::clone(tenant)), + _ => { + if active_only { + Err(GetTenantError::NotActive(tenant_shard_id.tenant_id)) + } else { + Ok(Arc::clone(tenant)) + } + } + }, + Some(TenantSlot::InProgress(_)) => { + Err(GetTenantError::NotActive(tenant_shard_id.tenant_id)) + } + None | Some(TenantSlot::Secondary) => { + Err(GetTenantError::NotFound(tenant_shard_id.tenant_id)) } } - _ => None, - }; - - // TODO: currently we risk concurrent operations interfering with the tenant - // while we await shutdown, but we also should not hold the TenantsMap lock - // across the whole operation. Before we start using this function in production, - // a follow-on change will revise how concurrency is handled in TenantsMap. - // (https://github.com/neondatabase/neon/issues/5378) - - if let Some(tenant) = shutdown_tenant { - let (_guard, progress) = utils::completion::channel(); - - match tenant.get_attach_mode() { - AttachmentMode::Single | AttachmentMode::Multi => { - // Before we leave our state as the presumed holder of the latest generation, - // flush any outstanding deletions to reduce the risk of leaking objects. - deletion_queue_client.flush_advisory() - } - AttachmentMode::Stale => { - // If we're stale there's not point trying to flush deletions - } - }; - - info!("Shutting down attached tenant"); - match tenant.shutdown(progress, false).await { - Ok(()) => {} - Err(barrier) => { - info!("Shutdown already in progress, waiting for it to complete"); - barrier.wait().await; - } - } - existing_tenant = None; } - if let Some(tenant) = existing_tenant { - // Update the existing tenant - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; - tenant.set_new_location_config(AttachedTenantConf::try_from(new_location_config)?); - } else { - // Upsert a fresh TenantSlot into TenantsMap. Do it within the map write lock, - // and re-check that the state of anything we are replacing is as expected. - tenant_map_upsert_slot(tenant_id, |old_value| async move { - if let Some(TenantSlot::Attached(t)) = old_value { - if !matches!(t.current_state(), TenantState::Stopping { .. }) { - anyhow::bail!("Tenant state changed during location configuration update"); + pub(crate) async fn delete_timeline( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + _ctx: &RequestContext, + ) -> Result<(), DeleteTimelineError> { + let tenant = self.get_attached_tenant_shard(tenant_shard_id, true)?; + DeleteTimelineFlow::run(&tenant, timeline_id, false).await?; + Ok(()) + } + + pub(crate) async fn upsert_location( + &self, + tenant_shard_id: TenantShardId, + new_location_config: LocationConf, + ctx: &RequestContext, + ) -> Result<(), anyhow::Error> { + debug_assert_current_span_has_tenant_id(); + info!("configuring tenant location to state {new_location_config:?}"); + + // Special case fast-path for updates to Tenant: if our upsert is only updating configuration, + // then we do not need to set the slot to InProgress, we can just call into the + // existng tenant. + { + let locked = self.tenants.read().unwrap(); + let peek_slot = + tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Write)?; + match (&new_location_config.mode, peek_slot) { + (LocationMode::Attached(attach_conf), Some(TenantSlot::Attached(tenant))) => { + if attach_conf.generation == tenant.generation { + // A transition from Attached to Attached in the same generation, we may + // take our fast path and just provide the updated configuration + // to the tenant. + tenant.set_new_location_config(AttachedTenantConf::try_from( + new_location_config, + )?); + + // Persist the new config in the background, to avoid holding up any + // locks while we do so. + // TODO + + return Ok(()); + } else { + // Different generations, fall through to general case + } + } + _ => { + // Not an Attached->Attached transition, fall through to general case } } + } - let tenant_path = conf.tenant_path(&tenant_id); + // General case for upserts to TenantsMap, excluding the case above: we will substitute an + // InProgress value to the slot while we make whatever changes are required. The state for + // the tenant is inaccessible to the outside world while we are doing this, but that is sensible: + // the state is ill-defined while we're in transition. Transitions are async, but fast: we do + // not do significant I/O, and shutdowns should be prompt via cancellation tokens. + let mut slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::Any)?; - let new_slot = match &new_location_config.mode { - LocationMode::Secondary(_) => { - // Directory doesn't need to be fsync'd because if we crash it can - // safely be recreated next time this tenant location is configured. - unsafe_create_dir_all(&tenant_path) - .await - .with_context(|| format!("Creating {tenant_path}"))?; + if let Some(TenantSlot::Attached(tenant)) = slot_guard.get_old_value() { + // The case where we keep a Tenant alive was covered above in the special case + // for Attached->Attached transitions in the same generation. By this point, + // if we see an attached tenant we know it will be discarded and should be + // shut down. + let (_guard, progress) = utils::completion::channel(); - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; - - TenantSlot::Secondary + match tenant.get_attach_mode() { + AttachmentMode::Single | AttachmentMode::Multi => { + // Before we leave our state as the presumed holder of the latest generation, + // flush any outstanding deletions to reduce the risk of leaking objects. + self.resources.deletion_queue_client.flush_advisory() } - LocationMode::Attached(_attach_config) => { - // FIXME: should avoid doing this disk I/O inside the TenantsMap lock, - // we have the same problem in load_tenant/attach_tenant. Probably - // need a lock in TenantSlot to fix this. - let timelines_path = conf.timelines_path(&tenant_id); - - // Directory doesn't need to be fsync'd because we do not depend on - // it to exist after crashes: it may be recreated when tenant is - // re-attached, see https://github.com/neondatabase/neon/issues/5550 - unsafe_create_dir_all(&timelines_path) - .await - .with_context(|| format!("Creating {timelines_path}"))?; - - Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) - .await - .map_err(SetNewTenantConfigError::Persist)?; - - let tenant = tenant_spawn( - conf, - tenant_id, - &tenant_path, - TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client, - }, - AttachedTenantConf::try_from(new_location_config)?, - None, - &TENANTS, - SpawnMode::Normal, - ctx, - )?; - - TenantSlot::Attached(tenant) + AttachmentMode::Stale => { + // If we're stale there's not point trying to flush deletions } }; - Ok(new_slot) - }) - .await?; + info!("Shutting down attached tenant"); + match tenant.shutdown(progress, false).await { + Ok(()) => {} + Err(barrier) => { + info!("Shutdown already in progress, waiting for it to complete"); + barrier.wait().await; + } + } + slot_guard.drop_old_value().expect("We just shut it down"); + } + + // TODO(sharding): make local paths sharding-aware + let tenant_path = self.conf.tenant_path(&tenant_shard_id.tenant_id); + + let new_slot = match &new_location_config.mode { + LocationMode::Secondary(_) => { + // Directory doesn't need to be fsync'd because if we crash it can + // safely be recreated next time this tenant location is configured. + unsafe_create_dir_all(&tenant_path) + .await + .with_context(|| format!("Creating {tenant_path}"))?; + + // TODO(sharding): make local paths sharding-aware + Tenant::persist_tenant_config( + self.conf, + &tenant_shard_id.tenant_id, + &new_location_config, + ) + .await + .map_err(SetNewTenantConfigError::Persist)?; + + TenantSlot::Secondary + } + LocationMode::Attached(_attach_config) => { + // TODO(sharding): make local paths sharding-aware + let timelines_path = self.conf.timelines_path(&tenant_shard_id.tenant_id); + + // Directory doesn't need to be fsync'd because we do not depend on + // it to exist after crashes: it may be recreated when tenant is + // re-attached, see https://github.com/neondatabase/neon/issues/5550 + unsafe_create_dir_all(&timelines_path) + .await + .with_context(|| format!("Creating {timelines_path}"))?; + + // TODO(sharding): make local paths sharding-aware + Tenant::persist_tenant_config( + self.conf, + &tenant_shard_id.tenant_id, + &new_location_config, + ) + .await + .map_err(SetNewTenantConfigError::Persist)?; + + // TODO(sharding): make spawn sharding-aware + let tenant = tenant_spawn( + self.conf, + tenant_shard_id.tenant_id, + &tenant_path, + self.resources.clone(), + AttachedTenantConf::try_from(new_location_config)?, + None, + self.tenants, + SpawnMode::Normal, + ctx, + )?; + + TenantSlot::Attached(tenant) + } + }; + + slot_guard.upsert(new_slot)?; + + Ok(()) } - Ok(()) } #[derive(Debug, thiserror::Error)] @@ -864,43 +991,213 @@ pub(crate) enum GetTenantError { /// is a stuck error state #[error("Tenant is broken: {0}")] Broken(String), + + // Initializing or shutting down: cannot authoritatively say whether we have this tenant + #[error("Tenant map is not available: {0}")] + MapState(#[from] TenantMapError), } /// Gets the tenant from the in-memory data, erroring if it's absent or is not fitting to the query. /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. /// /// This method is cancel-safe. -pub(crate) async fn get_tenant( +pub(crate) fn get_tenant( tenant_id: TenantId, active_only: bool, ) -> Result, GetTenantError> { - let m = TENANTS.read().await; - let tenant = m - .get(&tenant_id) - .ok_or(GetTenantError::NotFound(tenant_id))?; + let locked = TENANTS.read().unwrap(); - match tenant.current_state() { - TenantState::Broken { - reason, - backtrace: _, - } if active_only => Err(GetTenantError::Broken(reason)), - TenantState::Active => Ok(Arc::clone(tenant)), - _ => { - if active_only { - Err(GetTenantError::NotActive(tenant_id)) - } else { - Ok(Arc::clone(tenant)) + // TODO(sharding): make all callers of get_tenant shard-aware + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + let peek_slot = tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read)?; + + match peek_slot { + Some(TenantSlot::Attached(tenant)) => match tenant.current_state() { + TenantState::Broken { + reason, + backtrace: _, + } if active_only => Err(GetTenantError::Broken(reason)), + TenantState::Active => Ok(Arc::clone(tenant)), + _ => { + if active_only { + Err(GetTenantError::NotActive(tenant_id)) + } else { + Ok(Arc::clone(tenant)) + } + } + }, + Some(TenantSlot::InProgress(_)) => Err(GetTenantError::NotActive(tenant_id)), + None | Some(TenantSlot::Secondary) => Err(GetTenantError::NotFound(tenant_id)), + } +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum GetActiveTenantError { + /// We may time out either while TenantSlot is InProgress, or while the Tenant + /// is in a non-Active state + #[error( + "Timed out waiting {wait_time:?} for tenant active state. Latest state: {latest_state:?}" + )] + WaitForActiveTimeout { + latest_state: Option, + wait_time: Duration, + }, + + /// The TenantSlot is absent, or in secondary mode + #[error(transparent)] + NotFound(#[from] GetTenantError), + + /// Cancellation token fired while we were waiting + #[error("cancelled")] + Cancelled, + + /// Tenant exists, but is in a state that cannot become active (e.g. Stopping, Broken) + #[error("will not become active. Current state: {0}")] + WillNotBecomeActive(TenantState), +} + +/// Get a [`Tenant`] in its active state. If the tenant_id is currently in [`TenantSlot::InProgress`] +/// state, then wait for up to `timeout`. If the [`Tenant`] is not currently in [`TenantState::Active`], +/// then wait for up to `timeout` (minus however long we waited for the slot). +pub(crate) async fn get_active_tenant_with_timeout( + tenant_id: TenantId, + timeout: Duration, + cancel: &CancellationToken, +) -> Result, GetActiveTenantError> { + enum WaitFor { + Barrier(utils::completion::Barrier), + Tenant(Arc), + } + + // TODO(sharding): make page service interface sharding-aware (page service should apply ShardIdentity to the key + // to decide which shard services the request) + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + let wait_start = Instant::now(); + let deadline = wait_start + timeout; + + let wait_for = { + let locked = TENANTS.read().unwrap(); + let peek_slot = tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read) + .map_err(GetTenantError::MapState)?; + match peek_slot { + Some(TenantSlot::Attached(tenant)) => { + match tenant.current_state() { + TenantState::Active => { + // Fast path: we don't need to do any async waiting. + return Ok(tenant.clone()); + } + _ => WaitFor::Tenant(tenant.clone()), + } + } + Some(TenantSlot::Secondary) => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotActive( + tenant_id, + ))) + } + Some(TenantSlot::InProgress(barrier)) => WaitFor::Barrier(barrier.clone()), + None => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotFound( + tenant_id, + ))) } } + }; + + let tenant = match wait_for { + WaitFor::Barrier(barrier) => { + tracing::debug!("Waiting for tenant InProgress state to pass..."); + timeout_cancellable( + deadline.duration_since(Instant::now()), + cancel, + barrier.wait(), + ) + .await + .map_err(|e| match e { + TimeoutCancellableError::Timeout => GetActiveTenantError::WaitForActiveTimeout { + latest_state: None, + wait_time: wait_start.elapsed(), + }, + TimeoutCancellableError::Cancelled => GetActiveTenantError::Cancelled, + })?; + { + let locked = TENANTS.read().unwrap(); + let peek_slot = + tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Read) + .map_err(GetTenantError::MapState)?; + match peek_slot { + Some(TenantSlot::Attached(tenant)) => tenant.clone(), + _ => { + return Err(GetActiveTenantError::NotFound(GetTenantError::NotActive( + tenant_id, + ))) + } + } + } + } + WaitFor::Tenant(tenant) => tenant, + }; + + tracing::debug!("Waiting for tenant to enter active state..."); + match timeout_cancellable( + deadline.duration_since(Instant::now()), + cancel, + tenant.wait_to_become_active(), + ) + .await + { + Ok(Ok(())) => Ok(tenant), + Ok(Err(e)) => Err(e), + Err(TimeoutCancellableError::Timeout) => { + let latest_state = tenant.current_state(); + if latest_state == TenantState::Active { + Ok(tenant) + } else { + Err(GetActiveTenantError::WaitForActiveTimeout { + latest_state: Some(latest_state), + wait_time: timeout, + }) + } + } + Err(TimeoutCancellableError::Cancelled) => Err(GetActiveTenantError::Cancelled), } } pub(crate) async fn delete_tenant( conf: &'static PageServerConf, remote_storage: Option, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, ) -> Result<(), DeleteTenantError> { - DeleteTenantFlow::run(conf, remote_storage, &TENANTS, tenant_id).await + // We acquire a SlotGuard during this function to protect against concurrent + // changes while the ::prepare phase of DeleteTenantFlow executes, but then + // have to return the Tenant to the map while the background deletion runs. + // + // TODO: refactor deletion to happen outside the lifetime of a Tenant. + // Currently, deletion requires a reference to the tenants map in order to + // keep the Tenant in the map until deletion is complete, and then remove + // it at the end. + // + // See https://github.com/neondatabase/neon/issues/5080 + + // TODO(sharding): make delete API sharding-aware + let mut slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustExist)?; + + // unwrap is safe because we used MustExist mode when acquiring + let tenant = match slot_guard.get_old_value().as_ref().unwrap() { + TenantSlot::Attached(tenant) => tenant.clone(), + _ => { + // Express "not attached" as equivalent to "not found" + return Err(DeleteTenantError::NotAttached); + } + }; + + let result = DeleteTenantFlow::run(conf, remote_storage, &TENANTS, tenant).await; + + // The Tenant goes back into the map in Stopping state, it will eventually be removed by DeleteTenantFLow + slot_guard.revert(); + result } #[derive(Debug, thiserror::Error)] @@ -912,36 +1209,28 @@ pub(crate) enum DeleteTimelineError { Timeline(#[from] crate::tenant::DeleteTimelineError), } -pub(crate) async fn delete_timeline( - tenant_id: TenantId, - timeline_id: TimelineId, - _ctx: &RequestContext, -) -> Result<(), DeleteTimelineError> { - let tenant = get_tenant(tenant_id, true).await?; - DeleteTimelineFlow::run(&tenant, timeline_id, false).await?; - Ok(()) -} - #[derive(Debug, thiserror::Error)] pub(crate) enum TenantStateError { - #[error("Tenant {0} not found")] - NotFound(TenantId), #[error("Tenant {0} is stopping")] IsStopping(TenantId), #[error(transparent)] + SlotError(#[from] TenantSlotError), + #[error(transparent)] + SlotUpsertError(#[from] TenantSlotUpsertError), + #[error(transparent)] Other(#[from] anyhow::Error), } pub(crate) async fn detach_tenant( conf: &'static PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, ) -> Result<(), TenantStateError> { let tmp_path = detach_tenant0( conf, &TENANTS, - tenant_id, + tenant_shard_id, detach_ignored, deletion_queue_client, ) @@ -967,20 +1256,25 @@ pub(crate) async fn detach_tenant( async fn detach_tenant0( conf: &'static PageServerConf, - tenants: &tokio::sync::RwLock, - tenant_id: TenantId, + tenants: &std::sync::RwLock, + tenant_shard_id: TenantShardId, detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, ) -> Result { - let tenant_dir_rename_operation = |tenant_id_to_clean| async move { - let local_tenant_directory = conf.tenant_path(&tenant_id_to_clean); + let tenant_dir_rename_operation = |tenant_id_to_clean: TenantShardId| async move { + // TODO(sharding): make local path helpers shard-aware + let local_tenant_directory = conf.tenant_path(&tenant_id_to_clean.tenant_id); safe_rename_tenant_dir(&local_tenant_directory) .await .with_context(|| format!("local tenant directory {local_tenant_directory:?} rename")) }; - let removal_result = - remove_tenant_from_memory(tenants, tenant_id, tenant_dir_rename_operation(tenant_id)).await; + let removal_result = remove_tenant_from_memory( + tenants, + tenant_shard_id, + tenant_dir_rename_operation(tenant_shard_id), + ) + .await; // Flush pending deletions, so that they have a good chance of passing validation // before this tenant is potentially re-attached elsewhere. @@ -988,13 +1282,21 @@ async fn detach_tenant0( // Ignored tenants are not present in memory and will bail the removal from memory operation. // Before returning the error, check for ignored tenant removal case — we only need to clean its local files then. - if detach_ignored && matches!(removal_result, Err(TenantStateError::NotFound(_))) { - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); + if detach_ignored + && matches!( + removal_result, + Err(TenantStateError::SlotError(TenantSlotError::NotFound(_))) + ) + { + // TODO(sharding): make local paths sharding-aware + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id.tenant_id); if tenant_ignore_mark.exists() { info!("Detaching an ignored tenant"); - let tmp_path = tenant_dir_rename_operation(tenant_id) + let tmp_path = tenant_dir_rename_operation(tenant_shard_id) .await - .with_context(|| format!("Ignored tenant {tenant_id} local directory rename"))?; + .with_context(|| { + format!("Ignored tenant {tenant_shard_id} local directory rename") + })?; return Ok(tmp_path); } } @@ -1011,31 +1313,48 @@ pub(crate) async fn load_tenant( deletion_queue_client: DeletionQueueClient, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let tenant_path = conf.tenant_path(&tenant_id); - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); - if tenant_ignore_mark.exists() { - std::fs::remove_file(&tenant_ignore_mark) - .with_context(|| format!("Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading"))?; - } + // This is a legacy API (replaced by `/location_conf`). It does not support sharding + let tenant_shard_id = TenantShardId::unsharded(tenant_id); - let resources = TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client - }; + let slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; + let tenant_path = conf.tenant_path(&tenant_id); - let mut location_conf = Tenant::load_tenant_config(conf, &tenant_id).map_err( TenantMapInsertError::Other)?; - location_conf.attach_in_generation(generation); - Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); + if tenant_ignore_mark.exists() { + std::fs::remove_file(&tenant_ignore_mark).with_context(|| { + format!( + "Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading" + ) + })?; + } - let new_tenant = tenant_spawn(conf, tenant_id, &tenant_path, resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx) - .with_context(|| { - format!("Failed to schedule tenant processing in path {tenant_path:?}") - })?; + let resources = TenantSharedResources { + broker_client, + remote_storage, + deletion_queue_client, + }; - Ok(new_tenant) - }).await?; + let mut location_conf = + Tenant::load_tenant_config(conf, &tenant_id).map_err(TenantMapInsertError::Other)?; + location_conf.attach_in_generation(generation); + + Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + + let new_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + ) + .with_context(|| format!("Failed to schedule tenant processing in path {tenant_path:?}"))?; + + slot_guard.upsert(TenantSlot::Attached(new_tenant))?; Ok(()) } @@ -1048,10 +1367,13 @@ pub(crate) async fn ignore_tenant( async fn ignore_tenant0( conf: &'static PageServerConf, - tenants: &tokio::sync::RwLock, + tenants: &std::sync::RwLock, tenant_id: TenantId, ) -> Result<(), TenantStateError> { - remove_tenant_from_memory(tenants, tenant_id, async { + // This is a legacy API (replaced by `/location_conf`). It does not support sharding + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + remove_tenant_from_memory(tenants, tenant_shard_id, async { let ignore_mark_file = conf.tenant_ignore_mark_file_path(&tenant_id); fs::File::create(&ignore_mark_file) .await @@ -1060,7 +1382,7 @@ async fn ignore_tenant0( crashsafe::fsync_file_and_parent(&ignore_mark_file) .context("Failed to fsync ignore mark file") }) - .with_context(|| format!("Failed to crate ignore mark for tenant {tenant_id}"))?; + .with_context(|| format!("Failed to crate ignore mark for tenant {tenant_shard_id}"))?; Ok(()) }) .await @@ -1076,16 +1398,19 @@ pub(crate) enum TenantMapListError { /// Get list of tenants, for the mgmt API /// pub(crate) async fn list_tenants() -> Result, TenantMapListError> { - let tenants = TENANTS.read().await; + let tenants = TENANTS.read().unwrap(); let m = match &*tenants { TenantsMap::Initializing => return Err(TenantMapListError::Initializing), TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m, }; Ok(m.iter() .filter_map(|(id, tenant)| match tenant { - TenantSlot::Attached(tenant) => Some((*id, tenant.current_state())), + TenantSlot::Attached(tenant) => Some((id, tenant.current_state())), TenantSlot::Secondary => None, + TenantSlot::InProgress(_) => None, }) + // TODO(sharding): make callers of this function shard-aware + .map(|(k, v)| (k.tenant_id, v)) .collect()) } @@ -1101,101 +1426,436 @@ pub(crate) async fn attach_tenant( resources: TenantSharedResources, ctx: &RequestContext, ) -> Result<(), TenantMapInsertError> { - tenant_map_insert(tenant_id, || async { - let location_conf = LocationConf::attached_single(tenant_conf, generation); - let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; - // TODO: tenant directory remains on disk if we bail out from here on. - // See https://github.com/neondatabase/neon/issues/4233 + // This is a legacy API (replaced by `/location_conf`). It does not support sharding + let tenant_shard_id = TenantShardId::unsharded(tenant_id); - let attached_tenant = tenant_spawn(conf, tenant_id, &tenant_dir, - resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx)?; - // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. - // See https://github.com/neondatabase/neon/issues/4233 + let slot_guard = + tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; + let location_conf = LocationConf::attached_single(tenant_conf, generation); + let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; + // TODO: tenant directory remains on disk if we bail out from here on. + // See https://github.com/neondatabase/neon/issues/4233 - let attached_tenant_id = attached_tenant.tenant_id(); - anyhow::ensure!( - tenant_id == attached_tenant_id, + let attached_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_dir, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Normal, + ctx, + )?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 + + let attached_tenant_id = attached_tenant.tenant_id(); + if tenant_id != attached_tenant_id { + return Err(TenantMapInsertError::Other(anyhow::anyhow!( "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {attached_tenant_id})", - ); - Ok(attached_tenant) - }) - .await?; + ))); + } + + slot_guard.upsert(TenantSlot::Attached(attached_tenant))?; Ok(()) } #[derive(Debug, thiserror::Error)] pub(crate) enum TenantMapInsertError { - #[error("tenant map is still initializing")] - StillInitializing, - #[error("tenant map is shutting down")] - ShuttingDown, - #[error("tenant {0} already exists, state: {1:?}")] - TenantAlreadyExists(TenantId, TenantState), - #[error("tenant {0} already exists in secondary state")] - TenantExistsSecondary(TenantId), + #[error(transparent)] + SlotError(#[from] TenantSlotError), + #[error(transparent)] + SlotUpsertError(#[from] TenantSlotUpsertError), #[error(transparent)] Other(#[from] anyhow::Error), } -/// Give the given closure access to the tenants map entry for the given `tenant_id`, iff that -/// entry is vacant. The closure is responsible for creating the tenant object and inserting -/// it into the tenants map through the vacnt entry that it receives as argument. +/// Superset of TenantMapError: issues that can occur when acquiring a slot +/// for a particular tenant ID. +#[derive(Debug, thiserror::Error)] +pub enum TenantSlotError { + /// When acquiring a slot with the expectation that the tenant already exists. + #[error("Tenant {0} not found")] + NotFound(TenantShardId), + + /// When acquiring a slot with the expectation that the tenant does not already exist. + #[error("tenant {0} already exists, state: {1:?}")] + AlreadyExists(TenantShardId, TenantState), + + #[error("tenant {0} already exists in but is not attached")] + Conflict(TenantShardId), + + // Tried to read a slot that is currently being mutated by another administrative + // operation. + #[error("tenant has a state change in progress, try again later")] + InProgress, + + #[error(transparent)] + MapState(#[from] TenantMapError), +} + +/// Superset of TenantMapError: issues that can occur when using a SlotGuard +/// to insert a new value. +#[derive(Debug, thiserror::Error)] +pub enum TenantSlotUpsertError { + /// An error where the slot is in an unexpected state, indicating a code bug + #[error("Internal error updating Tenant")] + InternalError(Cow<'static, str>), + + #[error(transparent)] + MapState(#[from] TenantMapError), +} + +#[derive(Debug)] +enum TenantSlotDropError { + /// It is only legal to drop a TenantSlot if its contents are fully shut down + NotShutdown, +} + +/// Errors that can happen any time we are walking the tenant map to try and acquire +/// the TenantSlot for a particular tenant. +#[derive(Debug, thiserror::Error)] +pub enum TenantMapError { + // Tried to read while initializing + #[error("tenant map is still initializing")] + StillInitializing, + + // Tried to read while shutting down + #[error("tenant map is shutting down")] + ShuttingDown, +} + +/// Guards a particular tenant_id's content in the TenantsMap. While this +/// structure exists, the TenantsMap will contain a [`TenantSlot::InProgress`] +/// for this tenant, which acts as a marker for any operations targeting +/// this tenant to retry later, or wait for the InProgress state to end. /// -/// NB: the closure should return quickly because the current implementation of tenants map -/// serializes access through an `RwLock`. -async fn tenant_map_insert( - tenant_id: TenantId, - insert_fn: F, -) -> Result, TenantMapInsertError> -where - F: FnOnce() -> R, - R: std::future::Future>>, -{ - let mut guard = TENANTS.write().await; - let m = match &mut *guard { - TenantsMap::Initializing => return Err(TenantMapInsertError::StillInitializing), - TenantsMap::ShuttingDown(_) => return Err(TenantMapInsertError::ShuttingDown), - TenantsMap::Open(m) => m, - }; - match m.entry(tenant_id) { - hash_map::Entry::Occupied(e) => match e.get() { - TenantSlot::Attached(t) => Err(TenantMapInsertError::TenantAlreadyExists( - tenant_id, - t.current_state(), - )), - TenantSlot::Secondary => Err(TenantMapInsertError::TenantExistsSecondary(tenant_id)), - }, - hash_map::Entry::Vacant(v) => match insert_fn().await { - Ok(tenant) => { - v.insert(TenantSlot::Attached(tenant.clone())); - Ok(tenant) +/// This structure enforces the important invariant that we do not have overlapping +/// tasks that will try use local storage for a the same tenant ID: we enforce that +/// the previous contents of a slot have been shut down before the slot can be +/// left empty or used for something else +/// +/// Holders of a SlotGuard should explicitly dispose of it, using either `upsert` +/// to provide a new value, or `revert` to put the slot back into its initial +/// state. If the SlotGuard is dropped without calling either of these, then +/// we will leave the slot empty if our `old_value` is already shut down, else +/// we will replace the slot with `old_value` (equivalent to doing a revert). +/// +/// The `old_value` may be dropped before the SlotGuard is dropped, by calling +/// `drop_old_value`. It is an error to call this without shutting down +/// the conents of `old_value`. +pub struct SlotGuard { + tenant_shard_id: TenantShardId, + old_value: Option, + upserted: bool, + + /// [`TenantSlot::InProgress`] carries the corresponding Barrier: it will + /// release any waiters as soon as this SlotGuard is dropped. + _completion: utils::completion::Completion, +} + +impl SlotGuard { + fn new( + tenant_shard_id: TenantShardId, + old_value: Option, + completion: utils::completion::Completion, + ) -> Self { + Self { + tenant_shard_id, + old_value, + upserted: false, + _completion: completion, + } + } + + /// Take any value that was present in the slot before we acquired ownership + /// of it: in state transitions, this will be the old state. + fn get_old_value(&mut self) -> &Option { + &self.old_value + } + + /// Emplace a new value in the slot. This consumes the guard, and after + /// returning, the slot is no longer protected from concurrent changes. + fn upsert(mut self, new_value: TenantSlot) -> Result<(), TenantSlotUpsertError> { + if !self.old_value_is_shutdown() { + // This is a bug: callers should never try to drop an old value without + // shutting it down + return Err(TenantSlotUpsertError::InternalError( + "Old TenantSlot value not shut down".into(), + )); + } + + let replaced = { + let mut locked = TENANTS.write().unwrap(); + + if let TenantSlot::InProgress(_) = new_value { + // It is never expected to try and upsert InProgress via this path: it should + // only be written via the tenant_map_acquire_slot path. If we hit this it's a bug. + return Err(TenantSlotUpsertError::InternalError( + "Attempt to upsert an InProgress state".into(), + )); } - Err(e) => Err(TenantMapInsertError::Other(e)), - }, + + let m = match &mut *locked { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing.into()), + TenantsMap::ShuttingDown(_) => { + return Err(TenantMapError::ShuttingDown.into()); + } + TenantsMap::Open(m) => m, + }; + + let replaced = m.insert(self.tenant_shard_id, new_value); + self.upserted = true; + + METRICS.tenant_slots.set(m.len() as u64); + + replaced + }; + + // Sanity check: on an upsert we should always be replacing an InProgress marker + match replaced { + Some(TenantSlot::InProgress(_)) => { + // Expected case: we find our InProgress in the map: nothing should have + // replaced it because the code that acquires slots will not grant another + // one for the same TenantId. + Ok(()) + } + None => { + METRICS.unexpected_errors.inc(); + error!( + tenant_shard_id = %self.tenant_shard_id, + "Missing InProgress marker during tenant upsert, this is a bug." + ); + Err(TenantSlotUpsertError::InternalError( + "Missing InProgress marker during tenant upsert".into(), + )) + } + Some(slot) => { + METRICS.unexpected_errors.inc(); + error!(tenant_shard_id=%self.tenant_shard_id, "Unexpected contents of TenantSlot during upsert, this is a bug. Contents: {:?}", slot); + Err(TenantSlotUpsertError::InternalError( + "Unexpected contents of TenantSlot".into(), + )) + } + } + } + + /// Replace the InProgress slot with whatever was in the guard when we started + fn revert(mut self) { + if let Some(value) = self.old_value.take() { + match self.upsert(value) { + Err(TenantSlotUpsertError::InternalError(_)) => { + // We already logged the error, nothing else we can do. + } + Err(TenantSlotUpsertError::MapState(_)) => { + // If the map is shutting down, we need not replace anything + } + Ok(()) => {} + } + } + } + + /// We may never drop our old value until it is cleanly shut down: otherwise we might leave + /// rogue background tasks that would write to the local tenant directory that this guard + /// is responsible for protecting + fn old_value_is_shutdown(&self) -> bool { + match self.old_value.as_ref() { + Some(TenantSlot::Attached(tenant)) => tenant.gate.close_complete(), + Some(TenantSlot::Secondary) => { + // TODO: when adding secondary mode tenants, this will check for shutdown + // in the same way that we do for `Tenant` above + true + } + Some(TenantSlot::InProgress(_)) => { + // A SlotGuard cannot be constructed for a slot that was already InProgress + unreachable!() + } + None => true, + } + } + + /// The guard holder is done with the old value of the slot: they are obliged to already + /// shut it down before we reach this point. + fn drop_old_value(&mut self) -> Result<(), TenantSlotDropError> { + if !self.old_value_is_shutdown() { + Err(TenantSlotDropError::NotShutdown) + } else { + self.old_value.take(); + Ok(()) + } } } -async fn tenant_map_upsert_slot<'a, F, R>( - tenant_id: TenantId, - upsert_fn: F, -) -> Result<(), TenantMapInsertError> -where - F: FnOnce(Option) -> R, - R: std::future::Future>, -{ - let mut guard = TENANTS.write().await; - let m = match &mut *guard { - TenantsMap::Initializing => return Err(TenantMapInsertError::StillInitializing), - TenantsMap::ShuttingDown(_) => return Err(TenantMapInsertError::ShuttingDown), +impl Drop for SlotGuard { + fn drop(&mut self) { + if self.upserted { + return; + } + // Our old value is already shutdown, or it never existed: it is safe + // for us to fully release the TenantSlot back into an empty state + + let mut locked = TENANTS.write().unwrap(); + + let m = match &mut *locked { + TenantsMap::Initializing => { + // There is no map, this should never happen. + return; + } + TenantsMap::ShuttingDown(_) => { + // When we transition to shutdown, InProgress elements are removed + // from the map, so we do not need to clean up our Inprogress marker. + // See [`shutdown_all_tenants0`] + return; + } + TenantsMap::Open(m) => m, + }; + + use std::collections::btree_map::Entry; + match m.entry(self.tenant_shard_id) { + Entry::Occupied(mut entry) => { + if !matches!(entry.get(), TenantSlot::InProgress(_)) { + METRICS.unexpected_errors.inc(); + error!(tenant_shard_id=%self.tenant_shard_id, "Unexpected contents of TenantSlot during drop, this is a bug. Contents: {:?}", entry.get()); + } + + if self.old_value_is_shutdown() { + entry.remove(); + } else { + entry.insert(self.old_value.take().unwrap()); + } + } + Entry::Vacant(_) => { + METRICS.unexpected_errors.inc(); + error!( + tenant_shard_id = %self.tenant_shard_id, + "Missing InProgress marker during SlotGuard drop, this is a bug." + ); + } + } + + METRICS.tenant_slots.set(m.len() as u64); + } +} + +enum TenantSlotPeekMode { + /// In Read mode, peek will be permitted to see the slots even if the pageserver is shutting down + Read, + /// In Write mode, trying to peek at a slot while the pageserver is shutting down is an error + Write, +} + +fn tenant_map_peek_slot<'a>( + tenants: &'a std::sync::RwLockReadGuard<'a, TenantsMap>, + tenant_shard_id: &TenantShardId, + mode: TenantSlotPeekMode, +) -> Result, TenantMapError> { + let m = match tenants.deref() { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing), + TenantsMap::ShuttingDown(m) => match mode { + TenantSlotPeekMode::Read => m, + TenantSlotPeekMode::Write => { + return Err(TenantMapError::ShuttingDown); + } + }, TenantsMap::Open(m) => m, }; - match upsert_fn(m.remove(&tenant_id)).await { - Ok(upsert_val) => { - m.insert(tenant_id, upsert_val); - Ok(()) + Ok(m.get(tenant_shard_id)) +} + +enum TenantSlotAcquireMode { + /// Acquire the slot irrespective of current state, or whether it already exists + Any, + /// Return an error if trying to acquire a slot and it doesn't already exist + MustExist, + /// Return an error if trying to acquire a slot and it already exists + MustNotExist, +} + +fn tenant_map_acquire_slot( + tenant_shard_id: &TenantShardId, + mode: TenantSlotAcquireMode, +) -> Result { + tenant_map_acquire_slot_impl(tenant_shard_id, &TENANTS, mode) +} + +fn tenant_map_acquire_slot_impl( + tenant_shard_id: &TenantShardId, + tenants: &std::sync::RwLock, + mode: TenantSlotAcquireMode, +) -> Result { + use TenantSlotAcquireMode::*; + METRICS.tenant_slot_writes.inc(); + + let mut locked = tenants.write().unwrap(); + let span = tracing::info_span!("acquire_slot", tenant_id=%tenant_shard_id.tenant_id, shard=tenant_shard_id.shard_slug()); + let _guard = span.enter(); + + let m = match &mut *locked { + TenantsMap::Initializing => return Err(TenantMapError::StillInitializing.into()), + TenantsMap::ShuttingDown(_) => return Err(TenantMapError::ShuttingDown.into()), + TenantsMap::Open(m) => m, + }; + + use std::collections::btree_map::Entry; + + let entry = m.entry(*tenant_shard_id); + + match entry { + Entry::Vacant(v) => match mode { + MustExist => { + tracing::debug!("Vacant && MustExist: return NotFound"); + Err(TenantSlotError::NotFound(*tenant_shard_id)) + } + _ => { + let (completion, barrier) = utils::completion::channel(); + v.insert(TenantSlot::InProgress(barrier)); + tracing::debug!("Vacant, inserted InProgress"); + Ok(SlotGuard::new(*tenant_shard_id, None, completion)) + } + }, + Entry::Occupied(mut o) => { + // Apply mode-driven checks + match (o.get(), mode) { + (TenantSlot::InProgress(_), _) => { + tracing::debug!("Occupied, failing for InProgress"); + Err(TenantSlotError::InProgress) + } + (slot, MustNotExist) => match slot { + TenantSlot::Attached(tenant) => { + tracing::debug!("Attached && MustNotExist, return AlreadyExists"); + Err(TenantSlotError::AlreadyExists( + *tenant_shard_id, + tenant.current_state(), + )) + } + _ => { + // FIXME: the AlreadyExists error assumes that we have a Tenant + // to get the state from + tracing::debug!("Occupied & MustNotExist, return AlreadyExists"); + Err(TenantSlotError::AlreadyExists( + *tenant_shard_id, + TenantState::Broken { + reason: "Present but not attached".to_string(), + backtrace: "".to_string(), + }, + )) + } + }, + _ => { + // Happy case: the slot was not in any state that violated our mode + let (completion, barrier) = utils::completion::channel(); + let old_value = o.insert(TenantSlot::InProgress(barrier)); + tracing::debug!("Occupied, replaced with InProgress"); + Ok(SlotGuard::new( + *tenant_shard_id, + Some(old_value), + completion, + )) + } + } } - Err(e) => Err(TenantMapInsertError::Other(e)), } } @@ -1204,8 +1864,8 @@ where /// If the cleanup fails, tenant will stay in memory in [`TenantState::Broken`] state, and another removal /// operation would be needed to remove it. async fn remove_tenant_from_memory( - tenants: &tokio::sync::RwLock, - tenant_id: TenantId, + tenants: &std::sync::RwLock, + tenant_shard_id: TenantShardId, tenant_cleanup: F, ) -> Result where @@ -1213,20 +1873,14 @@ where { use utils::completion; - // It's important to keep the tenant in memory after the final cleanup, to avoid cleanup races. - // The exclusive lock here ensures we don't miss the tenant state updates before trying another removal. - // tenant-wde cleanup operations may take some time (removing the entire tenant directory), we want to - // avoid holding the lock for the entire process. - let tenant = { - match tenants - .write() - .await - .get_slot(&tenant_id) - .ok_or(TenantStateError::NotFound(tenant_id))? - { - TenantSlot::Attached(t) => Some(t.clone()), - TenantSlot::Secondary => None, - } + let mut slot_guard = + tenant_map_acquire_slot_impl(&tenant_shard_id, tenants, TenantSlotAcquireMode::MustExist)?; + + // The SlotGuard allows us to manipulate the Tenant object without fear of some + // concurrent API request doing something else for the same tenant ID. + let attached_tenant = match slot_guard.get_old_value() { + Some(TenantSlot::Attached(t)) => Some(t), + _ => None, }; // allow pageserver shutdown to await for our completion @@ -1234,7 +1888,7 @@ where // If the tenant was attached, shut it down gracefully. For secondary // locations this part is not necessary - match tenant { + match &attached_tenant { Some(attached_tenant) => { // whenever we remove a tenant from memory, we don't want to flush and wait for upload let freeze_and_flush = false; @@ -1246,7 +1900,8 @@ where Err(_other) => { // if pageserver shutdown or other detach/ignore is already ongoing, we don't want to // wait for it but return an error right away because these are distinct requests. - return Err(TenantStateError::IsStopping(tenant_id)); + slot_guard.revert(); + return Err(TenantStateError::IsStopping(tenant_shard_id.tenant_id)); } } } @@ -1257,26 +1912,24 @@ where match tenant_cleanup .await - .with_context(|| format!("Failed to run cleanup for tenant {tenant_id}")) + .with_context(|| format!("Failed to run cleanup for tenant {tenant_shard_id}")) { Ok(hook_value) => { - let mut tenants_accessor = tenants.write().await; - if tenants_accessor.remove(&tenant_id).is_none() { - warn!("Tenant {tenant_id} got removed from memory before operation finished"); - } + // Success: drop the old TenantSlot::Attached. + slot_guard + .drop_old_value() + .expect("We just called shutdown"); + Ok(hook_value) } Err(e) => { - let tenants_accessor = tenants.read().await; - match tenants_accessor.get(&tenant_id) { - Some(tenant) => { - tenant.set_broken(e.to_string()).await; - } - None => { - warn!("Tenant {tenant_id} got removed from memory"); - return Err(TenantStateError::NotFound(tenant_id)); - } + // If we had a Tenant, set it to Broken and put it back in the TenantsMap + if let Some(attached_tenant) = attached_tenant { + attached_tenant.set_broken(e.to_string()).await; } + // Leave the broken tenant in the map + slot_guard.revert(); + Err(TenantStateError::Other(e)) } } @@ -1293,7 +1946,7 @@ pub(crate) async fn immediate_gc( gc_req: TimelineGcRequest, ctx: &RequestContext, ) -> Result>, ApiError> { - let guard = TENANTS.read().await; + let guard = TENANTS.read().unwrap(); let tenant = guard .get(&tenant_id) .map(Arc::clone) @@ -1338,7 +1991,8 @@ pub(crate) async fn immediate_gc( #[cfg(test)] mod tests { - use std::collections::HashMap; + use pageserver_api::shard::TenantShardId; + use std::collections::BTreeMap; use std::sync::Arc; use tracing::{info_span, Instrument}; @@ -1347,32 +2001,31 @@ mod tests { use super::{super::harness::TenantHarness, TenantsMap}; #[tokio::test(start_paused = true)] - async fn shutdown_joins_remove_tenant_from_memory() { - // the test is a bit ugly with the lockstep together with spawned tasks. the aim is to make - // sure `shutdown_all_tenants0` per-tenant processing joins in any active - // remove_tenant_from_memory calls, which is enforced by making the operation last until - // we've ran `shutdown_all_tenants0` for a long time. + async fn shutdown_awaits_in_progress_tenant() { + // Test that if an InProgress tenant is in the map during shutdown, the shutdown will gracefully + // wait for it to complete before proceeding. - let (t, _ctx) = TenantHarness::create("shutdown_joins_detach") + let (t, _ctx) = TenantHarness::create("shutdown_awaits_in_progress_tenant") .unwrap() .load() .await; // harness loads it to active, which is forced and nothing is running on the tenant - let id = t.tenant_id(); + let id = TenantShardId::unsharded(t.tenant_id()); // tenant harness configures the logging and we cannot escape it let _e = info_span!("testing", tenant_id = %id).entered(); - let tenants = HashMap::from([(id, TenantSlot::Attached(t.clone()))]); - let tenants = Arc::new(tokio::sync::RwLock::new(TenantsMap::Open(tenants))); + let tenants = BTreeMap::from([(id, TenantSlot::Attached(t.clone()))]); + let tenants = Arc::new(std::sync::RwLock::new(TenantsMap::Open(tenants))); + + // Invoke remove_tenant_from_memory with a cleanup hook that blocks until we manually + // permit it to proceed: that will stick the tenant in InProgress let (until_cleanup_completed, can_complete_cleanup) = utils::completion::channel(); let (until_cleanup_started, cleanup_started) = utils::completion::channel(); - - // start a "detaching operation", which will take a while, until can_complete_cleanup - let cleanup_task = { + let mut remove_tenant_from_memory_task = { let jh = tokio::spawn({ let tenants = tenants.clone(); async move { @@ -1391,12 +2044,6 @@ mod tests { jh }; - let mut cleanup_progress = std::pin::pin!(t - .shutdown(utils::completion::Barrier::default(), false) - .await - .unwrap_err() - .wait()); - let mut shutdown_task = { let (until_shutdown_started, shutdown_started) = utils::completion::channel(); @@ -1409,37 +2056,17 @@ mod tests { shutdown_task }; - // if the joining in is removed from shutdown_all_tenants0, the shutdown_task should always - // get to complete within timeout and fail the test. it is expected to continue awaiting - // until completion or SIGKILL during normal shutdown. - // - // the timeout is long to cover anything that shutdown_task could be doing, but it is - // handled instantly because we use tokio's time pausing in this test. 100s is much more than - // what we get from systemd on shutdown (10s). - let long_time = std::time::Duration::from_secs(100); + let long_time = std::time::Duration::from_secs(15); tokio::select! { - _ = &mut shutdown_task => unreachable!("shutdown must continue, until_cleanup_completed is not dropped"), - _ = &mut cleanup_progress => unreachable!("cleanup progress must continue, until_cleanup_completed is not dropped"), + _ = &mut shutdown_task => unreachable!("shutdown should block on remove_tenant_from_memory completing"), + _ = &mut remove_tenant_from_memory_task => unreachable!("remove_tenant_from_memory_task should not complete until explicitly unblocked"), _ = tokio::time::sleep(long_time) => {}, } - // allow the remove_tenant_from_memory and thus eventually the shutdown to continue drop(until_cleanup_completed); - let (je, ()) = tokio::join!(shutdown_task, cleanup_progress); - je.expect("Tenant::shutdown shutdown not have panicked"); - cleanup_task - .await - .expect("no panicking") - .expect("remove_tenant_from_memory failed"); - - futures::future::poll_immediate( - t.shutdown(utils::completion::Barrier::default(), false) - .await - .unwrap_err() - .wait(), - ) - .await - .expect("the stopping progress must still be complete"); + // Now that we allow it to proceed, shutdown should complete immediately + remove_tenant_from_memory_task.await.unwrap().unwrap(); + shutdown_task.await.unwrap(); } } diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 76e75253fb..bbf6a0c5c5 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1542,7 +1542,7 @@ pub fn remote_index_path( } /// Given the key of an index, parse out the generation part of the name -pub(crate) fn parse_remote_index_path(path: RemotePath) -> Option { +pub fn parse_remote_index_path(path: RemotePath) -> Option { let file_name = match path.get_path().file_name() { Some(f) => f, None => { diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 7cf963ca9d..fa0679c7a2 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use utils::bin_ser::SerializeError; use crate::tenant::metadata::TimelineMetadata; @@ -58,7 +57,6 @@ impl LayerFileMetadata { /// /// This type needs to be backwards and forwards compatible. When changing the fields, /// remember to add a test case for the changed version. -#[serde_as] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct IndexPart { /// Debugging aid describing the version of this type. @@ -78,7 +76,6 @@ pub struct IndexPart { // 'disk_consistent_lsn' is a copy of the 'disk_consistent_lsn' in the metadata. // It's duplicated for convenience when reading the serialized structure, but is // private because internally we would read from metadata instead. - #[serde_as(as = "DisplayFromStr")] disk_consistent_lsn: Lsn, #[serde(rename = "metadata_bytes")] @@ -155,7 +152,7 @@ pub struct IndexLayerMetadata { #[serde(default = "Generation::none")] #[serde(skip_serializing_if = "Generation::is_none")] - pub(super) generation: Generation, + pub generation: Generation, } impl From for IndexLayerMetadata { diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index e737d3f59c..a85dc9231c 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use anyhow::{bail, Context}; use tokio::sync::oneshot::error::RecvError; use tokio::sync::Semaphore; -use tokio_util::sync::CancellationToken; use crate::context::RequestContext; use crate::pgdatadir_mapping::CalculateLogicalSizeError; @@ -29,7 +28,6 @@ use tenant_size_model::{Segment, StorageModel}; /// needs. We will convert this into a StorageModel when it's time to perform /// the calculation. /// -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct ModelInputs { pub segments: Vec, @@ -37,11 +35,9 @@ pub struct ModelInputs { } /// A [`Segment`], with some extra information for display purposes -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct SegmentMeta { pub segment: Segment, - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, pub kind: LsnKind, } @@ -77,32 +73,22 @@ pub enum LsnKind { /// Collect all relevant LSNs to the inputs. These will only be helpful in the serialized form as /// part of [`ModelInputs`] from the HTTP api, explaining the inputs. -#[serde_with::serde_as] #[derive(Debug, serde::Serialize, serde::Deserialize)] pub struct TimelineInputs { - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, - #[serde_as(as = "Option")] pub ancestor_id: Option, - #[serde_as(as = "serde_with::DisplayFromStr")] ancestor_lsn: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] last_record: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] latest_gc_cutoff: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] horizon_cutoff: Lsn, - #[serde_as(as = "serde_with::DisplayFromStr")] pitr_cutoff: Lsn, /// Cutoff point based on GC settings - #[serde_as(as = "serde_with::DisplayFromStr")] next_gc_cutoff: Lsn, /// Cutoff point calculated from the user-supplied 'max_retention_period' - #[serde_as(as = "Option")] retention_param_cutoff: Option, } @@ -363,10 +349,6 @@ async fn fill_logical_sizes( // our advantage with `?` error handling. let mut joinset = tokio::task::JoinSet::new(); - let cancel = tokio_util::sync::CancellationToken::new(); - // be sure to cancel all spawned tasks if we are dropped - let _dg = cancel.clone().drop_guard(); - // For each point that would benefit from having a logical size available, // spawn a Task to fetch it, unless we have it cached already. for seg in segments.iter() { @@ -384,15 +366,8 @@ async fn fill_logical_sizes( let parallel_size_calcs = Arc::clone(limit); let ctx = ctx.attached_child(); joinset.spawn( - calculate_logical_size( - parallel_size_calcs, - timeline, - lsn, - cause, - ctx, - cancel.child_token(), - ) - .in_current_span(), + calculate_logical_size(parallel_size_calcs, timeline, lsn, cause, ctx) + .in_current_span(), ); } e.insert(cached_size); @@ -419,10 +394,12 @@ async fn fill_logical_sizes( have_any_error = true; } Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Err(error)))) => { - warn!( - timeline_id=%timeline.timeline_id, - "failed to calculate logical size at {lsn}: {error:#}" - ); + if !matches!(error, CalculateLogicalSizeError::Cancelled) { + warn!( + timeline_id=%timeline.timeline_id, + "failed to calculate logical size at {lsn}: {error:#}" + ); + } have_any_error = true; } Ok(Ok(TimelineAtLsnSizeResult(timeline, lsn, Ok(size)))) => { @@ -498,14 +475,13 @@ async fn calculate_logical_size( lsn: utils::lsn::Lsn, cause: LogicalSizeCalculationCause, ctx: RequestContext, - cancel: CancellationToken, ) -> Result { let _permit = tokio::sync::Semaphore::acquire_owned(limit) .await .expect("global semaphore should not had been closed"); let size_res = timeline - .spawn_ondemand_logical_size_calculation(lsn, cause, ctx, cancel) + .spawn_ondemand_logical_size_calculation(lsn, cause, ctx) .instrument(info_span!("spawn_ondemand_logical_size_calculation")) .await?; Ok(TimelineAtLsnSizeResult(timeline, lsn, size_res)) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 4ccce3d6bd..79f37dcb2d 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -289,7 +289,9 @@ impl DeltaLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result> { let path = self.path(); - let loaded = DeltaLayerInner::load(&path, None, ctx).await?; + let loaded = DeltaLayerInner::load(&path, None, ctx) + .await + .and_then(|res| res)?; // not production code let actual_filename = path.file_name().unwrap().to_owned(); @@ -610,18 +612,28 @@ impl Drop for DeltaLayerWriter { } impl DeltaLayerInner { + /// Returns nested result following Result, Critical>: + /// - inner has the success or transient failure + /// - outer has the permanent failure pub(super) async fn load( path: &Utf8Path, summary: Option, ctx: &RequestContext, - ) -> anyhow::Result { - let file = VirtualFile::open(path) - .await - .with_context(|| format!("Failed to open file '{path}'"))?; + ) -> Result, anyhow::Error> { + let file = match VirtualFile::open(path).await { + Ok(file) => file, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("open layer file"))), + }; let file = FileBlockReader::new(file); - let summary_blk = file.read_blk(0, ctx).await?; - let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; + let summary_blk = match file.read_blk(0, ctx).await { + Ok(blk) => blk, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("read first block"))), + }; + + // TODO: this should be an assertion instead; see ImageLayerInner::load + let actual_summary = + Summary::des_prefix(summary_blk.as_ref()).context("deserialize first block")?; if let Some(mut expected_summary) = summary { // production code path @@ -636,11 +648,11 @@ impl DeltaLayerInner { } } - Ok(DeltaLayerInner { + Ok(Ok(DeltaLayerInner { file, index_start_blk: actual_summary.index_start_blk, index_root_blk: actual_summary.index_root_blk, - }) + })) } pub(super) async fn get_value_reconstruct_data( diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index b7b9ca69b0..c38a9f6883 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -249,7 +249,9 @@ impl ImageLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result { let path = self.path(); - let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, ctx).await?; + let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, ctx) + .await + .and_then(|res| res)?; // not production code let actual_filename = path.file_name().unwrap().to_owned(); @@ -295,18 +297,31 @@ impl ImageLayer { } impl ImageLayerInner { + /// Returns nested result following Result, Critical>: + /// - inner has the success or transient failure + /// - outer has the permanent failure pub(super) async fn load( path: &Utf8Path, lsn: Lsn, summary: Option, ctx: &RequestContext, - ) -> anyhow::Result { - let file = VirtualFile::open(path) - .await - .with_context(|| format!("Failed to open file '{}'", path))?; + ) -> Result, anyhow::Error> { + let file = match VirtualFile::open(path).await { + Ok(file) => file, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("open layer file"))), + }; let file = FileBlockReader::new(file); - let summary_blk = file.read_blk(0, ctx).await?; - let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; + let summary_blk = match file.read_blk(0, ctx).await { + Ok(blk) => blk, + Err(e) => return Ok(Err(anyhow::Error::new(e).context("read first block"))), + }; + + // length is the only way how this could fail, so it's not actually likely at all unless + // read_blk returns wrong sized block. + // + // TODO: confirm and make this into assertion + let actual_summary = + Summary::des_prefix(summary_blk.as_ref()).context("deserialize first block")?; if let Some(mut expected_summary) = summary { // production code path @@ -322,12 +337,12 @@ impl ImageLayerInner { } } - Ok(ImageLayerInner { + Ok(Ok(ImageLayerInner { index_start_blk: actual_summary.index_start_blk, index_root_blk: actual_summary.index_root_blk, lsn, file, - }) + })) } pub(super) async fn get_value_reconstruct_data( diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 1275250bf0..2cb1e55b26 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -345,14 +345,19 @@ impl InMemoryLayer { let cursor = inner.file.block_cursor(); - let mut keys: Vec<(&Key, &VecMap)> = inner.index.iter().collect(); - keys.sort_by_key(|k| k.0); + // Sort the keys because delta layer writer expects them sorted. + // + // NOTE: this sort can take up significant time if the layer has millions of + // keys. To speed up all the comparisons we convert the key to i128 and + // keep the value as a reference. + let mut keys: Vec<_> = inner.index.iter().map(|(k, m)| (k.to_i128(), m)).collect(); + keys.sort_unstable_by_key(|k| k.0); let ctx = RequestContextBuilder::extend(ctx) .page_content_kind(PageContentKind::InMemoryLayer) .build(); for (key, vec_map) in keys.iter() { - let key = **key; + let key = Key::from_i128(*key); // Write all page versions for (lsn, pos) in vec_map.as_slice() { cursor.read_blob_into_buf(*pos, &mut buf, &ctx).await?; diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index d72982a9a0..f28f1c9444 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -251,6 +251,7 @@ impl Layer { layer .get_value_reconstruct_data(key, lsn_range, reconstruct_data, &self.0, ctx) + .instrument(tracing::info_span!("get_value_reconstruct_data", layer=%self)) .await } @@ -867,6 +868,9 @@ impl LayerInner { } Ok((Err(e), _permit)) => { // FIXME: this should be with the spawned task and be cancellation sensitive + // + // while we should not need this, this backoff has turned out to be useful with + // a bug of unexpectedly deleted remote layer file (#5787). let consecutive_failures = self.consecutive_failures.fetch_add(1, Ordering::Relaxed); tracing::error!(consecutive_failures, "layer file download failed: {e:#}"); @@ -1195,7 +1199,7 @@ impl DownloadedLayer { )); delta_layer::DeltaLayerInner::load(&owner.path, summary, ctx) .await - .map(LayerKind::Delta) + .map(|res| res.map(LayerKind::Delta)) } else { let lsn = owner.desc.image_layer_lsn(); let summary = Some(image_layer::Summary::expected( @@ -1206,21 +1210,32 @@ impl DownloadedLayer { )); image_layer::ImageLayerInner::load(&owner.path, lsn, summary, ctx) .await - .map(LayerKind::Image) - } - // this will be a permanent failure - .context("load layer"); + .map(|res| res.map(LayerKind::Image)) + }; - if res.is_err() { - LAYER_IMPL_METRICS.inc_permanent_loading_failures(); + match res { + Ok(Ok(layer)) => Ok(Ok(layer)), + Ok(Err(transient)) => Err(transient), + Err(permanent) => { + LAYER_IMPL_METRICS.inc_permanent_loading_failures(); + // TODO(#5815): we are not logging all errors, so temporarily log them **once** + // here as well + let permanent = permanent.context("load layer"); + tracing::error!("layer loading failed permanently: {permanent:#}"); + Ok(Err(permanent)) + } } - res }; - self.kind.get_or_init(init).await.as_ref().map_err(|e| { - // errors are not clonabled, cannot but stringify - // test_broken_timeline matches this string - anyhow::anyhow!("layer loading failed: {e:#}") - }) + self.kind + .get_or_try_init(init) + // return transient errors using `?` + .await? + .as_ref() + .map_err(|e| { + // errors are not clonabled, cannot but stringify + // test_broken_timeline matches this string + anyhow::anyhow!("layer loading failed: {e:#}") + }) } async fn get_value_reconstruct_data( @@ -1291,6 +1306,7 @@ impl ResidentLayer { } /// Loads all keys stored in the layer. Returns key, lsn and value size. + #[tracing::instrument(skip_all, fields(layer=%self))] pub(crate) async fn load_keys<'a>( &'a self, ctx: &RequestContext, diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index eb77f7c83a..e59001297c 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -180,7 +180,7 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { // Run compaction if let Err(e) = tenant.compaction_iteration(&cancel, &ctx).await { let wait_duration = backoff::exponential_backoff_duration_seconds( - error_run_count, + error_run_count + 1, 1.0, MAX_BACKOFF_SECS, ); @@ -198,6 +198,10 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Compaction); + // Perhaps we did no work and the walredo process has been idle for some time: + // give it a chance to shut down to avoid leaving walredo process running indefinitely. + tenant.walredo_mgr.maybe_quiesce(period * 10); + // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) .await @@ -261,7 +265,7 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { .await; if let Err(e) = res { let wait_duration = backoff::exponential_backoff_duration_seconds( - error_run_count, + error_run_count + 1, 1.0, MAX_BACKOFF_SECS, ); diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 4608683c4a..bbb96cb172 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -23,7 +23,7 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::id::TenantTimelineId; +use utils::{id::TenantTimelineId, sync::gate::Gate}; use std::cmp::{max, min, Ordering}; use std::collections::{BinaryHeap, HashMap, HashSet}; @@ -36,7 +36,6 @@ use std::time::{Duration, Instant, SystemTime}; use crate::context::{ AccessStatsBehavior, DownloadBehavior, RequestContext, RequestContextBuilder, }; -use crate::deletion_queue::DeletionQueueClient; use crate::tenant::storage_layer::delta_layer::DeltaEntry; use crate::tenant::storage_layer::{ AsLayerDesc, DeltaLayerWriter, EvictionError, ImageLayerWriter, InMemoryLayer, Layer, @@ -50,6 +49,7 @@ use crate::tenant::{ metadata::{save_metadata, TimelineMetadata}, par_fsync, }; +use crate::{deletion_queue::DeletionQueueClient, tenant::remote_timeline_client::StopError}; use crate::config::PageServerConf; use crate::keyspace::{KeyPartitioning, KeySpace, KeySpaceRandomAccum}; @@ -247,7 +247,7 @@ pub struct Timeline { /// the flush finishes. You can use that to wait for the flush to finish. layer_flush_start_tx: tokio::sync::watch::Sender, /// to be notified when layer flushing has finished, subscribe to the layer_flush_done channel - layer_flush_done_tx: tokio::sync::watch::Sender<(u64, anyhow::Result<()>)>, + layer_flush_done_tx: tokio::sync::watch::Sender<(u64, Result<(), FlushLayerError>)>, /// Layer removal lock. /// A lock to ensure that no layer of the timeline is removed concurrently by other tasks. @@ -310,6 +310,13 @@ pub struct Timeline { /// Load or creation time information about the disk_consistent_lsn and when the loading /// happened. Used for consumption metrics. pub(crate) loaded_at: (Lsn, SystemTime), + + /// Gate to prevent shutdown completing while I/O is still happening to this timeline's data + pub(crate) gate: Gate, + + /// Cancellation token scoped to this timeline: anything doing long-running work relating + /// to the timeline should drop out when this token fires. + pub(crate) cancel: CancellationToken, } pub struct WalReceiverInfo { @@ -367,6 +374,19 @@ pub enum PageReconstructError { WalRedo(anyhow::Error), } +#[derive(thiserror::Error, Debug)] +enum FlushLayerError { + /// Timeline cancellation token was cancelled + #[error("timeline shutting down")] + Cancelled, + + #[error(transparent)] + PageReconstructError(#[from] PageReconstructError), + + #[error(transparent)] + Other(#[from] anyhow::Error), +} + impl std::fmt::Debug for PageReconstructError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { match self { @@ -786,7 +806,11 @@ impl Timeline { // as an empty timeline. Also in unit tests, when we use the timeline // as a simple key-value store, ignoring the datadir layout. Log the // error but continue. - error!("could not compact, repartitioning keyspace failed: {err:?}"); + // + // Suppress error when it's due to cancellation + if !self.cancel.is_cancelled() { + error!("could not compact, repartitioning keyspace failed: {err:?}"); + } } }; @@ -880,11 +904,17 @@ impl Timeline { self.launch_eviction_task(background_jobs_can_start); } + /// Graceful shutdown, may do a lot of I/O as we flush any open layers to disk and then + /// also to remote storage. This method can easily take multiple seconds for a busy timeline. + /// + /// While we are flushing, we continue to accept read I/O. #[instrument(skip_all, fields(timeline_id=%self.timeline_id))] - pub async fn shutdown(self: &Arc, freeze_and_flush: bool) { + pub(crate) async fn flush_and_shutdown(&self) { debug_assert_current_span_has_tenant_and_timeline_id(); - // prevent writes to the InMemoryLayer + // Stop ingesting data, so that we are not still writing to an InMemoryLayer while + // trying to flush + tracing::debug!("Waiting for WalReceiverManager..."); task_mgr::shutdown_tasks( Some(TaskKind::WalReceiverManager), Some(self.tenant_id), @@ -892,34 +922,74 @@ impl Timeline { ) .await; + // Since we have shut down WAL ingest, we should not let anyone start waiting for the LSN to advance + self.last_record_lsn.shutdown(); + // now all writers to InMemory layer are gone, do the final flush if requested - if freeze_and_flush { - match self.freeze_and_flush().await { - Ok(()) => {} - Err(e) => { - warn!("failed to freeze and flush: {e:#}"); - return; // TODO: should probably drain remote timeline client anyways? + match self.freeze_and_flush().await { + Ok(_) => { + // drain the upload queue + if let Some(client) = self.remote_client.as_ref() { + // if we did not wait for completion here, it might be our shutdown process + // didn't wait for remote uploads to complete at all, as new tasks can forever + // be spawned. + // + // what is problematic is the shutting down of RemoteTimelineClient, because + // obviously it does not make sense to stop while we wait for it, but what + // about corner cases like s3 suddenly hanging up? + if let Err(e) = client.wait_completion().await { + // Non-fatal. Shutdown is infallible. Failures to flush just mean that + // we have some extra WAL replay to do next time the timeline starts. + warn!("failed to flush to remote storage: {e:#}"); + } } } - - // drain the upload queue - let res = if let Some(client) = self.remote_client.as_ref() { - // if we did not wait for completion here, it might be our shutdown process - // didn't wait for remote uploads to complete at all, as new tasks can forever - // be spawned. - // - // what is problematic is the shutting down of RemoteTimelineClient, because - // obviously it does not make sense to stop while we wait for it, but what - // about corner cases like s3 suddenly hanging up? - client.wait_completion().await - } else { - Ok(()) - }; - - if let Err(e) = res { - warn!("failed to await for frozen and flushed uploads: {e:#}"); + Err(e) => { + // Non-fatal. Shutdown is infallible. Failures to flush just mean that + // we have some extra WAL replay to do next time the timeline starts. + warn!("failed to freeze and flush: {e:#}"); } } + + self.shutdown().await; + } + + /// Shut down immediately, without waiting for any open layers to flush to disk. This is a subset of + /// the graceful [`Timeline::flush_and_shutdown`] function. + pub(crate) async fn shutdown(&self) { + // Signal any subscribers to our cancellation token to drop out + tracing::debug!("Cancelling CancellationToken"); + self.cancel.cancel(); + + // Page request handlers might be waiting for LSN to advance: they do not respect Timeline::cancel + // while doing so. + self.last_record_lsn.shutdown(); + + // Shut down the layer flush task before the remote client, as one depends on the other + task_mgr::shutdown_tasks( + Some(TaskKind::LayerFlushTask), + Some(self.tenant_id), + Some(self.timeline_id), + ) + .await; + + // Shut down remote timeline client: this gracefully moves its metadata into its Stopping state in + // case our caller wants to use that for a deletion + if let Some(remote_client) = self.remote_client.as_ref() { + match remote_client.stop() { + Ok(()) => {} + Err(StopError::QueueUninitialized) => { + // Shutting down during initialization is legal + } + } + } + + tracing::debug!("Waiting for tasks..."); + + task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(self.timeline_id)).await; + + // Finally wait until any gate-holders are complete + self.gate.close().await; } pub fn set_state(&self, new_state: TimelineState) { @@ -959,7 +1029,12 @@ impl Timeline { reason, backtrace: backtrace_str, }; - self.set_state(broken_state) + self.set_state(broken_state); + + // Although the Broken state is not equivalent to shutdown() (shutdown will be called + // later when this tenant is detach or the process shuts down), firing the cancellation token + // here avoids the need for other tasks to watch for the Broken state explicitly. + self.cancel.cancel(); } pub fn current_state(&self) -> TimelineState { @@ -1048,6 +1123,11 @@ impl Timeline { /// Like [`evict_layer_batch`](Self::evict_layer_batch), but for just one layer. /// Additional case `Ok(None)` covers the case where the layer could not be found by its `layer_file_name`. pub async fn evict_layer(&self, layer_file_name: &str) -> anyhow::Result> { + let _gate = self + .gate + .enter() + .map_err(|_| anyhow::anyhow!("Shutting down"))?; + let Some(local_layer) = self.find_layer(layer_file_name).await else { return Ok(None); }; @@ -1063,9 +1143,8 @@ impl Timeline { .as_ref() .ok_or_else(|| anyhow::anyhow!("remote storage not configured; cannot evict"))?; - let cancel = CancellationToken::new(); let results = self - .evict_layer_batch(remote_client, &[local_layer], &cancel) + .evict_layer_batch(remote_client, &[local_layer]) .await?; assert_eq!(results.len(), 1); let result: Option> = results.into_iter().next().unwrap(); @@ -1080,15 +1159,18 @@ impl Timeline { pub(crate) async fn evict_layers( &self, layers_to_evict: &[Layer], - cancel: &CancellationToken, ) -> anyhow::Result>>> { + let _gate = self + .gate + .enter() + .map_err(|_| anyhow::anyhow!("Shutting down"))?; + let remote_client = self .remote_client .as_ref() .context("timeline must have RemoteTimelineClient")?; - self.evict_layer_batch(remote_client, layers_to_evict, cancel) - .await + self.evict_layer_batch(remote_client, layers_to_evict).await } /// Evict multiple layers at once, continuing through errors. @@ -1109,7 +1191,6 @@ impl Timeline { &self, remote_client: &Arc, layers_to_evict: &[Layer], - cancel: &CancellationToken, ) -> anyhow::Result>>> { // ensure that the layers have finished uploading // (don't hold the layer_removal_cs while we do it, we're not removing anything yet) @@ -1157,7 +1238,7 @@ impl Timeline { }; tokio::select! { - _ = cancel.cancelled() => {}, + _ = self.cancel.cancelled() => {}, _ = join => {} } @@ -1267,6 +1348,7 @@ impl Timeline { initial_logical_size_can_start: Option, initial_logical_size_attempt: Option, state: TimelineState, + cancel: CancellationToken, ) -> Arc { let disk_consistent_lsn = metadata.disk_consistent_lsn(); let (state, _) = watch::channel(state); @@ -1367,6 +1449,8 @@ impl Timeline { initial_logical_size_can_start, initial_logical_size_attempt: Mutex::new(initial_logical_size_attempt), + cancel, + gate: Gate::new(format!("Timeline<{tenant_id}/{timeline_id}>")), }; result.repartition_threshold = result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; @@ -1706,12 +1790,8 @@ impl Timeline { // delay will be terminated by a timeout regardless. let _completion = { self_clone.initial_logical_size_attempt.lock().expect("unexpected initial_logical_size_attempt poisoned").take() }; - // no extra cancellation here, because nothing really waits for this to complete compared - // to spawn_ondemand_logical_size_calculation. - let cancel = CancellationToken::new(); - let calculated_size = match self_clone - .logical_size_calculation_task(lsn, LogicalSizeCalculationCause::Initial, &background_ctx, cancel) + .logical_size_calculation_task(lsn, LogicalSizeCalculationCause::Initial, &background_ctx) .await { Ok(s) => s, @@ -1780,7 +1860,6 @@ impl Timeline { lsn: Lsn, cause: LogicalSizeCalculationCause, ctx: RequestContext, - cancel: CancellationToken, ) -> oneshot::Receiver> { let (sender, receiver) = oneshot::channel(); let self_clone = Arc::clone(self); @@ -1801,7 +1880,7 @@ impl Timeline { false, async move { let res = self_clone - .logical_size_calculation_task(lsn, cause, &ctx, cancel) + .logical_size_calculation_task(lsn, cause, &ctx) .await; let _ = sender.send(res).ok(); Ok(()) // Receiver is responsible for handling errors @@ -1817,58 +1896,28 @@ impl Timeline { lsn: Lsn, cause: LogicalSizeCalculationCause, ctx: &RequestContext, - cancel: CancellationToken, ) -> Result { span::debug_assert_current_span_has_tenant_and_timeline_id(); - let mut timeline_state_updates = self.subscribe_for_state_updates(); + let _guard = self.gate.enter(); + let self_calculation = Arc::clone(self); let mut calculation = pin!(async { - let cancel = cancel.child_token(); let ctx = ctx.attached_child(); self_calculation - .calculate_logical_size(lsn, cause, cancel, &ctx) + .calculate_logical_size(lsn, cause, &ctx) .await }); - let timeline_state_cancellation = async { - loop { - match timeline_state_updates.changed().await { - Ok(()) => { - let new_state = timeline_state_updates.borrow().clone(); - match new_state { - // we're running this job for active timelines only - TimelineState::Active => continue, - TimelineState::Broken { .. } - | TimelineState::Stopping - | TimelineState::Loading => { - break format!("aborted because timeline became inactive (new state: {new_state:?})") - } - } - } - Err(_sender_dropped_error) => { - // can't happen, the sender is not dropped as long as the Timeline exists - break "aborted because state watch was dropped".to_string(); - } - } - } - }; - - let taskmgr_shutdown_cancellation = async { - task_mgr::shutdown_watcher().await; - "aborted because task_mgr shutdown requested".to_string() - }; tokio::select! { res = &mut calculation => { res } - reason = timeline_state_cancellation => { - debug!(reason = reason, "cancelling calculation"); - cancel.cancel(); + _ = self.cancel.cancelled() => { + debug!("cancelling logical size calculation for timeline shutdown"); calculation.await } - reason = taskmgr_shutdown_cancellation => { - debug!(reason = reason, "cancelling calculation"); - cancel.cancel(); + _ = task_mgr::shutdown_watcher() => { + debug!("cancelling logical size calculation for task shutdown"); calculation.await } } @@ -1882,7 +1931,6 @@ impl Timeline { &self, up_to_lsn: Lsn, cause: LogicalSizeCalculationCause, - cancel: CancellationToken, ctx: &RequestContext, ) -> Result { info!( @@ -1925,7 +1973,7 @@ impl Timeline { }; let timer = storage_time_metrics.start_timer(); let logical_size = self - .get_current_logical_size_non_incremental(up_to_lsn, cancel, ctx) + .get_current_logical_size_non_incremental(up_to_lsn, ctx) .await?; debug!("calculated logical size: {logical_size}"); timer.stop_and_record(); @@ -2030,6 +2078,10 @@ impl Timeline { let mut cont_lsn = Lsn(request_lsn.0 + 1); 'outer: loop { + if self.cancel.is_cancelled() { + return Err(PageReconstructError::Cancelled); + } + // The function should have updated 'state' //info!("CALLED for {} at {}: {:?} with {} records, cached {}", key, cont_lsn, result, reconstruct_state.records.len(), cached_lsn); match result { @@ -2334,6 +2386,10 @@ impl Timeline { info!("started flush loop"); loop { tokio::select! { + _ = self.cancel.cancelled() => { + info!("shutting down layer flush task"); + break; + }, _ = task_mgr::shutdown_watcher() => { info!("shutting down layer flush task"); break; @@ -2345,6 +2401,14 @@ impl Timeline { let timer = self.metrics.flush_time_histo.start_timer(); let flush_counter = *layer_flush_start_rx.borrow(); let result = loop { + if self.cancel.is_cancelled() { + info!("dropping out of flush loop for timeline shutdown"); + // Note: we do not bother transmitting into [`layer_flush_done_tx`], because + // anyone waiting on that will respect self.cancel as well: they will stop + // waiting at the same time we as drop out of this loop. + return; + } + let layer_to_flush = { let guard = self.layers.read().await; guard.layer_map().frozen_layers.front().cloned() @@ -2353,9 +2417,18 @@ impl Timeline { let Some(layer_to_flush) = layer_to_flush else { break Ok(()); }; - if let Err(err) = self.flush_frozen_layer(layer_to_flush, ctx).await { - error!("could not flush frozen layer: {err:?}"); - break Err(err); + match self.flush_frozen_layer(layer_to_flush, ctx).await { + Ok(()) => {} + Err(FlushLayerError::Cancelled) => { + info!("dropping out of flush loop for timeline shutdown"); + return; + } + err @ Err( + FlushLayerError::Other(_) | FlushLayerError::PageReconstructError(_), + ) => { + error!("could not flush frozen layer: {err:?}"); + break err; + } } }; // Notify any listeners that we're done @@ -2404,7 +2477,17 @@ impl Timeline { } } trace!("waiting for flush to complete"); - rx.changed().await?; + tokio::select! { + rx_e = rx.changed() => { + rx_e?; + }, + // Cancellation safety: we are not leaving an I/O in-flight for the flush, we're just ignoring + // the notification from [`flush_loop`] that it completed. + _ = self.cancel.cancelled() => { + tracing::info!("Cancelled layer flush due on timeline shutdown"); + return Ok(()) + } + }; trace!("done") } } @@ -2419,7 +2502,7 @@ impl Timeline { self: &Arc, frozen_layer: Arc, ctx: &RequestContext, - ) -> anyhow::Result<()> { + ) -> Result<(), FlushLayerError> { // As a special case, when we have just imported an image into the repository, // instead of writing out a L0 delta layer, we directly write out image layer // files instead. This is possible as long as *all* the data imported into the @@ -2444,6 +2527,11 @@ impl Timeline { let (partitioning, _lsn) = self .repartition(self.initdb_lsn, self.get_compaction_target_size(), ctx) .await?; + + if self.cancel.is_cancelled() { + return Err(FlushLayerError::Cancelled); + } + // For image layers, we add them immediately into the layer map. ( self.create_image_layers(&partitioning, self.initdb_lsn, true, ctx) @@ -2475,6 +2563,10 @@ impl Timeline { ) }; + if self.cancel.is_cancelled() { + return Err(FlushLayerError::Cancelled); + } + let disk_consistent_lsn = Lsn(lsn_range.end.0 - 1); let old_disk_consistent_lsn = self.disk_consistent_lsn.load(); @@ -2484,6 +2576,10 @@ impl Timeline { let metadata = { let mut guard = self.layers.write().await; + if self.cancel.is_cancelled() { + return Err(FlushLayerError::Cancelled); + } + guard.finish_flush_l0_layer(delta_layer_to_add.as_ref(), &frozen_layer, &self.metrics); if disk_consistent_lsn != old_disk_consistent_lsn { @@ -2936,13 +3032,10 @@ struct CompactLevel0Phase1StatsBuilder { new_deltas_size: Option, } -#[serde_as] #[derive(serde::Serialize)] struct CompactLevel0Phase1Stats { version: u64, - #[serde_as(as = "serde_with::DisplayFromStr")] tenant_id: TenantId, - #[serde_as(as = "serde_with::DisplayFromStr")] timeline_id: TimelineId, read_lock_acquisition_micros: RecordedDuration, read_lock_held_spawn_blocking_startup_micros: RecordedDuration, @@ -4369,25 +4462,10 @@ mod tests { .expect("should had been resident") .drop_eviction_guard(); - let cancel = tokio_util::sync::CancellationToken::new(); let batch = [layer]; - let first = { - let cancel = cancel.child_token(); - async { - let cancel = cancel; - timeline - .evict_layer_batch(&rc, &batch, &cancel) - .await - .unwrap() - } - }; - let second = async { - timeline - .evict_layer_batch(&rc, &batch, &cancel) - .await - .unwrap() - }; + let first = async { timeline.evict_layer_batch(&rc, &batch).await.unwrap() }; + let second = async { timeline.evict_layer_batch(&rc, &batch).await.unwrap() }; let (first, second) = tokio::join!(first, second); diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 6d30664515..56a99a25cf 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -17,6 +17,7 @@ use crate::{ deletion_queue::DeletionQueueClient, task_mgr::{self, TaskKind}, tenant::{ + debug_assert_current_span_has_tenant_and_timeline_id, metadata::TimelineMetadata, remote_timeline_client::{ self, PersistIndexPartWithDeletedFlagError, RemoteTimelineClient, @@ -30,6 +31,11 @@ use super::{Timeline, TimelineResources}; /// Now that the Timeline is in Stopping state, request all the related tasks to shut down. async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { + debug_assert_current_span_has_tenant_and_timeline_id(); + // Notify any timeline work to drop out of loops/requests + tracing::debug!("Cancelling CancellationToken"); + timeline.cancel.cancel(); + // Stop the walreceiver first. debug!("waiting for wal receiver to shutdown"); let maybe_started_walreceiver = { timeline.walreceiver.lock().unwrap().take() }; @@ -74,6 +80,11 @@ async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { "failpoint: timeline-delete-before-index-deleted-at" ))? }); + + tracing::debug!("Waiting for gate..."); + timeline.gate.close().await; + tracing::debug!("Shutdown complete"); + Ok(()) } diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index dc5c71bbe1..79bc434a2a 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -26,6 +26,7 @@ use tracing::{debug, error, info, info_span, instrument, warn, Instrument}; use crate::{ context::{DownloadBehavior, RequestContext}, + pgdatadir_mapping::CollectKeySpaceError, task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}, tenant::{ config::{EvictionPolicy, EvictionPolicyLayerAccessThreshold}, @@ -277,10 +278,7 @@ impl Timeline { Some(c) => c, }; - let results = match self - .evict_layer_batch(remote_client, &candidates, cancel) - .await - { + let results = match self.evict_layer_batch(remote_client, &candidates).await { Err(pre_err) => { stats.errors += candidates.len(); error!("could not do any evictions: {pre_err:#}"); @@ -329,8 +327,7 @@ impl Timeline { match state.last_layer_access_imitation { Some(ts) if ts.elapsed() < inter_imitate_period => { /* no need to run */ } _ => { - self.imitate_timeline_cached_layer_accesses(cancel, ctx) - .await; + self.imitate_timeline_cached_layer_accesses(ctx).await; state.last_layer_access_imitation = Some(tokio::time::Instant::now()) } } @@ -344,20 +341,7 @@ impl Timeline { // Make one of the tenant's timelines draw the short straw and run the calculation. // The others wait until the calculation is done so that they take into account the // imitated accesses that the winner made. - // - // It is critical we are responsive to cancellation here. Otherwise, we deadlock with - // tenant deletion (holds TENANTS in read mode) any other task that attempts to - // acquire TENANTS in write mode before we here call get_tenant. - // See https://github.com/neondatabase/neon/issues/5284. - let res = tokio::select! { - _ = cancel.cancelled() => { - return ControlFlow::Break(()); - } - res = crate::tenant::mgr::get_tenant(self.tenant_id, true) => { - res - } - }; - let tenant = match res { + let tenant = match crate::tenant::mgr::get_tenant(self.tenant_id, true) { Ok(t) => t, Err(_) => { return ControlFlow::Break(()); @@ -383,21 +367,12 @@ impl Timeline { /// Recompute the values which would cause on-demand downloads during restart. #[instrument(skip_all)] - async fn imitate_timeline_cached_layer_accesses( - &self, - cancel: &CancellationToken, - ctx: &RequestContext, - ) { + async fn imitate_timeline_cached_layer_accesses(&self, ctx: &RequestContext) { let lsn = self.get_last_record_lsn(); // imitiate on-restart initial logical size let size = self - .calculate_logical_size( - lsn, - LogicalSizeCalculationCause::EvictionTaskImitation, - cancel.clone(), - ctx, - ) + .calculate_logical_size(lsn, LogicalSizeCalculationCause::EvictionTaskImitation, ctx) .instrument(info_span!("calculate_logical_size")) .await; @@ -423,9 +398,16 @@ impl Timeline { if size.is_err() { // ignore, see above comment } else { - warn!( - "failed to collect keyspace but succeeded in calculating logical size: {e:#}" - ); + match e { + CollectKeySpaceError::Cancelled => { + // Shutting down, ignore + } + err => { + warn!( + "failed to collect keyspace but succeeded in calculating logical size: {err:#}" + ); + } + } } } } diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index e28c4a5f69..3077712445 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -426,7 +426,7 @@ impl ConnectionManagerState { timeline, new_sk.wal_source_connconf, events_sender, - cancellation, + cancellation.clone(), connect_timeout, ctx, node_id, @@ -447,7 +447,14 @@ impl ConnectionManagerState { } WalReceiverError::Other(e) => { // give out an error to have task_mgr give it a really verbose logging - Err(e).context("walreceiver connection handling failure") + if cancellation.is_cancelled() { + // Ideally we would learn about this via some path other than Other, but + // that requires refactoring all the intermediate layers of ingest code + // that only emit anyhow::Error + Ok(()) + } else { + Err(e).context("walreceiver connection handling failure") + } } } } diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index e1d699fcba..5d8cc0e181 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -43,8 +43,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use crate::config::PageServerConf; use crate::metrics::{ - WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, - WAL_REDO_WAIT_TIME, + WalRedoKillCause, WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_PROCESS_COUNTERS, + WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, }; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; use crate::repository::Key; @@ -91,6 +91,7 @@ struct ProcessOutput { pub struct PostgresRedoManager { tenant_id: TenantId, conf: &'static PageServerConf, + last_redo_at: std::sync::Mutex>, redo_process: RwLock>>, } @@ -187,10 +188,26 @@ impl PostgresRedoManager { PostgresRedoManager { tenant_id, conf, + last_redo_at: std::sync::Mutex::default(), redo_process: RwLock::new(None), } } + /// This type doesn't have its own background task to check for idleness: we + /// rely on our owner calling this function periodically in its own housekeeping + /// loops. + pub(crate) fn maybe_quiesce(&self, idle_timeout: Duration) { + if let Ok(g) = self.last_redo_at.try_lock() { + if let Some(last_redo_at) = *g { + if last_redo_at.elapsed() >= idle_timeout { + drop(g); + let mut guard = self.redo_process.write().unwrap(); + *guard = None; + } + } + } + } + /// /// Process one request for WAL redo using wal-redo postgres /// @@ -205,13 +222,12 @@ impl PostgresRedoManager { wal_redo_timeout: Duration, pg_version: u32, ) -> anyhow::Result { + *(self.last_redo_at.lock().unwrap()) = Some(Instant::now()); + let (rel, blknum) = key_to_rel_block(key).context("invalid record")?; const MAX_RETRY_ATTEMPTS: u32 = 1; - let start_time = Instant::now(); let mut n_attempts = 0u32; loop { - let lock_time = Instant::now(); - // launch the WAL redo process on first use let proc: Arc = { let proc_guard = self.redo_process.read().unwrap(); @@ -236,7 +252,7 @@ impl PostgresRedoManager { } }; - WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64()); + let started_at = std::time::Instant::now(); // Relational WAL records are applied using wal-redo-postgres let buf_tag = BufferTag { rel, blknum }; @@ -244,8 +260,7 @@ impl PostgresRedoManager { .apply_wal_records(buf_tag, &base_img, records, wal_redo_timeout) .context("apply_wal_records"); - let end_time = Instant::now(); - let duration = end_time.duration_since(lock_time); + let duration = started_at.elapsed(); let len = records.len(); let nbytes = records.iter().fold(0, |acumulator, record| { @@ -352,12 +367,13 @@ impl PostgresRedoManager { self.apply_record_neon(key, &mut page, *record_lsn, record)?; } // Success! - let end_time = Instant::now(); - let duration = end_time.duration_since(start_time); + let duration = start_time.elapsed(); + // FIXME: using the same metric here creates a bimodal distribution by default, and because + // there could be multiple batch sizes this would be N+1 modal. WAL_REDO_TIME.observe(duration.as_secs_f64()); debug!( - "neon applied {} WAL records in {} ms to reconstruct page image at LSN {}", + "neon applied {} WAL records in {} us to reconstruct page image at LSN {}", records.len(), duration.as_micros(), lsn @@ -596,21 +612,21 @@ trait CloseFileDescriptors: CommandExt { impl CloseFileDescriptors for C { fn close_fds(&mut self) -> &mut Command { + // SAFETY: Code executed inside pre_exec should have async-signal-safety, + // which means it should be safe to execute inside a signal handler. + // The precise meaning depends on platform. See `man signal-safety` + // for the linux definition. + // + // The set_fds_cloexec_threadsafe function is documented to be + // async-signal-safe. + // + // Aside from this function, the rest of the code is re-entrant and + // doesn't make any syscalls. We're just passing constants. + // + // NOTE: It's easy to indirectly cause a malloc or lock a mutex, + // which is not async-signal-safe. Be careful. unsafe { self.pre_exec(move || { - // SAFETY: Code executed inside pre_exec should have async-signal-safety, - // which means it should be safe to execute inside a signal handler. - // The precise meaning depends on platform. See `man signal-safety` - // for the linux definition. - // - // The set_fds_cloexec_threadsafe function is documented to be - // async-signal-safe. - // - // Aside from this function, the rest of the code is re-entrant and - // doesn't make any syscalls. We're just passing constants. - // - // NOTE: It's easy to indirectly cause a malloc or lock a mutex, - // which is not async-signal-safe. Be careful. close_fds::set_fds_cloexec_threadsafe(3, &[]); Ok(()) }) @@ -667,10 +683,10 @@ impl WalRedoProcess { .close_fds() .spawn_no_leak_child(tenant_id) .context("spawn process")?; - + WAL_REDO_PROCESS_COUNTERS.started.inc(); let mut child = scopeguard::guard(child, |child| { error!("killing wal-redo-postgres process due to a problem during launch"); - child.kill_and_wait(); + child.kill_and_wait(WalRedoKillCause::Startup); }); let stdin = child.stdin.take().unwrap(); @@ -1001,7 +1017,7 @@ impl Drop for WalRedoProcess { self.child .take() .expect("we only do this once") - .kill_and_wait(); + .kill_and_wait(WalRedoKillCause::WalRedoProcessDrop); self.stderr_logger_cancel.cancel(); // no way to wait for stderr_logger_task from Drop because that is async only } @@ -1037,16 +1053,19 @@ impl NoLeakChild { }) } - fn kill_and_wait(mut self) { + fn kill_and_wait(mut self, cause: WalRedoKillCause) { let child = match self.child.take() { Some(child) => child, None => return, }; - Self::kill_and_wait_impl(child); + Self::kill_and_wait_impl(child, cause); } - #[instrument(skip_all, fields(pid=child.id()))] - fn kill_and_wait_impl(mut child: Child) { + #[instrument(skip_all, fields(pid=child.id(), ?cause))] + fn kill_and_wait_impl(mut child: Child, cause: WalRedoKillCause) { + scopeguard::defer! { + WAL_REDO_PROCESS_COUNTERS.killed_by_cause[cause].inc(); + } let res = child.kill(); if let Err(e) = res { // This branch is very unlikely because: @@ -1091,7 +1110,7 @@ impl Drop for NoLeakChild { // This thread here is going to outlive of our dropper. let span = tracing::info_span!("walredo", %tenant_id); let _entered = span.enter(); - Self::kill_and_wait_impl(child); + Self::kill_and_wait_impl(child, WalRedoKillCause::NoLeakChildDrop); }) .await }); diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index 0a944a6bf0..cc09fb849d 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -19,7 +19,10 @@ #include "access/xlog.h" #include "access/xlogutils.h" #include "storage/buf_internals.h" +#include "storage/lwlock.h" +#include "storage/ipc.h" #include "c.h" +#include "postmaster/interrupt.h" #include "libpq-fe.h" #include "libpq/pqformat.h" @@ -61,23 +64,63 @@ int flush_every_n_requests = 8; int n_reconnect_attempts = 0; int max_reconnect_attempts = 60; +#define MAX_PAGESERVER_CONNSTRING_SIZE 256 + +typedef struct +{ + LWLockId lock; + pg_atomic_uint64 update_counter; + char pageserver_connstring[MAX_PAGESERVER_CONNSTRING_SIZE]; +} PagestoreShmemState; + +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void walproposer_shmem_request(void); +#endif +static shmem_startup_hook_type prev_shmem_startup_hook; +static PagestoreShmemState *pagestore_shared; +static uint64 pagestore_local_counter = 0; +static char local_pageserver_connstring[MAX_PAGESERVER_CONNSTRING_SIZE]; + bool (*old_redo_read_buffer_filter) (XLogReaderState *record, uint8 block_id) = NULL; static bool pageserver_flush(void); static void pageserver_disconnect(void); - -static pqsigfunc prev_signal_handler; +static bool +CheckPageserverConnstring(char **newval, void **extra, GucSource source) +{ + return strlen(*newval) < MAX_PAGESERVER_CONNSTRING_SIZE; +} static void -pageserver_sighup_handler(SIGNAL_ARGS) +AssignPageserverConnstring(const char *newval, void *extra) { - if (prev_signal_handler) - { - prev_signal_handler(postgres_signal_arg); - } - neon_log(LOG, "Received SIGHUP, disconnecting pageserver. New pageserver connstring is %s", page_server_connstring); - pageserver_disconnect(); + if(!pagestore_shared) + return; + LWLockAcquire(pagestore_shared->lock, LW_EXCLUSIVE); + strlcpy(pagestore_shared->pageserver_connstring, newval, MAX_PAGESERVER_CONNSTRING_SIZE); + pg_atomic_fetch_add_u64(&pagestore_shared->update_counter, 1); + LWLockRelease(pagestore_shared->lock); +} + +static bool +CheckConnstringUpdated() +{ + if(!pagestore_shared) + return false; + return pagestore_local_counter < pg_atomic_read_u64(&pagestore_shared->update_counter); +} + +static void +ReloadConnstring() +{ + if(!pagestore_shared) + return; + LWLockAcquire(pagestore_shared->lock, LW_SHARED); + strlcpy(local_pageserver_connstring, pagestore_shared->pageserver_connstring, sizeof(local_pageserver_connstring)); + pagestore_local_counter = pg_atomic_read_u64(&pagestore_shared->update_counter); + LWLockRelease(pagestore_shared->lock); } static bool @@ -91,6 +134,11 @@ pageserver_connect(int elevel) Assert(!connected); + if(CheckConnstringUpdated()) + { + ReloadConnstring(); + } + /* * Connect using the connection string we got from the * neon.pageserver_connstring GUC. If the NEON_AUTH_TOKEN environment @@ -110,7 +158,7 @@ pageserver_connect(int elevel) n++; } keywords[n] = "dbname"; - values[n] = page_server_connstring; + values[n] = local_pageserver_connstring; n++; keywords[n] = NULL; values[n] = NULL; @@ -254,6 +302,12 @@ pageserver_send(NeonRequest * request) { StringInfoData req_buff; + if(CheckConnstringUpdated()) + { + pageserver_disconnect(); + ReloadConnstring(); + } + /* If the connection was lost for some reason, reconnect */ if (connected && PQstatus(pageserver_conn) == CONNECTION_BAD) { @@ -274,6 +328,7 @@ pageserver_send(NeonRequest * request) { while (!pageserver_connect(n_reconnect_attempts < max_reconnect_attempts ? LOG : ERROR)) { + HandleMainLoopInterrupts(); n_reconnect_attempts += 1; pg_usleep(RECONNECT_INTERVAL_USEC); } @@ -391,7 +446,8 @@ pageserver_flush(void) return true; } -page_server_api api = { +page_server_api api = +{ .send = pageserver_send, .flush = pageserver_flush, .receive = pageserver_receive @@ -405,12 +461,72 @@ check_neon_id(char **newval, void **extra, GucSource source) return **newval == '\0' || HexDecodeString(id, *newval, 16); } +static Size +PagestoreShmemSize(void) +{ + return sizeof(PagestoreShmemState); +} + +static bool +PagestoreShmemInit(void) +{ + bool found; + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + pagestore_shared = ShmemInitStruct("libpagestore shared state", + PagestoreShmemSize(), + &found); + if(!found) + { + pagestore_shared->lock = &(GetNamedLWLockTranche("neon_libpagestore")->lock); + pg_atomic_init_u64(&pagestore_shared->update_counter, 0); + AssignPageserverConnstring(page_server_connstring, NULL); + } + LWLockRelease(AddinShmemInitLock); + return found; +} + +static void +pagestore_shmem_startup_hook(void) +{ + if(prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + PagestoreShmemInit(); +} + +static void +pagestore_shmem_request(void) +{ +#if PG_VERSION_NUM >= 150000 + if(prev_shmem_request_hook) + prev_shmem_request_hook(); +#endif + + RequestAddinShmemSpace(PagestoreShmemSize()); + RequestNamedLWLockTranche("neon_libpagestore", 1); +} + +static void +pagestore_prepare_shmem(void) +{ +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = pagestore_shmem_request; +#else + pagestore_shmem_request(); +#endif + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = pagestore_shmem_startup_hook; +} + /* * Module initialization function */ void pg_init_libpagestore(void) { + pagestore_prepare_shmem(); + DefineCustomStringVariable("neon.pageserver_connstring", "connection string to the page server", NULL, @@ -418,7 +534,7 @@ pg_init_libpagestore(void) "", PGC_SIGHUP, 0, /* no flags required */ - NULL, NULL, NULL); + CheckPageserverConnstring, AssignPageserverConnstring, NULL); DefineCustomStringVariable("neon.timeline_id", "Neon timeline_id the server is running on", @@ -499,7 +615,5 @@ pg_init_libpagestore(void) redo_read_buffer_filter = neon_redo_read_buffer_filter; } - prev_signal_handler = pqsignal(SIGHUP, pageserver_sighup_handler); - lfc_init(); } diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index 3a841b04ec..84b26198a7 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -1687,9 +1687,9 @@ neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, if (current_size >= ((uint64) max_cluster_size) * 1024 * 1024) ereport(ERROR, (errcode(ERRCODE_DISK_FULL), - errmsg("could not extend file because cluster size limit (%d MB) has been exceeded", + errmsg("could not extend file because project size limit (%d MB) has been exceeded", max_cluster_size), - errhint("This limit is defined by neon.max_cluster_size GUC"))); + errhint("This limit is defined externally by the project size limit, and internally by neon.max_cluster_size GUC"))); } /* diff --git a/pgxn/neon/walproposer_pg.c b/pgxn/neon/walproposer_pg.c index 865f91165b..f83a08d407 100644 --- a/pgxn/neon/walproposer_pg.c +++ b/pgxn/neon/walproposer_pg.c @@ -88,7 +88,7 @@ static void StartProposerReplication(WalProposer *wp, StartReplicationCmd *cmd); static void WalSndLoop(WalProposer *wp); static void XLogBroadcastWalProposer(WalProposer *wp); -static void XLogWalPropWrite(char *buf, Size nbytes, XLogRecPtr recptr); +static void XLogWalPropWrite(WalProposer *wp, char *buf, Size nbytes, XLogRecPtr recptr); static void XLogWalPropClose(XLogRecPtr recptr); static void @@ -1241,7 +1241,7 @@ WalProposerRecovery(Safekeeper *sk, TimeLineID timeline, XLogRecPtr startpos, XL rec_end_lsn = rec_start_lsn + len - XLOG_HDR_SIZE; /* write WAL to disk */ - XLogWalPropWrite(&buf[XLOG_HDR_SIZE], len - XLOG_HDR_SIZE, rec_start_lsn); + XLogWalPropWrite(sk->wp, &buf[XLOG_HDR_SIZE], len - XLOG_HDR_SIZE, rec_start_lsn); ereport(DEBUG1, (errmsg("Recover message %X/%X length %d", @@ -1283,11 +1283,24 @@ static XLogSegNo walpropSegNo = 0; * Write XLOG data to disk. */ static void -XLogWalPropWrite(char *buf, Size nbytes, XLogRecPtr recptr) +XLogWalPropWrite(WalProposer *wp, char *buf, Size nbytes, XLogRecPtr recptr) { int startoff; int byteswritten; + /* + * Apart from walproposer, basebackup LSN page is also written out by + * postgres itself which writes WAL only in pages, and in basebackup it is + * inherently dummy (only safekeepers have historic WAL). Update WAL buffers + * here to avoid dummy page overwriting correct one we download here. Ugly, + * but alternatives are about the same ugly. We won't need that if we switch + * to on-demand WAL download from safekeepers, without writing to disk. + * + * https://github.com/neondatabase/neon/issues/5749 + */ + if (!wp->config->syncSafekeepers) + XLogUpdateWalBuffers(buf, recptr, nbytes); + while (nbytes > 0) { int segbytes; diff --git a/poetry.lock b/poetry.lock index 0beae18cdf..58ab4e70f2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,98 +2,98 @@ [[package]] name = "aiohttp" -version = "3.8.5" +version = "3.8.6" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.6" files = [ - {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, - {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, - {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"}, - {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"}, - {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"}, - {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"}, - {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"}, - {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"}, - {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"}, - {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"}, - {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"}, - {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"}, - {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"}, - {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"}, - {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, - {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, + {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41d55fc043954cddbbd82503d9cc3f4814a40bcef30b3569bc7b5e34130718c1"}, + {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1d84166673694841d8953f0a8d0c90e1087739d24632fe86b1a08819168b4566"}, + {file = "aiohttp-3.8.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:253bf92b744b3170eb4c4ca2fa58f9c4b87aeb1df42f71d4e78815e6e8b73c9e"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fd194939b1f764d6bb05490987bfe104287bbf51b8d862261ccf66f48fb4096"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c5f938d199a6fdbdc10bbb9447496561c3a9a565b43be564648d81e1102ac22"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2817b2f66ca82ee699acd90e05c95e79bbf1dc986abb62b61ec8aaf851e81c93"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fa375b3d34e71ccccf172cab401cd94a72de7a8cc01847a7b3386204093bb47"}, + {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9de50a199b7710fa2904be5a4a9b51af587ab24c8e540a7243ab737b45844543"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e1d8cb0b56b3587c5c01de3bf2f600f186da7e7b5f7353d1bf26a8ddca57f965"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8e31e9db1bee8b4f407b77fd2507337a0a80665ad7b6c749d08df595d88f1cf5"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7bc88fc494b1f0311d67f29fee6fd636606f4697e8cc793a2d912ac5b19aa38d"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ec00c3305788e04bf6d29d42e504560e159ccaf0be30c09203b468a6c1ccd3b2"}, + {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad1407db8f2f49329729564f71685557157bfa42b48f4b93e53721a16eb813ed"}, + {file = "aiohttp-3.8.6-cp310-cp310-win32.whl", hash = "sha256:ccc360e87341ad47c777f5723f68adbb52b37ab450c8bc3ca9ca1f3e849e5fe2"}, + {file = "aiohttp-3.8.6-cp310-cp310-win_amd64.whl", hash = "sha256:93c15c8e48e5e7b89d5cb4613479d144fda8344e2d886cf694fd36db4cc86865"}, + {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e2f9cc8e5328f829f6e1fb74a0a3a939b14e67e80832975e01929e320386b34"}, + {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e6a00ffcc173e765e200ceefb06399ba09c06db97f401f920513a10c803604ca"}, + {file = "aiohttp-3.8.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:41bdc2ba359032e36c0e9de5a3bd00d6fb7ea558a6ce6b70acedf0da86458321"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14cd52ccf40006c7a6cd34a0f8663734e5363fd981807173faf3a017e202fec9"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2d5b785c792802e7b275c420d84f3397668e9d49ab1cb52bd916b3b3ffcf09ad"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1bed815f3dc3d915c5c1e556c397c8667826fbc1b935d95b0ad680787896a358"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96603a562b546632441926cd1293cfcb5b69f0b4159e6077f7c7dbdfb686af4d"}, + {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d76e8b13161a202d14c9584590c4df4d068c9567c99506497bdd67eaedf36403"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e3f1e3f1a1751bb62b4a1b7f4e435afcdade6c17a4fd9b9d43607cebd242924a"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:76b36b3124f0223903609944a3c8bf28a599b2cc0ce0be60b45211c8e9be97f8"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:a2ece4af1f3c967a4390c284797ab595a9f1bc1130ef8b01828915a05a6ae684"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:16d330b3b9db87c3883e565340d292638a878236418b23cc8b9b11a054aaa887"}, + {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:42c89579f82e49db436b69c938ab3e1559e5a4409eb8639eb4143989bc390f2f"}, + {file = "aiohttp-3.8.6-cp311-cp311-win32.whl", hash = "sha256:efd2fcf7e7b9d7ab16e6b7d54205beded0a9c8566cb30f09c1abe42b4e22bdcb"}, + {file = "aiohttp-3.8.6-cp311-cp311-win_amd64.whl", hash = "sha256:3b2ab182fc28e7a81f6c70bfbd829045d9480063f5ab06f6e601a3eddbbd49a0"}, + {file = "aiohttp-3.8.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fdee8405931b0615220e5ddf8cd7edd8592c606a8e4ca2a00704883c396e4479"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d25036d161c4fe2225d1abff2bd52c34ed0b1099f02c208cd34d8c05729882f0"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d791245a894be071d5ab04bbb4850534261a7d4fd363b094a7b9963e8cdbd31"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0cccd1de239afa866e4ce5c789b3032442f19c261c7d8a01183fd956b1935349"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f13f60d78224f0dace220d8ab4ef1dbc37115eeeab8c06804fec11bec2bbd07"}, + {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a9b5a0606faca4f6cc0d338359d6fa137104c337f489cd135bb7fbdbccb1e39"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:13da35c9ceb847732bf5c6c5781dcf4780e14392e5d3b3c689f6d22f8e15ae31"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:4d4cbe4ffa9d05f46a28252efc5941e0462792930caa370a6efaf491f412bc66"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:229852e147f44da0241954fc6cb910ba074e597f06789c867cb7fb0621e0ba7a"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:713103a8bdde61d13490adf47171a1039fd880113981e55401a0f7b42c37d071"}, + {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:45ad816b2c8e3b60b510f30dbd37fe74fd4a772248a52bb021f6fd65dff809b6"}, + {file = "aiohttp-3.8.6-cp36-cp36m-win32.whl", hash = "sha256:2b8d4e166e600dcfbff51919c7a3789ff6ca8b3ecce16e1d9c96d95dd569eb4c"}, + {file = "aiohttp-3.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:0912ed87fee967940aacc5306d3aa8ba3a459fcd12add0b407081fbefc931e53"}, + {file = "aiohttp-3.8.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e2a988a0c673c2e12084f5e6ba3392d76c75ddb8ebc6c7e9ead68248101cd446"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf3fd9f141700b510d4b190094db0ce37ac6361a6806c153c161dc6c041ccda"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3161ce82ab85acd267c8f4b14aa226047a6bee1e4e6adb74b798bd42c6ae1f80"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95fc1bf33a9a81469aa760617b5971331cdd74370d1214f0b3109272c0e1e3c"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c43ecfef7deaf0617cee936836518e7424ee12cb709883f2c9a1adda63cc460"}, + {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca80e1b90a05a4f476547f904992ae81eda5c2c85c66ee4195bb8f9c5fb47f28"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:90c72ebb7cb3a08a7f40061079817133f502a160561d0675b0a6adf231382c92"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bb54c54510e47a8c7c8e63454a6acc817519337b2b78606c4e840871a3e15349"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:de6a1c9f6803b90e20869e6b99c2c18cef5cc691363954c93cb9adeb26d9f3ae"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:a3628b6c7b880b181a3ae0a0683698513874df63783fd89de99b7b7539e3e8a8"}, + {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:fc37e9aef10a696a5a4474802930079ccfc14d9f9c10b4662169671ff034b7df"}, + {file = "aiohttp-3.8.6-cp37-cp37m-win32.whl", hash = "sha256:f8ef51e459eb2ad8e7a66c1d6440c808485840ad55ecc3cafefadea47d1b1ba2"}, + {file = "aiohttp-3.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:b2fe42e523be344124c6c8ef32a011444e869dc5f883c591ed87f84339de5976"}, + {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9e2ee0ac5a1f5c7dd3197de309adfb99ac4617ff02b0603fd1e65b07dc772e4b"}, + {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01770d8c04bd8db568abb636c1fdd4f7140b284b8b3e0b4584f070180c1e5c62"}, + {file = "aiohttp-3.8.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c68330a59506254b556b99a91857428cab98b2f84061260a67865f7f52899f5"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89341b2c19fb5eac30c341133ae2cc3544d40d9b1892749cdd25892bbc6ac951"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71783b0b6455ac8f34b5ec99d83e686892c50498d5d00b8e56d47f41b38fbe04"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f628dbf3c91e12f4d6c8b3f092069567d8eb17814aebba3d7d60c149391aee3a"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b04691bc6601ef47c88f0255043df6f570ada1a9ebef99c34bd0b72866c217ae"}, + {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee912f7e78287516df155f69da575a0ba33b02dd7c1d6614dbc9463f43066e3"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9c19b26acdd08dd239e0d3669a3dddafd600902e37881f13fbd8a53943079dbc"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:99c5ac4ad492b4a19fc132306cd57075c28446ec2ed970973bbf036bcda1bcc6"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f0f03211fd14a6a0aed2997d4b1c013d49fb7b50eeb9ffdf5e51f23cfe2c77fa"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:8d399dade330c53b4106160f75f55407e9ae7505263ea86f2ccca6bfcbdb4921"}, + {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ec4fd86658c6a8964d75426517dc01cbf840bbf32d055ce64a9e63a40fd7b771"}, + {file = "aiohttp-3.8.6-cp38-cp38-win32.whl", hash = "sha256:33164093be11fcef3ce2571a0dccd9041c9a93fa3bde86569d7b03120d276c6f"}, + {file = "aiohttp-3.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:bdf70bfe5a1414ba9afb9d49f0c912dc524cf60141102f3a11143ba3d291870f"}, + {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d52d5dc7c6682b720280f9d9db41d36ebe4791622c842e258c9206232251ab2b"}, + {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ac39027011414dbd3d87f7edb31680e1f430834c8cef029f11c66dad0670aa5"}, + {file = "aiohttp-3.8.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3f5c7ce535a1d2429a634310e308fb7d718905487257060e5d4598e29dc17f0b"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b30e963f9e0d52c28f284d554a9469af073030030cef8693106d918b2ca92f54"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:918810ef188f84152af6b938254911055a72e0f935b5fbc4c1a4ed0b0584aed1"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:002f23e6ea8d3dd8d149e569fd580c999232b5fbc601c48d55398fbc2e582e8c"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fcf3eabd3fd1a5e6092d1242295fa37d0354b2eb2077e6eb670accad78e40e1"}, + {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:255ba9d6d5ff1a382bb9a578cd563605aa69bec845680e21c44afc2670607a95"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d67f8baed00870aa390ea2590798766256f31dc5ed3ecc737debb6e97e2ede78"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:86f20cee0f0a317c76573b627b954c412ea766d6ada1a9fcf1b805763ae7feeb"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:39a312d0e991690ccc1a61f1e9e42daa519dcc34ad03eb6f826d94c1190190dd"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e827d48cf802de06d9c935088c2924e3c7e7533377d66b6f31ed175c1620e05e"}, + {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bd111d7fc5591ddf377a408ed9067045259ff2770f37e2d94e6478d0f3fc0c17"}, + {file = "aiohttp-3.8.6-cp39-cp39-win32.whl", hash = "sha256:caf486ac1e689dda3502567eb89ffe02876546599bbf915ec94b1fa424eeffd4"}, + {file = "aiohttp-3.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:3f0e27e5b733803333bb2371249f41cf42bae8884863e8e8965ec69bebe53132"}, + {file = "aiohttp-3.8.6.tar.gz", hash = "sha256:b0cf2a4501bff9330a8a5248b4ce951851e415bdcce9dc158e76cfd55e15085c"}, ] [package.dependencies] @@ -2719,4 +2719,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "74649cf47c52f21b01b096a42044750b1c9677576b405be0489c2909127a9bf1" +content-hash = "0834e5cb69e5457741d4f476c3e49a4dc83598b5730685c8755da651b96ad3ec" diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 92498d3ecd..0ec7efd316 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -51,6 +51,7 @@ serde_json.workspace = true sha2.workspace = true socket2.workspace = true sync_wrapper.workspace = true +task-local-extensions.workspace = true thiserror.workspace = true tls-listener.workspace = true tokio-postgres.workspace = true diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index 7dc304d7ac..9fe9c26f0c 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -1,6 +1,10 @@ //! User credentials used in authentication. -use crate::{auth::password_hack::parse_endpoint_param, error::UserFacingError}; +use crate::{ + auth::password_hack::parse_endpoint_param, + error::UserFacingError, + proxy::{neon_options, NUM_CONNECTION_ACCEPTED_BY_SNI}, +}; use itertools::Itertools; use pq_proto::StartupMessageParams; use std::collections::HashSet; @@ -38,6 +42,8 @@ pub struct ClientCredentials<'a> { pub user: &'a str, // TODO: this is a severe misnomer! We should think of a new name ASAP. pub project: Option, + + pub cache_key: String, } impl ClientCredentials<'_> { @@ -53,6 +59,7 @@ impl<'a> ClientCredentials<'a> { ClientCredentials { user: "", project: None, + cache_key: "".to_string(), } } @@ -119,8 +126,34 @@ impl<'a> ClientCredentials<'a> { .transpose()?; info!(user, project = project.as_deref(), "credentials"); + if sni.is_some() { + info!("Connection with sni"); + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["sni"]) + .inc(); + } else if project.is_some() { + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["no_sni"]) + .inc(); + info!("Connection without sni"); + } else { + NUM_CONNECTION_ACCEPTED_BY_SNI + .with_label_values(&["password_hack"]) + .inc(); + info!("Connection with password hack"); + } - Ok(Self { user, project }) + let cache_key = format!( + "{}{}", + project.as_deref().unwrap_or(""), + neon_options(params).unwrap_or("".to_string()) + ); + + Ok(Self { + user, + project, + cache_key, + }) } } @@ -176,6 +209,7 @@ mod tests { let creds = ClientCredentials::parse(&options, sni, common_names)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project.as_deref(), Some("foo")); + assert_eq!(creds.cache_key, "foo"); Ok(()) } @@ -303,4 +337,23 @@ mod tests { _ => panic!("bad error: {err:?}"), } } + + #[test] + fn parse_neon_options() -> anyhow::Result<()> { + let options = StartupMessageParams::new([ + ("user", "john_doe"), + ("options", "neon_lsn:0/2 neon_endpoint_type:read_write"), + ]); + + let sni = Some("project.localhost"); + let common_names = Some(["localhost".into()].into()); + let creds = ClientCredentials::parse(&options, sni, common_names)?; + assert_eq!(creds.project.as_deref(), Some("project")); + assert_eq!( + creds.cache_key, + "projectneon_endpoint_type:read_write neon_lsn:0/2" + ); + + Ok(()) + } } diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 28d0d95c5b..570cf0943a 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -4,6 +4,7 @@ use proxy::config::AuthenticationConfig; use proxy::config::HttpConfig; use proxy::console; use proxy::http; +use proxy::rate_limiter::RateLimiterConfig; use proxy::usage_metrics; use anyhow::bail; @@ -80,6 +81,9 @@ struct ProxyCliArgs { /// cache for `wake_compute` api method (use `size=0` to disable) #[clap(long, default_value = config::CacheOptions::DEFAULT_OPTIONS_NODE_INFO)] wake_compute_cache: String, + /// lock for `wake_compute` api method. example: "shards=32,permits=4,epoch=10m,timeout=1s". (use `permits=0` to disable). + #[clap(long, default_value = config::WakeComputeLockOptions::DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK)] + wake_compute_lock: String, /// Allow self-signed certificates for compute nodes (for testing) #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] allow_self_signed_compute: bool, @@ -92,6 +96,20 @@ struct ProxyCliArgs { /// Require that all incoming requests have a Proxy Protocol V2 packet **and** have an IP address associated. #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] require_client_ip: bool, + /// Disable dynamic rate limiter and store the metrics to ensure its production behaviour. + #[clap(long, default_value_t = true, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] + disable_dynamic_rate_limiter: bool, + /// Rate limit algorithm. Makes sense only if `disable_rate_limiter` is `false`. + #[clap(value_enum, long, default_value_t = proxy::rate_limiter::RateLimitAlgorithm::Aimd)] + rate_limit_algorithm: proxy::rate_limiter::RateLimitAlgorithm, + /// Timeout for rate limiter. If it didn't manage to aquire a permit in this time, it will return an error. + #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] + rate_limiter_timeout: tokio::time::Duration, + /// Initial limit for dynamic rate limiter. Makes sense only if `rate_limit_algorithm` is *not* `None`. + #[clap(long, default_value_t = 100)] + initial_limit: usize, + #[clap(flatten)] + aimd_config: proxy::rate_limiter::AimdConfig, } #[tokio::main] @@ -210,6 +228,13 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { and metric-collection-interval must be specified" ), }; + let rate_limiter_config = RateLimiterConfig { + disable: args.disable_dynamic_rate_limiter, + algorithm: args.rate_limit_algorithm, + timeout: args.rate_limiter_timeout, + initial_limit: args.initial_limit, + aimd_config: Some(args.aimd_config), + }; let auth_backend = match &args.auth_backend { AuthBackend::Console => { @@ -220,10 +245,23 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { node_info: console::caches::NodeInfoCache::new("node_info_cache", size, ttl), })); - let url = args.auth_endpoint.parse()?; - let endpoint = http::Endpoint::new(url, http::new_client()); + let config::WakeComputeLockOptions { + shards, + permits, + epoch, + timeout, + } = args.wake_compute_lock.parse()?; + info!(permits, shards, ?epoch, "Using NodeLocks (wake_compute)"); + let locks = Box::leak(Box::new( + console::locks::ApiLocks::new("wake_compute_lock", permits, shards, timeout) + .unwrap(), + )); + tokio::spawn(locks.garbage_collect_worker(epoch)); - let api = console::provider::neon::Api::new(endpoint, caches); + let url = args.auth_endpoint.parse()?; + let endpoint = http::Endpoint::new(url, http::new_client(rate_limiter_config)); + + let api = console::provider::neon::Api::new(endpoint, caches, locks); auth::BackendType::Console(Cow::Owned(api), ()) } AuthBackend::Postgres => { diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index e96b79ed92..0741ad0623 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -3,6 +3,7 @@ use crate::{ cancellation::CancelClosure, console::errors::WakeComputeError, error::{io_error, UserFacingError}, + proxy::is_neon_param, }; use futures::{FutureExt, TryFutureExt}; use itertools::Itertools; @@ -247,6 +248,7 @@ impl ConnCfg { // connect_raw() will not use TLS if sslmode is "disable" let (client, connection) = self.0.connect_raw(stream, tls).await?; + tracing::Span::current().record("pid", &tracing::field::display(client.get_process_id())); let stream = connection.stream.into_inner(); info!( @@ -278,7 +280,7 @@ fn filtered_options(params: &StartupMessageParams) -> Option { #[allow(unstable_name_collisions)] let options: String = params .options_raw()? - .filter(|opt| parse_endpoint_param(opt).is_none()) + .filter(|opt| parse_endpoint_param(opt).is_none() && !is_neon_param(opt)) .intersperse(" ") // TODO: use impl from std once it's stabilized .collect(); @@ -313,5 +315,11 @@ mod tests { let params = StartupMessageParams::new([("options", "project = foo")]); assert_eq!(filtered_options(¶ms).as_deref(), Some("project = foo")); + + let params = StartupMessageParams::new([( + "options", + "project = foo neon_endpoint_type:read_write neon_lsn:0/2", + )]); + assert_eq!(filtered_options(¶ms).as_deref(), Some("project = foo")); } } diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 9607ecd153..bd00123905 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -264,6 +264,79 @@ impl FromStr for CacheOptions { } } +/// Helper for cmdline cache options parsing. +pub struct WakeComputeLockOptions { + /// The number of shards the lock map should have + pub shards: usize, + /// The number of allowed concurrent requests for each endpoitn + pub permits: usize, + /// Garbage collection epoch + pub epoch: Duration, + /// Lock timeout + pub timeout: Duration, +} + +impl WakeComputeLockOptions { + /// Default options for [`crate::console::provider::ApiLocks`]. + pub const DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK: &'static str = "permits=0"; + + // pub const DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK: &'static str = "shards=32,permits=4,epoch=10m,timeout=1s"; + + /// Parse lock options passed via cmdline. + /// Example: [`Self::DEFAULT_OPTIONS_WAKE_COMPUTE_LOCK`]. + fn parse(options: &str) -> anyhow::Result { + let mut shards = None; + let mut permits = None; + let mut epoch = None; + let mut timeout = None; + + for option in options.split(',') { + let (key, value) = option + .split_once('=') + .with_context(|| format!("bad key-value pair: {option}"))?; + + match key { + "shards" => shards = Some(value.parse()?), + "permits" => permits = Some(value.parse()?), + "epoch" => epoch = Some(humantime::parse_duration(value)?), + "timeout" => timeout = Some(humantime::parse_duration(value)?), + unknown => bail!("unknown key: {unknown}"), + } + } + + // these dont matter if lock is disabled + if let Some(0) = permits { + timeout = Some(Duration::default()); + epoch = Some(Duration::default()); + shards = Some(2); + } + + let out = Self { + shards: shards.context("missing `shards`")?, + permits: permits.context("missing `permits`")?, + epoch: epoch.context("missing `epoch`")?, + timeout: timeout.context("missing `timeout`")?, + }; + + ensure!(out.shards > 1, "shard count must be > 1"); + ensure!( + out.shards.is_power_of_two(), + "shard count must be a power of two" + ); + + Ok(out) + } +} + +impl FromStr for WakeComputeLockOptions { + type Err = anyhow::Error; + + fn from_str(options: &str) -> Result { + let error = || format!("failed to parse cache lock options '{options}'"); + Self::parse(options).with_context(error) + } +} + #[cfg(test)] mod tests { use super::*; @@ -288,4 +361,42 @@ mod tests { Ok(()) } + + #[test] + fn test_parse_lock_options() -> anyhow::Result<()> { + let WakeComputeLockOptions { + epoch, + permits, + shards, + timeout, + } = "shards=32,permits=4,epoch=10m,timeout=1s".parse()?; + assert_eq!(epoch, Duration::from_secs(10 * 60)); + assert_eq!(timeout, Duration::from_secs(1)); + assert_eq!(shards, 32); + assert_eq!(permits, 4); + + let WakeComputeLockOptions { + epoch, + permits, + shards, + timeout, + } = "epoch=60s,shards=16,timeout=100ms,permits=8".parse()?; + assert_eq!(epoch, Duration::from_secs(60)); + assert_eq!(timeout, Duration::from_millis(100)); + assert_eq!(shards, 16); + assert_eq!(permits, 8); + + let WakeComputeLockOptions { + epoch, + permits, + shards, + timeout, + } = "permits=0".parse()?; + assert_eq!(epoch, Duration::ZERO); + assert_eq!(timeout, Duration::ZERO); + assert_eq!(shards, 2); + assert_eq!(permits, 0); + + Ok(()) + } } diff --git a/proxy/src/console.rs b/proxy/src/console.rs index 0e5eaaf845..6da627389e 100644 --- a/proxy/src/console.rs +++ b/proxy/src/console.rs @@ -13,5 +13,10 @@ pub mod caches { pub use super::provider::{ApiCaches, NodeInfoCache}; } +/// Various cache-related types. +pub mod locks { + pub use super::provider::ApiLocks; +} + /// Console's management API. pub mod mgmt; diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index 32c3467092..54bcd1f081 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -8,7 +8,13 @@ use crate::{ compute, scram, }; use async_trait::async_trait; -use std::sync::Arc; +use dashmap::DashMap; +use std::{sync::Arc, time::Duration}; +use tokio::{ + sync::{OwnedSemaphorePermit, Semaphore}, + time::Instant, +}; +use tracing::info; pub mod errors { use crate::{ @@ -149,6 +155,9 @@ pub mod errors { #[error(transparent)] ApiError(ApiError), + + #[error("Timeout waiting to acquire wake compute lock")] + TimeoutError, } // This allows more useful interactions than `#[from]`. @@ -158,6 +167,17 @@ pub mod errors { } } + impl From for WakeComputeError { + fn from(_: tokio::sync::AcquireError) -> Self { + WakeComputeError::TimeoutError + } + } + impl From for WakeComputeError { + fn from(_: tokio::time::error::Elapsed) -> Self { + WakeComputeError::TimeoutError + } + } + impl UserFacingError for WakeComputeError { fn to_string_client(&self) -> String { use WakeComputeError::*; @@ -167,6 +187,8 @@ pub mod errors { BadComputeAddress(_) => REQUEST_FAILED.to_owned(), // However, API might return a meaningful error. ApiError(e) => e.to_string_client(), + + TimeoutError => "timeout while acquiring the compute resource lock".to_owned(), } } } @@ -178,6 +200,7 @@ pub struct ConsoleReqExtra<'a> { pub session_id: uuid::Uuid, /// Name of client application, if set. pub application_name: Option<&'a str>, + pub options: Option<&'a str>, } /// Auth secret which is managed by the cloud. @@ -232,3 +255,145 @@ pub struct ApiCaches { /// Cache for the `wake_compute` API method. pub node_info: NodeInfoCache, } + +/// Various caches for [`console`](super). +pub struct ApiLocks { + name: &'static str, + node_locks: DashMap, Arc>, + permits: usize, + timeout: Duration, + registered: prometheus::IntCounter, + unregistered: prometheus::IntCounter, + reclamation_lag: prometheus::Histogram, + lock_acquire_lag: prometheus::Histogram, +} + +impl ApiLocks { + pub fn new( + name: &'static str, + permits: usize, + shards: usize, + timeout: Duration, + ) -> prometheus::Result { + let registered = prometheus::IntCounter::with_opts( + prometheus::Opts::new( + "semaphores_registered", + "Number of semaphores registered in this api lock", + ) + .namespace(name), + )?; + prometheus::register(Box::new(registered.clone()))?; + let unregistered = prometheus::IntCounter::with_opts( + prometheus::Opts::new( + "semaphores_unregistered", + "Number of semaphores unregistered in this api lock", + ) + .namespace(name), + )?; + prometheus::register(Box::new(unregistered.clone()))?; + let reclamation_lag = prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "reclamation_lag_seconds", + "Time it takes to reclaim unused semaphores in the api lock", + ) + .namespace(name) + // 1us -> 65ms + // benchmarks on my mac indicate it's usually in the range of 256us and 512us + .buckets(prometheus::exponential_buckets(1e-6, 2.0, 16)?), + )?; + prometheus::register(Box::new(reclamation_lag.clone()))?; + let lock_acquire_lag = prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "semaphore_acquire_seconds", + "Time it takes to reclaim unused semaphores in the api lock", + ) + .namespace(name) + // 0.1ms -> 6s + .buckets(prometheus::exponential_buckets(1e-4, 2.0, 16)?), + )?; + prometheus::register(Box::new(lock_acquire_lag.clone()))?; + + Ok(Self { + name, + node_locks: DashMap::with_shard_amount(shards), + permits, + timeout, + lock_acquire_lag, + registered, + unregistered, + reclamation_lag, + }) + } + + pub async fn get_wake_compute_permit( + &self, + key: &Arc, + ) -> Result { + if self.permits == 0 { + return Ok(WakeComputePermit { permit: None }); + } + let now = Instant::now(); + let semaphore = { + // get fast path + if let Some(semaphore) = self.node_locks.get(key) { + semaphore.clone() + } else { + self.node_locks + .entry(key.clone()) + .or_insert_with(|| { + self.registered.inc(); + Arc::new(Semaphore::new(self.permits)) + }) + .clone() + } + }; + let permit = tokio::time::timeout_at(now + self.timeout, semaphore.acquire_owned()).await; + + self.lock_acquire_lag + .observe((Instant::now() - now).as_secs_f64()); + + Ok(WakeComputePermit { + permit: Some(permit??), + }) + } + + pub async fn garbage_collect_worker(&self, epoch: std::time::Duration) { + if self.permits == 0 { + return; + } + + let mut interval = tokio::time::interval(epoch / (self.node_locks.shards().len()) as u32); + loop { + for (i, shard) in self.node_locks.shards().iter().enumerate() { + interval.tick().await; + // temporary lock a single shard and then clear any semaphores that aren't currently checked out + // race conditions: if strong_count == 1, there's no way that it can increase while the shard is locked + // therefore releasing it is safe from race conditions + info!( + name = self.name, + shard = i, + "performing epoch reclamation on api lock" + ); + let mut lock = shard.write(); + let timer = self.reclamation_lag.start_timer(); + let count = lock + .extract_if(|_, semaphore| Arc::strong_count(semaphore.get_mut()) == 1) + .count(); + drop(lock); + self.unregistered.inc_by(count as u64); + timer.observe_duration() + } + } + } +} + +pub struct WakeComputePermit { + // None if the lock is disabled + permit: Option, +} + +impl WakeComputePermit { + pub fn should_check_cache(&self) -> bool { + self.permit.is_some() + } +} diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 927fea0a13..0dc7c71534 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -3,12 +3,12 @@ use super::{ super::messages::{ConsoleError, GetRoleSecret, WakeCompute}, errors::{ApiError, GetAuthInfoError, WakeComputeError}, - ApiCaches, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, + ApiCaches, ApiLocks, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, }; use crate::{auth::ClientCredentials, compute, http, scram}; use async_trait::async_trait; use futures::TryFutureExt; -use std::net::SocketAddr; +use std::{net::SocketAddr, sync::Arc}; use tokio::time::Instant; use tokio_postgres::config::SslMode; use tracing::{error, info, info_span, warn, Instrument}; @@ -17,12 +17,17 @@ use tracing::{error, info, info_span, warn, Instrument}; pub struct Api { endpoint: http::Endpoint, caches: &'static ApiCaches, + locks: &'static ApiLocks, jwt: String, } impl Api { /// Construct an API object containing the auth parameters. - pub fn new(endpoint: http::Endpoint, caches: &'static ApiCaches) -> Self { + pub fn new( + endpoint: http::Endpoint, + caches: &'static ApiCaches, + locks: &'static ApiLocks, + ) -> Self { let jwt: String = match std::env::var("NEON_PROXY_TO_CONTROLPLANE_TOKEN") { Ok(v) => v, Err(_) => "".to_string(), @@ -30,6 +35,7 @@ impl Api { Self { endpoint, caches, + locks, jwt, } } @@ -99,6 +105,7 @@ impl Api { .query(&[ ("application_name", extra.application_name), ("project", Some(project)), + ("options", extra.options), ]) .build()?; @@ -151,7 +158,7 @@ impl super::Api for Api { extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials, ) -> Result { - let key = creds.project().expect("impossible"); + let key: &str = &creds.cache_key; // Every time we do a wakeup http request, the compute node will stay up // for some time (highly depends on the console's scale-to-zero policy); @@ -162,9 +169,22 @@ impl super::Api for Api { return Ok(cached); } + let key: Arc = key.into(); + + let permit = self.locks.get_wake_compute_permit(&key).await?; + + // after getting back a permit - it's possible the cache was filled + // double check + if permit.should_check_cache() { + if let Some(cached) = self.caches.node_info.get(&key) { + info!(key = &*key, "found cached compute node info"); + return Ok(cached); + } + } + let node = self.do_wake_compute(extra, creds).await?; - let (_, cached) = self.caches.node_info.insert(key.into(), node); - info!(key = key, "created a cache entry for compute node info"); + let (_, cached) = self.caches.node_info.insert(key.clone(), node); + info!(key = &*key, "created a cache entry for compute node info"); Ok(cached) } diff --git a/proxy/src/http.rs b/proxy/src/http.rs index 14a9072a45..159b949da3 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -13,13 +13,13 @@ pub use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; use tokio::time::Instant; use tracing::trace; -use crate::url::ApiUrl; +use crate::{rate_limiter, url::ApiUrl}; use reqwest_middleware::RequestBuilder; /// This is the preferred way to create new http clients, /// because it takes care of observability (OpenTelemetry). /// We deliberately don't want to replace this with a public static. -pub fn new_client() -> ClientWithMiddleware { +pub fn new_client(rate_limiter_config: rate_limiter::RateLimiterConfig) -> ClientWithMiddleware { let client = reqwest::ClientBuilder::new() .dns_resolver(Arc::new(GaiResolver::default())) .connection_verbose(true) @@ -28,6 +28,7 @@ pub fn new_client() -> ClientWithMiddleware { reqwest_middleware::ClientBuilder::new(client) .with(reqwest_tracing::TracingMiddleware::default()) + .with(rate_limiter::Limiter::new(rate_limiter_config)) .build() } diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index 803b278482..a22600cbb3 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(clippy::undocumented_unsafe_blocks)] + use std::convert::Infallible; use anyhow::{bail, Context}; @@ -17,6 +19,7 @@ pub mod logging; pub mod parse; pub mod protocol2; pub mod proxy; +pub mod rate_limiter; pub mod sasl; pub mod scram; pub mod serverless; diff --git a/proxy/src/parse.rs b/proxy/src/parse.rs index cbd48d91e9..0d03574901 100644 --- a/proxy/src/parse.rs +++ b/proxy/src/parse.rs @@ -3,10 +3,9 @@ use std::ffi::CStr; pub fn split_cstr(bytes: &[u8]) -> Option<(&CStr, &[u8])> { - let pos = bytes.iter().position(|&x| x == 0)?; - let (cstr, other) = bytes.split_at(pos + 1); - // SAFETY: we've already checked that there's a terminator - Some((unsafe { CStr::from_bytes_with_nul_unchecked(cstr) }, other)) + let cstr = CStr::from_bytes_until_nul(bytes).ok()?; + let (_, other) = bytes.split_at(cstr.to_bytes_with_nul().len()); + Some((cstr, other)) } /// See . diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 884aae1651..b05b902303 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -15,10 +15,15 @@ use crate::{ use anyhow::{bail, Context}; use async_trait::async_trait; use futures::TryFutureExt; +use itertools::Itertools; use metrics::{exponential_buckets, register_int_counter_vec, IntCounterVec}; -use once_cell::sync::Lazy; +use once_cell::sync::{Lazy, OnceCell}; use pq_proto::{BeMessage as Be, FeStartupPacket, StartupMessageParams}; -use prometheus::{register_histogram_vec, HistogramVec}; +use prometheus::{ + register_histogram, register_histogram_vec, register_int_gauge_vec, Histogram, HistogramVec, + IntGaugeVec, +}; +use regex::Regex; use std::{error::Error, io, ops::ControlFlow, sync::Arc, time::Instant}; use tokio::{ io::{AsyncRead, AsyncWrite, AsyncWriteExt}, @@ -105,6 +110,34 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { .unwrap() }); +pub static RATE_LIMITER_ACQUIRE_LATENCY: Lazy = Lazy::new(|| { + register_histogram!( + "semaphore_control_plane_token_acquire_seconds", + "Time it took for proxy to establish a connection to the compute endpoint", + // largest bucket = 2^16 * 0.5ms = 32s + exponential_buckets(0.0005, 2.0, 16).unwrap(), + ) + .unwrap() +}); + +pub static RATE_LIMITER_LIMIT: Lazy = Lazy::new(|| { + register_int_gauge_vec!( + "semaphore_control_plane_limit", + "Current limit of the semaphore control plane", + &["limit"], // 2 counters + ) + .unwrap() +}); + +pub static NUM_CONNECTION_ACCEPTED_BY_SNI: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "proxy_accepted_connections_by_sni", + "Number of connections (per sni).", + &["kind"], + ) + .unwrap() +}); + pub struct LatencyTimer { // time since the stopwatch was started start: Option, @@ -481,7 +514,7 @@ pub fn invalidate_cache(node_info: console::CachedNodeInfo) -> compute::ConnCfg } /// Try to connect to the compute node once. -#[tracing::instrument(name = "connect_once", skip_all)] +#[tracing::instrument(name = "connect_once", fields(pid = tracing::field::Empty), skip_all)] async fn connect_to_compute_once( node_info: &console::CachedNodeInfo, timeout: time::Duration, @@ -568,6 +601,7 @@ fn report_error(e: &WakeComputeError, retry: bool) { "api_console_other_server_error" } WakeComputeError::ApiError(ApiError::Console { .. }) => "api_console_other_error", + WakeComputeError::TimeoutError => "timeout_error", }; NUM_WAKEUP_FAILURES.with_label_values(&[retry, kind]).inc(); } @@ -881,9 +915,12 @@ impl Client<'_, S> { allow_self_signed_compute, } = self; + let console_options = neon_options(params); + let extra = console::ConsoleReqExtra { session_id, // aka this connection's id application_name: params.get("application_name"), + options: console_options.as_deref(), }; let mut latency_timer = LatencyTimer::new(mode.protocol_label()); @@ -945,3 +982,27 @@ impl Client<'_, S> { proxy_pass(stream, node.stream, &aux).await } } + +pub fn neon_options(params: &StartupMessageParams) -> Option { + #[allow(unstable_name_collisions)] + let options: String = params + .options_raw()? + .filter(|opt| is_neon_param(opt)) + .sorted() // we sort it to use as cache key + .intersperse(" ") // TODO: use impl from std once it's stabilized + .collect(); + + // Don't even bother with empty options. + if options.is_empty() { + return None; + } + + Some(options) +} + +pub fn is_neon_param(bytes: &str) -> bool { + static RE: OnceCell = OnceCell::new(); + RE.get_or_init(|| Regex::new(r"^neon_\w+:").unwrap()); + + RE.get().unwrap().is_match(bytes) +} diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index 142c32fb84..3ae4df46ef 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -440,6 +440,7 @@ fn helper_create_connect_info( let extra = console::ConsoleReqExtra { session_id: uuid::Uuid::new_v4(), application_name: Some("TEST"), + options: None, }; let creds = auth::BackendType::Test(mechanism); (cache, extra, creds) diff --git a/proxy/src/rate_limiter.rs b/proxy/src/rate_limiter.rs new file mode 100644 index 0000000000..5622c44a68 --- /dev/null +++ b/proxy/src/rate_limiter.rs @@ -0,0 +1,6 @@ +mod aimd; +mod limit_algorithm; +mod limiter; +pub use aimd::Aimd; +pub use limit_algorithm::{AimdConfig, Fixed, RateLimitAlgorithm, RateLimiterConfig}; +pub use limiter::Limiter; diff --git a/proxy/src/rate_limiter/aimd.rs b/proxy/src/rate_limiter/aimd.rs new file mode 100644 index 0000000000..c6c532ae53 --- /dev/null +++ b/proxy/src/rate_limiter/aimd.rs @@ -0,0 +1,199 @@ +use std::usize; + +use async_trait::async_trait; + +use super::limit_algorithm::{AimdConfig, LimitAlgorithm, Sample}; + +use super::limiter::Outcome; + +/// Loss-based congestion avoidance. +/// +/// Additive-increase, multiplicative decrease. +/// +/// Adds available currency when: +/// 1. no load-based errors are observed, and +/// 2. the utilisation of the current limit is high. +/// +/// Reduces available concurrency by a factor when load-based errors are detected. +pub struct Aimd { + min_limit: usize, + max_limit: usize, + decrease_factor: f32, + increase_by: usize, + min_utilisation_threshold: f32, +} + +impl Aimd { + pub fn new(config: AimdConfig) -> Self { + Self { + min_limit: config.aimd_min_limit, + max_limit: config.aimd_max_limit, + decrease_factor: config.aimd_decrease_factor, + increase_by: config.aimd_increase_by, + min_utilisation_threshold: config.aimd_min_utilisation_threshold, + } + } + + pub fn decrease_factor(self, factor: f32) -> Self { + assert!((0.5..1.0).contains(&factor)); + Self { + decrease_factor: factor, + ..self + } + } + + pub fn increase_by(self, increase: usize) -> Self { + assert!(increase > 0); + Self { + increase_by: increase, + ..self + } + } + + pub fn with_max_limit(self, max: usize) -> Self { + assert!(max > 0); + Self { + max_limit: max, + ..self + } + } + + /// A threshold below which the limit won't be increased. 0.5 = 50%. + pub fn with_min_utilisation_threshold(self, min_util: f32) -> Self { + assert!(min_util > 0. && min_util < 1.); + Self { + min_utilisation_threshold: min_util, + ..self + } + } +} + +#[async_trait] +impl LimitAlgorithm for Aimd { + async fn update(&mut self, old_limit: usize, sample: Sample) -> usize { + use Outcome::*; + match sample.outcome { + Success => { + let utilisation = sample.in_flight as f32 / old_limit as f32; + + if utilisation > self.min_utilisation_threshold { + let limit = old_limit + self.increase_by; + limit.clamp(self.min_limit, self.max_limit) + } else { + old_limit + } + } + Overload => { + let limit = old_limit as f32 * self.decrease_factor; + + // Floor instead of round, so the limit reduces even with small numbers. + // E.g. round(2 * 0.9) = 2, but floor(2 * 0.9) = 1 + let limit = limit.floor() as usize; + + limit.clamp(self.min_limit, self.max_limit) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use tokio::sync::Notify; + + use super::*; + + use crate::rate_limiter::{Limiter, RateLimiterConfig}; + + #[tokio::test] + async fn should_decrease_limit_on_overload() { + let config = RateLimiterConfig { + initial_limit: 10, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let release_notifier = Arc::new(Notify::new()); + + let limiter = Limiter::new(config).with_release_notifier(release_notifier.clone()); + + let token = limiter.try_acquire().unwrap(); + limiter.release(token, Some(Outcome::Overload)).await; + release_notifier.notified().await; + assert_eq!(limiter.state().limit(), 5, "overload: decrease"); + } + + #[tokio::test] + async fn should_increase_limit_on_success_when_using_gt_util_threshold() { + let config = RateLimiterConfig { + initial_limit: 4, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + aimd_min_utilisation_threshold: 0.5, + aimd_increase_by: 1, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + let _token = limiter.try_acquire().unwrap(); + let _token = limiter.try_acquire().unwrap(); + + limiter.release(token, Some(Outcome::Success)).await; + assert_eq!(limiter.state().limit(), 5, "success: increase"); + } + + #[tokio::test] + async fn should_not_change_limit_on_success_when_using_lt_util_threshold() { + let config = RateLimiterConfig { + initial_limit: 4, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + aimd_min_utilisation_threshold: 0.5, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + + limiter.release(token, Some(Outcome::Success)).await; + assert_eq!( + limiter.state().limit(), + 4, + "success: ignore when < half limit" + ); + } + + #[tokio::test] + async fn should_not_change_limit_when_no_outcome() { + let config = RateLimiterConfig { + initial_limit: 10, + aimd_config: Some(AimdConfig { + aimd_decrease_factor: 0.5, + aimd_min_utilisation_threshold: 0.5, + ..Default::default() + }), + disable: false, + ..Default::default() + }; + + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + limiter.release(token, None).await; + assert_eq!(limiter.state().limit(), 10, "ignore"); + } +} diff --git a/proxy/src/rate_limiter/limit_algorithm.rs b/proxy/src/rate_limiter/limit_algorithm.rs new file mode 100644 index 0000000000..5cd2d5ebb7 --- /dev/null +++ b/proxy/src/rate_limiter/limit_algorithm.rs @@ -0,0 +1,98 @@ +//! Algorithms for controlling concurrency limits. +use async_trait::async_trait; +use std::time::Duration; + +use super::{limiter::Outcome, Aimd}; + +/// An algorithm for controlling a concurrency limit. +#[async_trait] +pub trait LimitAlgorithm: Send + Sync + 'static { + /// Update the concurrency limit in response to a new job completion. + async fn update(&mut self, old_limit: usize, sample: Sample) -> usize; +} + +/// The result of a job (or jobs), including the [Outcome] (loss) and latency (delay). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Sample { + pub(crate) latency: Duration, + /// Jobs in flight when the sample was taken. + pub(crate) in_flight: usize, + pub(crate) outcome: Outcome, +} + +#[derive(Clone, Copy, Debug, Default, clap::ValueEnum)] +pub enum RateLimitAlgorithm { + Fixed, + #[default] + Aimd, +} + +pub struct Fixed; + +#[async_trait] +impl LimitAlgorithm for Fixed { + async fn update(&mut self, old_limit: usize, _sample: Sample) -> usize { + old_limit + } +} + +#[derive(Clone, Copy, Debug)] +pub struct RateLimiterConfig { + pub disable: bool, + pub algorithm: RateLimitAlgorithm, + pub timeout: Duration, + pub initial_limit: usize, + pub aimd_config: Option, +} + +impl RateLimiterConfig { + pub fn create_rate_limit_algorithm(self) -> Box { + match self.algorithm { + RateLimitAlgorithm::Fixed => Box::new(Fixed), + RateLimitAlgorithm::Aimd => Box::new(Aimd::new(self.aimd_config.unwrap())), // For aimd algorithm config is mandatory. + } + } +} + +impl Default for RateLimiterConfig { + fn default() -> Self { + Self { + disable: true, + algorithm: RateLimitAlgorithm::Aimd, + timeout: Duration::from_secs(1), + initial_limit: 100, + aimd_config: Some(AimdConfig::default()), + } + } +} + +#[derive(clap::Parser, Clone, Copy, Debug)] +pub struct AimdConfig { + /// Minimum limit for AIMD algorithm. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 1)] + pub aimd_min_limit: usize, + /// Maximum limit for AIMD algorithm. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 1500)] + pub aimd_max_limit: usize, + /// Increase AIMD increase by value in case of success. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 10)] + pub aimd_increase_by: usize, + /// Decrease AIMD decrease by value in case of timout/429. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 0.9)] + pub aimd_decrease_factor: f32, + /// A threshold below which the limit won't be increased. Makes sense only if `rate_limit_algorithm` is `Aimd`. + #[clap(long, default_value_t = 0.8)] + pub aimd_min_utilisation_threshold: f32, +} + +impl Default for AimdConfig { + fn default() -> Self { + Self { + aimd_min_limit: 1, + aimd_max_limit: 1500, + aimd_increase_by: 10, + aimd_decrease_factor: 0.9, + aimd_min_utilisation_threshold: 0.8, + } + } +} diff --git a/proxy/src/rate_limiter/limiter.rs b/proxy/src/rate_limiter/limiter.rs new file mode 100644 index 0000000000..3a9fed3919 --- /dev/null +++ b/proxy/src/rate_limiter/limiter.rs @@ -0,0 +1,441 @@ +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +use tokio::sync::{Mutex as AsyncMutex, Semaphore, SemaphorePermit}; +use tokio::time::{timeout, Instant}; +use tracing::info; + +use super::{ + limit_algorithm::{LimitAlgorithm, Sample}, + RateLimiterConfig, +}; + +/// Limits the number of concurrent jobs. +/// +/// Concurrency is limited through the use of [Token]s. Acquire a token to run a job, and release the +/// token once the job is finished. +/// +/// The limit will be automatically adjusted based on observed latency (delay) and/or failures +/// caused by overload (loss). +pub struct Limiter { + limit_algo: AsyncMutex>, + semaphore: std::sync::Arc, + config: RateLimiterConfig, + + // ONLY WRITE WHEN LIMIT_ALGO IS LOCKED + limits: AtomicUsize, + + // ONLY USE ATOMIC ADD/SUB + in_flight: Arc, + + #[cfg(test)] + notifier: Option>, +} + +/// A concurrency token, required to run a job. +/// +/// Release the token back to the [Limiter] after the job is complete. +#[derive(Debug)] +pub struct Token<'t> { + permit: Option>, + start: Instant, + in_flight: Arc, +} + +/// A snapshot of the state of the [Limiter]. +/// +/// Not guaranteed to be consistent under high concurrency. +#[derive(Debug, Clone, Copy)] +pub struct LimiterState { + limit: usize, + available: usize, + in_flight: usize, +} + +/// Whether a job succeeded or failed as a result of congestion/overload. +/// +/// Errors not considered to be caused by overload should be ignored. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Outcome { + /// The job succeeded, or failed in a way unrelated to overload. + Success, + /// The job failed because of overload, e.g. it timed out or an explicit backpressure signal + /// was observed. + Overload, +} + +impl Outcome { + fn from_reqwest_error(error: &reqwest_middleware::Error) -> Self { + match error { + reqwest_middleware::Error::Middleware(_) => Outcome::Success, + reqwest_middleware::Error::Reqwest(e) => { + if let Some(status) = e.status() { + if status.is_server_error() + || reqwest::StatusCode::TOO_MANY_REQUESTS.as_u16() == status + { + Outcome::Overload + } else { + Outcome::Success + } + } else { + Outcome::Success + } + } + } + } + fn from_reqwest_response(response: &reqwest::Response) -> Self { + if response.status().is_server_error() + || response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS + { + Outcome::Overload + } else { + Outcome::Success + } + } +} + +impl Limiter { + /// Create a limiter with a given limit control algorithm. + pub fn new(config: RateLimiterConfig) -> Self { + assert!(config.initial_limit > 0); + Self { + limit_algo: AsyncMutex::new(config.create_rate_limit_algorithm()), + semaphore: Arc::new(Semaphore::new(config.initial_limit)), + config, + limits: AtomicUsize::new(config.initial_limit), + in_flight: Arc::new(AtomicUsize::new(0)), + #[cfg(test)] + notifier: None, + } + } + // pub fn new(limit_algorithm: T, timeout: Duration, initial_limit: usize) -> Self { + // assert!(initial_limit > 0); + + // Self { + // limit_algo: AsyncMutex::new(limit_algorithm), + // semaphore: Arc::new(Semaphore::new(initial_limit)), + // timeout, + // limits: AtomicUsize::new(initial_limit), + // in_flight: Arc::new(AtomicUsize::new(0)), + // #[cfg(test)] + // notifier: None, + // } + // } + + /// In some cases [Token]s are acquired asynchronously when updating the limit. + #[cfg(test)] + pub fn with_release_notifier(mut self, n: std::sync::Arc) -> Self { + self.notifier = Some(n); + self + } + + /// Try to immediately acquire a concurrency [Token]. + /// + /// Returns `None` if there are none available. + pub fn try_acquire(&self) -> Option { + let result = if self.config.disable { + // If the rate limiter is disabled, we can always acquire a token. + Some(Token::new(None, self.in_flight.clone())) + } else { + self.semaphore + .try_acquire() + .map(|permit| Token::new(Some(permit), self.in_flight.clone())) + .ok() + }; + if result.is_some() { + self.in_flight.fetch_add(1, Ordering::AcqRel); + } + result + } + + /// Try to acquire a concurrency [Token], waiting for `duration` if there are none available. + /// + /// Returns `None` if there are none available after `duration`. + pub async fn acquire_timeout(&self, duration: Duration) -> Option> { + info!("acquiring token: {:?}", self.semaphore.available_permits()); + let result = if self.config.disable { + // If the rate limiter is disabled, we can always acquire a token. + Some(Token::new(None, self.in_flight.clone())) + } else { + match timeout(duration, self.semaphore.acquire()).await { + Ok(maybe_permit) => maybe_permit + .map(|permit| Token::new(Some(permit), self.in_flight.clone())) + .ok(), + Err(_) => None, + } + }; + if result.is_some() { + self.in_flight.fetch_add(1, Ordering::AcqRel); + } + result + } + + /// Return the concurrency [Token], along with the outcome of the job. + /// + /// The [Outcome] of the job, and the time taken to perform it, may be used + /// to update the concurrency limit. + /// + /// Set the outcome to `None` to ignore the job. + pub async fn release(&self, mut token: Token<'_>, outcome: Option) { + tracing::info!("outcome is {:?}", outcome); + let in_flight = self.in_flight.load(Ordering::Acquire); + let old_limit = self.limits.load(Ordering::Acquire); + let available = if self.config.disable { + 0 // This is not used in the algorithm and can be anything. If the config disable it makes sense to set it to 0. + } else { + self.semaphore.available_permits() + }; + let total = in_flight + available; + + let mut algo = self.limit_algo.lock().await; + + let new_limit = if let Some(outcome) = outcome { + let sample = Sample { + latency: token.start.elapsed(), + in_flight, + outcome, + }; + algo.update(old_limit, sample).await + } else { + old_limit + }; + tracing::info!("new limit is {}", new_limit); + let actual_limit = if new_limit < total { + token.forget(); + total.saturating_sub(1) + } else { + if !self.config.disable { + self.semaphore.add_permits(new_limit.saturating_sub(total)); + } + new_limit + }; + crate::proxy::RATE_LIMITER_LIMIT + .with_label_values(&["expected"]) + .set(new_limit as i64); + crate::proxy::RATE_LIMITER_LIMIT + .with_label_values(&["actual"]) + .set(actual_limit as i64); + self.limits.store(new_limit, Ordering::Release); + #[cfg(test)] + if let Some(n) = &self.notifier { + n.notify_one(); + } + } + + /// The current state of the limiter. + pub fn state(&self) -> LimiterState { + let limit = self.limits.load(Ordering::Relaxed); + let in_flight = self.in_flight.load(Ordering::Relaxed); + LimiterState { + limit, + available: limit.saturating_sub(in_flight), + in_flight, + } + } +} + +impl<'t> Token<'t> { + fn new(permit: Option>, in_flight: Arc) -> Self { + Self { + permit, + start: Instant::now(), + in_flight, + } + } + + #[cfg(test)] + pub fn set_latency(&mut self, latency: Duration) { + use std::ops::Sub; + + self.start = Instant::now().sub(latency); + } + + pub fn forget(&mut self) { + if let Some(permit) = self.permit.take() { + permit.forget(); + } + } +} + +impl Drop for Token<'_> { + fn drop(&mut self) { + self.in_flight.fetch_sub(1, Ordering::AcqRel); + } +} + +impl LimiterState { + /// The current concurrency limit. + pub fn limit(&self) -> usize { + self.limit + } + /// The amount of concurrency available to use. + pub fn available(&self) -> usize { + self.available + } + /// The number of jobs in flight. + pub fn in_flight(&self) -> usize { + self.in_flight + } +} + +#[async_trait::async_trait] +impl reqwest_middleware::Middleware for Limiter { + async fn handle( + &self, + req: reqwest::Request, + extensions: &mut task_local_extensions::Extensions, + next: reqwest_middleware::Next<'_>, + ) -> reqwest_middleware::Result { + let start = Instant::now(); + let token = self + .acquire_timeout(self.config.timeout) + .await + .ok_or_else(|| { + reqwest_middleware::Error::Middleware( + // TODO: Should we map it into user facing errors? + crate::console::errors::ApiError::Console { + status: crate::http::StatusCode::TOO_MANY_REQUESTS, + text: "Too many requests".into(), + } + .into(), + ) + })?; + info!(duration = ?start.elapsed(), "waiting for token to connect to the control plane"); + crate::proxy::RATE_LIMITER_ACQUIRE_LATENCY.observe(start.elapsed().as_secs_f64()); + match next.run(req, extensions).await { + Ok(response) => { + self.release(token, Some(Outcome::from_reqwest_response(&response))) + .await; + Ok(response) + } + Err(e) => { + self.release(token, Some(Outcome::from_reqwest_error(&e))) + .await; + Err(e) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{pin::pin, task::Context, time::Duration}; + + use futures::{task::noop_waker_ref, Future}; + + use super::{Limiter, Outcome}; + use crate::rate_limiter::RateLimitAlgorithm; + + #[tokio::test] + async fn it_works() { + let config = super::RateLimiterConfig { + algorithm: RateLimitAlgorithm::Fixed, + timeout: Duration::from_secs(1), + initial_limit: 10, + disable: false, + ..Default::default() + }; + let limiter = Limiter::new(config); + + let token = limiter.try_acquire().unwrap(); + + limiter.release(token, Some(Outcome::Success)).await; + + assert_eq!(limiter.state().limit(), 10); + } + + #[tokio::test] + async fn is_fair() { + let config = super::RateLimiterConfig { + algorithm: RateLimitAlgorithm::Fixed, + timeout: Duration::from_secs(1), + initial_limit: 1, + disable: false, + ..Default::default() + }; + let limiter = Limiter::new(config); + + // === TOKEN 1 === + let token1 = limiter.try_acquire().unwrap(); + + let mut token2_fut = pin!(limiter.acquire_timeout(Duration::from_secs(1))); + assert!( + token2_fut + .as_mut() + .poll(&mut Context::from_waker(noop_waker_ref())) + .is_pending(), + "token is acquired by token1" + ); + + let mut token3_fut = pin!(limiter.acquire_timeout(Duration::from_secs(1))); + assert!( + token3_fut + .as_mut() + .poll(&mut Context::from_waker(noop_waker_ref())) + .is_pending(), + "token is acquired by token1" + ); + + limiter.release(token1, Some(Outcome::Success)).await; + // === END TOKEN 1 === + + // === TOKEN 2 === + assert!( + limiter.try_acquire().is_none(), + "token is acquired by token2" + ); + + assert!( + token3_fut + .as_mut() + .poll(&mut Context::from_waker(noop_waker_ref())) + .is_pending(), + "token is acquired by token2" + ); + + let token2 = token2_fut.await.unwrap(); + + limiter.release(token2, Some(Outcome::Success)).await; + // === END TOKEN 2 === + + // === TOKEN 3 === + assert!( + limiter.try_acquire().is_none(), + "token is acquired by token3" + ); + + let token3 = token3_fut.await.unwrap(); + limiter.release(token3, Some(Outcome::Success)).await; + // === END TOKEN 3 === + + // === TOKEN 4 === + let token4 = limiter.try_acquire().unwrap(); + limiter.release(token4, Some(Outcome::Success)).await; + } + + #[tokio::test] + async fn disable() { + let config = super::RateLimiterConfig { + algorithm: RateLimitAlgorithm::Fixed, + timeout: Duration::from_secs(1), + initial_limit: 1, + disable: true, + ..Default::default() + }; + let limiter = Limiter::new(config); + + // === TOKEN 1 === + let token1 = limiter.try_acquire().unwrap(); + let token2 = limiter.try_acquire().unwrap(); + let state = limiter.state(); + assert_eq!(state.limit(), 1); + assert_eq!(state.in_flight(), 2); // For disabled limiter, it's expected. + limiter.release(token1, None).await; + limiter.release(token2, None).await; + } +} diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index c5bfc32568..b753bc8918 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -22,7 +22,10 @@ use tokio_postgres::{AsyncMessage, ReadyForQueryStatus}; use crate::{ auth, console, - proxy::{LatencyTimer, NUM_DB_CONNECTIONS_CLOSED_COUNTER, NUM_DB_CONNECTIONS_OPENED_COUNTER}, + proxy::{ + neon_options, LatencyTimer, NUM_DB_CONNECTIONS_CLOSED_COUNTER, + NUM_DB_CONNECTIONS_OPENED_COUNTER, + }, usage_metrics::{Ids, MetricCounter, USAGE_METRICS}, }; use crate::{compute, config}; @@ -41,6 +44,7 @@ pub struct ConnInfo { pub dbname: String, pub hostname: String, pub password: String, + pub options: Option, } impl ConnInfo { @@ -204,14 +208,13 @@ impl GlobalConnPool { } else { info!("pool: reusing connection '{conn_info}'"); client.session.send(session_id)?; + tracing::Span::current().record( + "pid", + &tracing::field::display(client.inner.get_process_id()), + ); latency_timer.pool_hit(); latency_timer.success(); - return Ok(Client { - conn_id: client.conn_id, - inner: Some(client), - span: Span::current(), - pool, - }); + return Ok(Client::new(client, pool).await); } } else { let conn_id = uuid::Uuid::new_v4(); @@ -225,6 +228,12 @@ impl GlobalConnPool { ) .await }; + if let Ok(client) = &new_client { + tracing::Span::current().record( + "pid", + &tracing::field::display(client.inner.get_process_id()), + ); + } match &new_client { // clear the hash. it's no longer valid @@ -258,13 +267,8 @@ impl GlobalConnPool { } _ => {} } - - new_client.map(|inner| Client { - conn_id: inner.conn_id, - inner: Some(inner), - span: Span::current(), - pool, - }) + let new_client = new_client?; + Ok(Client::new(new_client, pool).await) } fn put(&self, conn_info: &ConnInfo, client: ClientInner) -> anyhow::Result<()> { @@ -390,7 +394,7 @@ impl ConnectMechanism for TokioMechanism<'_> { // Wake up the destination if needed. Code here is a bit involved because // we reuse the code from the usual proxy and we need to prepare few structures // that this code expects. -#[tracing::instrument(skip_all)] +#[tracing::instrument(fields(pid = tracing::field::Empty), skip_all)] async fn connect_to_compute( config: &config::ProxyConfig, conn_info: &ConnInfo, @@ -401,26 +405,25 @@ async fn connect_to_compute( let tls = config.tls_config.as_ref(); let common_names = tls.and_then(|tls| tls.common_names.clone()); - let credential_params = StartupMessageParams::new([ + let params = StartupMessageParams::new([ ("user", &conn_info.username), ("database", &conn_info.dbname), ("application_name", APP_NAME), + ("options", conn_info.options.as_deref().unwrap_or("")), ]); let creds = config .auth_backend .as_ref() - .map(|_| { - auth::ClientCredentials::parse( - &credential_params, - Some(&conn_info.hostname), - common_names, - ) - }) + .map(|_| auth::ClientCredentials::parse(¶ms, Some(&conn_info.hostname), common_names)) .transpose()?; + + let console_options = neon_options(¶ms); + let extra = console::ConsoleReqExtra { session_id: uuid::Uuid::new_v4(), application_name: Some(APP_NAME), + options: console_options.as_deref(), }; let node_info = creds @@ -458,6 +461,7 @@ async fn connect_to_compute_once( .connect_timeout(timeout) .connect(tokio_postgres::NoTls) .await?; + tracing::Span::current().record("pid", &tracing::field::display(client.get_process_id())); let (tx, mut rx) = tokio::sync::watch::channel(session); @@ -544,6 +548,17 @@ pub struct Discard<'a> { } impl Client { + pub(self) async fn new( + inner: ClientInner, + pool: Option<(ConnInfo, Arc)>, + ) -> Self { + Self { + conn_id: inner.conn_id, + inner: Some(inner), + span: Span::current(), + pool, + } + } pub fn inner(&mut self) -> (&mut tokio_postgres::Client, Discard<'_>) { let Self { inner, diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 8f7cf7fbaf..4a9829e360 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -174,11 +174,23 @@ fn get_conn_info( } } + let pairs = connection_url.query_pairs(); + + let mut options = Option::None; + + for (key, value) in pairs { + if key == "options" { + options = Some(value.to_string()); + break; + } + } + Ok(ConnInfo { username: username.to_owned(), dbname: dbname.to_owned(), hostname: hostname.to_owned(), password: password.to_owned(), + options, }) } @@ -238,7 +250,7 @@ pub async fn handle( Ok(response) } -#[instrument(name = "sql-over-http", skip_all)] +#[instrument(name = "sql-over-http", fields(pid = tracing::field::Empty), skip_all)] async fn handle_inner( request: Request, sni_hostname: Option, diff --git a/proxy/src/usage_metrics.rs b/proxy/src/usage_metrics.rs index cfeec5622b..180b5f7199 100644 --- a/proxy/src/usage_metrics.rs +++ b/proxy/src/usage_metrics.rs @@ -249,7 +249,7 @@ mod tests { use url::Url; use super::{collect_metrics_iteration, Ids, Metrics}; - use crate::http; + use crate::{http, rate_limiter::RateLimiterConfig}; #[tokio::test] async fn metrics() { @@ -279,7 +279,7 @@ mod tests { tokio::spawn(server); let metrics = Metrics::default(); - let client = http::new_client(); + let client = http::new_client(RateLimiterConfig::default()); let endpoint = Url::parse(&format!("http://{addr}")).unwrap(); let now = Utc::now(); diff --git a/pyproject.toml b/pyproject.toml index ebf69c636e..396edabe10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ psutil = "^5.9.4" types-psutil = "^5.9.5.12" types-toml = "^0.10.8.6" pytest-httpserver = "^1.0.8" -aiohttp = "3.8.5" +aiohttp = "3.8.6" pytest-rerunfailures = "^11.1.2" types-pytest-lazy-fixture = "^0.6.3.3" pytest-split = "^0.8.1" diff --git a/s3_scrubber/Cargo.toml b/s3_scrubber/Cargo.toml index f3ea6e222c..0f3e5630e8 100644 --- a/s3_scrubber/Cargo.toml +++ b/s3_scrubber/Cargo.toml @@ -33,6 +33,7 @@ reqwest = { workspace = true, default-features = false, features = ["rustls-tls" aws-config = { workspace = true, default-features = false, features = ["rustls", "credentials-sso"] } pageserver = { path = "../pageserver" } +remote_storage = { path = "../libs/remote_storage" } tracing.workspace = true tracing-subscriber.workspace = true diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs index c11f1f9779..64702fca3d 100644 --- a/s3_scrubber/src/checks.rs +++ b/s3_scrubber/src/checks.rs @@ -1,13 +1,18 @@ use std::collections::HashSet; use anyhow::Context; -use aws_sdk_s3::Client; +use aws_sdk_s3::{types::ObjectIdentifier, Client}; use tracing::{error, info, warn}; +use utils::generation::Generation; use crate::cloud_admin_api::BranchData; -use crate::{download_object_with_retries, list_objects_with_retries, RootTarget}; +use crate::metadata_stream::stream_listing; +use crate::{download_object_with_retries, RootTarget}; +use futures_util::{pin_mut, StreamExt}; +use pageserver::tenant::remote_timeline_client::parse_remote_index_path; use pageserver::tenant::storage_layer::LayerFileName; use pageserver::tenant::IndexPart; +use remote_storage::RemotePath; use utils::id::TenantTimelineId; pub(crate) struct TimelineAnalysis { @@ -68,6 +73,7 @@ pub(crate) async fn branch_cleanup_and_check_errors( match s3_data.blob_data { BlobDataParseResult::Parsed { index_part, + index_part_generation, mut s3_layers, } => { if !IndexPart::KNOWN_VERSIONS.contains(&index_part.get_version()) { @@ -107,33 +113,62 @@ pub(crate) async fn branch_cleanup_and_check_errors( )) } - if !s3_layers.remove(&layer) { + let layer_map_key = (layer, metadata.generation); + if !s3_layers.remove(&layer_map_key) { + // FIXME: this will emit false positives if an index was + // uploaded concurrently with our scan. To make this check + // correct, we need to try sending a HEAD request for the + // layer we think is missing. result.errors.push(format!( - "index_part.json contains a layer {} that is not present in S3", - layer.file_name(), + "index_part.json contains a layer {}{} that is not present in remote storage", + layer_map_key.0.file_name(), + layer_map_key.1.get_suffix() )) } } - if !s3_layers.is_empty() { + let orphan_layers: Vec<(LayerFileName, Generation)> = s3_layers + .into_iter() + .filter(|(_layer_name, gen)| + // A layer is only considered orphaned if it has a generation below + // the index. If the generation is >= the index, then the layer may + // be an upload from a running pageserver, or even an upload from + // a new generation that didn't upload an index yet. + // + // Even so, a layer that is not referenced by the index could just + // be something enqueued for deletion, so while this check is valid + // for indicating that a layer is garbage, it is not an indicator + // of a problem. + gen < &index_part_generation) + .collect(); + + if !orphan_layers.is_empty() { result.errors.push(format!( "index_part.json does not contain layers from S3: {:?}", - s3_layers + orphan_layers .iter() - .map(|layer_name| layer_name.file_name()) + .map(|(layer_name, gen)| format!( + "{}{}", + layer_name.file_name(), + gen.get_suffix() + )) .collect::>(), )); - result - .garbage_keys - .extend(s3_layers.iter().map(|layer_name| { + result.garbage_keys.extend(orphan_layers.iter().map( + |(layer_name, layer_gen)| { let mut key = s3_root.timeline_root(id).prefix_in_bucket; let delimiter = s3_root.delimiter(); if !key.ends_with(delimiter) { key.push_str(delimiter); } - key.push_str(&layer_name.file_name()); + key.push_str(&format!( + "{}{}", + &layer_name.file_name(), + layer_gen.get_suffix() + )); key - })); + }, + )); } } BlobDataParseResult::Incorrect(parse_errors) => result.errors.extend( @@ -178,70 +213,97 @@ pub(crate) struct S3TimelineBlobData { pub(crate) enum BlobDataParseResult { Parsed { index_part: IndexPart, - s3_layers: HashSet, + index_part_generation: Generation, + s3_layers: HashSet<(LayerFileName, Generation)>, }, Incorrect(Vec), } +fn parse_layer_object_name(name: &str) -> Result<(LayerFileName, Generation), String> { + match name.rsplit_once('-') { + // FIXME: this is gross, just use a regex? + Some((layer_filename, gen)) if gen.len() == 8 => { + let layer = layer_filename.parse::()?; + let gen = + Generation::parse_suffix(gen).ok_or("Malformed generation suffix".to_string())?; + Ok((layer, gen)) + } + _ => Ok((name.parse::()?, Generation::none())), + } +} + pub(crate) async fn list_timeline_blobs( s3_client: &Client, id: TenantTimelineId, s3_root: &RootTarget, ) -> anyhow::Result { let mut s3_layers = HashSet::new(); - let mut index_part_object = None; - - let timeline_dir_target = s3_root.timeline_root(&id); - let mut continuation_token = None; let mut errors = Vec::new(); let mut keys_to_remove = Vec::new(); - loop { - let fetch_response = - list_objects_with_retries(s3_client, &timeline_dir_target, continuation_token.clone()) - .await?; + let mut timeline_dir_target = s3_root.timeline_root(&id); + timeline_dir_target.delimiter = String::new(); - let subdirectories = fetch_response.common_prefixes().unwrap_or_default(); - if !subdirectories.is_empty() { - errors.push(format!( - "S3 list response should not contain any subdirectories, but got {subdirectories:?}" - )); - } + let mut index_parts: Vec = Vec::new(); - for (object, key) in fetch_response - .contents() - .unwrap_or_default() - .iter() - .filter_map(|object| Some((object, object.key()?))) - { - let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); - match blob_name { - Some("index_part.json") => index_part_object = Some(object.clone()), - Some(maybe_layer_name) => match maybe_layer_name.parse::() { - Ok(new_layer) => { - s3_layers.insert(new_layer); - } - Err(e) => { - errors.push( - format!("S3 list response got an object with key {key} that is not a layer name: {e}"), - ); - keys_to_remove.push(key.to_string()); - } - }, - None => { - errors.push(format!("S3 list response got an object with odd key {key}")); + let stream = stream_listing(s3_client, &timeline_dir_target); + pin_mut!(stream); + while let Some(obj) = stream.next().await { + let obj = obj?; + let key = match obj.key() { + Some(k) => k, + None => continue, + }; + + let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); + match blob_name { + Some(name) if name.starts_with("index_part.json") => { + tracing::info!("Index key {key}"); + index_parts.push(obj) + } + Some(maybe_layer_name) => match parse_layer_object_name(maybe_layer_name) { + Ok((new_layer, gen)) => { + tracing::info!("Parsed layer key: {} {:?}", new_layer, gen); + s3_layers.insert((new_layer, gen)); + } + Err(e) => { + tracing::info!("Error parsing key {maybe_layer_name}"); + errors.push( + format!("S3 list response got an object with key {key} that is not a layer name: {e}"), + ); keys_to_remove.push(key.to_string()); } + }, + None => { + tracing::info!("Peculiar key {}", key); + errors.push(format!("S3 list response got an object with odd key {key}")); + keys_to_remove.push(key.to_string()); } } - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } } + // Choose the index_part with the highest generation + let (index_part_object, index_part_generation) = match index_parts + .iter() + .filter_map(|k| { + let key = k.key().unwrap(); + // Stripping the index key to the last part, because RemotePath doesn't + // like absolute paths, and depending on prefix_in_bucket it's possible + // for the keys we read back to start with a slash. + let basename = key.rsplit_once('/').unwrap().1; + parse_remote_index_path(RemotePath::from_string(basename).unwrap()).map(|g| (k, g)) + }) + .max_by_key(|i| i.1) + .map(|(k, g)| (k.clone(), g)) + { + Some((key, gen)) => (Some(key), gen), + None => { + // Legacy/missing case: one or zero index parts, which did not have a generation + (index_parts.pop(), Generation::none()) + } + }; + if index_part_object.is_none() { errors.push("S3 list response got no index_part.json file".to_string()); } @@ -261,6 +323,7 @@ pub(crate) async fn list_timeline_blobs( return Ok(S3TimelineBlobData { blob_data: BlobDataParseResult::Parsed { index_part, + index_part_generation, s3_layers, }, keys_to_remove, diff --git a/s3_scrubber/src/cloud_admin_api.rs b/s3_scrubber/src/cloud_admin_api.rs index 1b28dc4dff..151421c84f 100644 --- a/s3_scrubber/src/cloud_admin_api.rs +++ b/s3_scrubber/src/cloud_admin_api.rs @@ -5,6 +5,7 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use hex::FromHex; +use pageserver::tenant::Tenant; use reqwest::{header, Client, StatusCode, Url}; use serde::Deserialize; use tokio::sync::Semaphore; @@ -118,13 +119,18 @@ fn from_nullable_id<'de, D>(deserializer: D) -> Result where D: serde::de::Deserializer<'de>, { - let id_str = String::deserialize(deserializer)?; - if id_str.is_empty() { - // This is a bogus value, but for the purposes of the scrubber all that - // matters is that it doesn't collide with any real IDs. - Ok(TenantId::from([0u8; 16])) + if deserializer.is_human_readable() { + let id_str = String::deserialize(deserializer)?; + if id_str.is_empty() { + // This is a bogus value, but for the purposes of the scrubber all that + // matters is that it doesn't collide with any real IDs. + Ok(TenantId::from([0u8; 16])) + } else { + TenantId::from_hex(&id_str).map_err(|e| serde::de::Error::custom(format!("{e}"))) + } } else { - TenantId::from_hex(&id_str).map_err(|e| serde::de::Error::custom(format!("{e}"))) + let id_arr = <[u8; 16]>::deserialize(deserializer)?; + Ok(TenantId::from(id_arr)) } } @@ -153,7 +159,6 @@ pub struct ProjectData { pub maintenance_set: Option, } -#[serde_with::serde_as] #[derive(Debug, serde::Deserialize)] pub struct BranchData { pub id: BranchId, @@ -161,12 +166,10 @@ pub struct BranchData { pub updated_at: DateTime, pub name: String, pub project_id: ProjectId, - #[serde_as(as = "serde_with::DisplayFromStr")] pub timeline_id: TimelineId, #[serde(default)] pub parent_id: Option, #[serde(default)] - #[serde_as(as = "Option")] pub parent_lsn: Option, pub default: bool, pub deleted: bool, diff --git a/s3_scrubber/src/lib.rs b/s3_scrubber/src/lib.rs index 98e7ed5334..777276a4d1 100644 --- a/s3_scrubber/src/lib.rs +++ b/s3_scrubber/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(unsafe_code)] +#![deny(clippy::undocumented_unsafe_blocks)] pub mod checks; pub mod cloud_admin_api; pub mod garbage; @@ -34,6 +36,9 @@ const CLOUD_ADMIN_API_TOKEN_ENV_VAR: &str = "CLOUD_ADMIN_API_TOKEN"; #[derive(Debug, Clone)] pub struct S3Target { pub bucket_name: String, + /// This `prefix_in_bucket` is only equal to the PS/SK config of the same + /// name for the RootTarget: other instances of S3Target will have prefix_in_bucket + /// with extra parts. pub prefix_in_bucket: String, pub delimiter: String, } @@ -77,9 +82,13 @@ impl Display for NodeKind { impl S3Target { pub fn with_sub_segment(&self, new_segment: &str) -> Self { let mut new_self = self.clone(); - let _ = new_self.prefix_in_bucket.pop(); - new_self.prefix_in_bucket = - [&new_self.prefix_in_bucket, new_segment, ""].join(&new_self.delimiter); + if new_self.prefix_in_bucket.is_empty() { + new_self.prefix_in_bucket = format!("/{}/", new_segment); + } else { + let _ = new_self.prefix_in_bucket.pop(); + new_self.prefix_in_bucket = + [&new_self.prefix_in_bucket, new_segment, ""].join(&new_self.delimiter); + } new_self } } @@ -91,10 +100,10 @@ pub enum RootTarget { } impl RootTarget { - pub fn tenants_root(&self) -> &S3Target { + pub fn tenants_root(&self) -> S3Target { match self { - Self::Pageserver(root) => root, - Self::Safekeeper(root) => root, + Self::Pageserver(root) => root.with_sub_segment(TENANTS_SEGMENT_NAME), + Self::Safekeeper(root) => root.with_sub_segment("wal"), } } @@ -133,6 +142,7 @@ impl RootTarget { pub struct BucketConfig { pub region: String, pub bucket: String, + pub prefix_in_bucket: Option, /// Use SSO if this is set, else rely on AWS_* environment vars pub sso_account_id: Option, @@ -155,10 +165,12 @@ impl BucketConfig { let sso_account_id = env::var("SSO_ACCOUNT_ID").ok(); let region = env::var("REGION").context("'REGION' param retrieval")?; let bucket = env::var("BUCKET").context("'BUCKET' param retrieval")?; + let prefix_in_bucket = env::var("BUCKET_PREFIX").ok(); Ok(Self { region, bucket, + prefix_in_bucket, sso_account_id, }) } @@ -191,14 +203,14 @@ pub fn init_logging(file_name: &str) -> WorkerGuard { .with_target(false) .with_ansi(false) .with_writer(file_writer); - let stdout_logs = fmt::Layer::new() - .with_ansi(std::io::stdout().is_terminal()) + let stderr_logs = fmt::Layer::new() + .with_ansi(std::io::stderr().is_terminal()) .with_target(false) - .with_writer(std::io::stdout); + .with_writer(std::io::stderr); tracing_subscriber::registry() .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) .with(file_logs) - .with(stdout_logs) + .with(stderr_logs) .init(); guard @@ -250,15 +262,20 @@ fn init_remote( let bucket_region = Region::new(bucket_config.region); let delimiter = "/".to_string(); let s3_client = Arc::new(init_s3_client(bucket_config.sso_account_id, bucket_region)); + let s3_root = match node_kind { NodeKind::Pageserver => RootTarget::Pageserver(S3Target { bucket_name: bucket_config.bucket, - prefix_in_bucket: ["pageserver", "v1", TENANTS_SEGMENT_NAME, ""].join(&delimiter), + prefix_in_bucket: bucket_config + .prefix_in_bucket + .unwrap_or("pageserver/v1".to_string()), delimiter, }), NodeKind::Safekeeper => RootTarget::Safekeeper(S3Target { bucket_name: bucket_config.bucket, - prefix_in_bucket: ["safekeeper", "v1", "wal", ""].join(&delimiter), + prefix_in_bucket: bucket_config + .prefix_in_bucket + .unwrap_or("safekeeper/v1".to_string()), delimiter, }), }; diff --git a/s3_scrubber/src/main.rs b/s3_scrubber/src/main.rs index 9a0d6c9ae8..1f0ceebdaf 100644 --- a/s3_scrubber/src/main.rs +++ b/s3_scrubber/src/main.rs @@ -31,7 +31,10 @@ enum Command { #[arg(short, long, default_value_t = PurgeMode::DeletedOnly)] mode: PurgeMode, }, - ScanMetadata {}, + ScanMetadata { + #[arg(short, long, default_value_t = false)] + json: bool, + }, } #[tokio::main] @@ -54,13 +57,17 @@ async fn main() -> anyhow::Result<()> { )); match cli.command { - Command::ScanMetadata {} => match scan_metadata(bucket_config).await { + Command::ScanMetadata { json } => match scan_metadata(bucket_config).await { Err(e) => { tracing::error!("Failed: {e}"); Err(e) } Ok(summary) => { - println!("{}", summary.summary_string()); + if json { + println!("{}", serde_json::to_string(&summary).unwrap()) + } else { + println!("{}", summary.summary_string()); + } if summary.is_fatal() { Err(anyhow::anyhow!("Fatal scrub errors detected")) } else { diff --git a/s3_scrubber/src/metadata_stream.rs b/s3_scrubber/src/metadata_stream.rs index 125a370e90..8095071c1f 100644 --- a/s3_scrubber/src/metadata_stream.rs +++ b/s3_scrubber/src/metadata_stream.rs @@ -13,10 +13,10 @@ pub fn stream_tenants<'a>( ) -> impl Stream> + 'a { try_stream! { let mut continuation_token = None; + let tenants_target = target.tenants_root(); loop { - let tenants_target = target.tenants_root(); let fetch_response = - list_objects_with_retries(s3_client, tenants_target, continuation_token.clone()).await?; + list_objects_with_retries(s3_client, &tenants_target, continuation_token.clone()).await?; let new_entry_ids = fetch_response .common_prefixes() diff --git a/s3_scrubber/src/scan_metadata.rs b/s3_scrubber/src/scan_metadata.rs index 33acd41a5b..ad82db1e76 100644 --- a/s3_scrubber/src/scan_metadata.rs +++ b/s3_scrubber/src/scan_metadata.rs @@ -10,8 +10,10 @@ use aws_sdk_s3::Client; use futures_util::{pin_mut, StreamExt, TryStreamExt}; use histogram::Histogram; use pageserver::tenant::IndexPart; +use serde::Serialize; use utils::id::TenantTimelineId; +#[derive(Serialize)] pub struct MetadataSummary { count: usize, with_errors: HashSet, @@ -25,7 +27,9 @@ pub struct MetadataSummary { } /// A histogram plus minimum and maximum tracking +#[derive(Serialize)] struct MinMaxHisto { + #[serde(skip)] histo: Histogram, min: u64, max: u64, @@ -109,6 +113,7 @@ impl MetadataSummary { self.count += 1; if let BlobDataParseResult::Parsed { index_part, + index_part_generation: _, s3_layers: _, } = &data.blob_data { diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index 64ef9f6997..30516c0763 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -47,6 +47,7 @@ pq_proto.workspace = true remote_storage.workspace = true safekeeper_api.workspace = true storage_broker.workspace = true +tokio-stream.workspace = true utils.workspace = true workspace_hack.workspace = true diff --git a/safekeeper/src/auth.rs b/safekeeper/src/auth.rs index 2684d82365..bf4905aaa7 100644 --- a/safekeeper/src/auth.rs +++ b/safekeeper/src/auth.rs @@ -1,19 +1,20 @@ -use anyhow::{bail, Result}; -use utils::auth::{Claims, Scope}; +use utils::auth::{AuthError, Claims, Scope}; use utils::id::TenantId; -pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<()> { +pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<(), AuthError> { match (&claims.scope, tenant_id) { - (Scope::Tenant, None) => { - bail!("Attempt to access management api with tenant scope. Permission denied") - } + (Scope::Tenant, None) => Err(AuthError( + "Attempt to access management api with tenant scope. Permission denied".into(), + )), (Scope::Tenant, Some(tenant_id)) => { if claims.tenant_id.unwrap() != tenant_id { - bail!("Tenant id mismatch. Permission denied") + return Err(AuthError("Tenant id mismatch. Permission denied".into())); } Ok(()) } - (Scope::PageServerApi, _) => bail!("PageServerApi scope makes no sense for Safekeeper"), + (Scope::PageServerApi, _) => Err(AuthError( + "PageServerApi scope makes no sense for Safekeeper".into(), + )), (Scope::SafekeeperData, _) => Ok(()), } } diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index d476cf33ae..2e54380471 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -38,7 +38,7 @@ use safekeeper::{http, WAL_REMOVER_RUNTIME}; use safekeeper::{remove_wal, WAL_BACKUP_RUNTIME}; use safekeeper::{wal_backup, HTTP_RUNTIME}; use storage_broker::DEFAULT_ENDPOINT; -use utils::auth::{JwtAuth, Scope}; +use utils::auth::{JwtAuth, Scope, SwappableJwtAuth}; use utils::{ id::NodeId, logging::{self, LogFormat}, @@ -251,10 +251,9 @@ async fn main() -> anyhow::Result<()> { None } Some(path) => { - info!("loading http auth JWT key from {path}"); - Some(Arc::new( - JwtAuth::from_key_path(path).context("failed to load the auth key")?, - )) + info!("loading http auth JWT key(s) from {path}"); + let jwt_auth = JwtAuth::from_key_path(path).context("failed to load the auth key")?; + Some(Arc::new(SwappableJwtAuth::new(jwt_auth))) } }; diff --git a/safekeeper/src/control_file_upgrade.rs b/safekeeper/src/control_file_upgrade.rs index a7f17df797..a0be2b2054 100644 --- a/safekeeper/src/control_file_upgrade.rs +++ b/safekeeper/src/control_file_upgrade.rs @@ -13,7 +13,7 @@ use utils::{ }; /// Persistent consensus state of the acceptor. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct AcceptorStateV1 { /// acceptor's last term it voted for (advanced in 1 phase) term: Term, @@ -21,7 +21,7 @@ struct AcceptorStateV1 { epoch: Term, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] struct SafeKeeperStateV1 { /// persistent acceptor state acceptor_state: AcceptorStateV1, @@ -50,7 +50,7 @@ pub struct ServerInfoV2 { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV2 { /// persistent acceptor state pub acceptor_state: AcceptorState, @@ -81,7 +81,7 @@ pub struct ServerInfoV3 { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV3 { /// persistent acceptor state pub acceptor_state: AcceptorState, @@ -101,7 +101,7 @@ pub struct SafeKeeperStateV3 { pub wal_start_lsn: Lsn, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperStateV4 { #[serde(with = "hex")] pub tenant_id: TenantId, @@ -264,3 +264,245 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result } bail!("unsupported safekeeper control file version {}", version) } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use utils::{id::NodeId, Hex}; + + use crate::safekeeper::PersistedPeerInfo; + + use super::*; + + #[test] + fn roundtrip_v1() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV1 { + acceptor_state: AcceptorStateV1 { + term: 42, + epoch: 43, + }, + server: ServerInfoV2 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + #[rustfmt::skip] + let expected = [ + // term + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // epoch + 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // pg_version + 0x0e, 0x00, 0x00, 0x00, + // system_id + 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, + // tenant_id + 0xcf, 0x04, 0x80, 0x92, 0x97, 0x07, 0xee, 0x75, 0x37, 0x23, 0x37, 0xef, 0xaa, 0x5e, 0xcf, 0x96, + // timeline_id + 0x11, 0x2d, 0xed, 0x66, 0x42, 0x2a, 0xa5, 0xe9, 0x53, 0xe5, 0x44, 0x0f, 0xa5, 0x42, 0x7a, 0xc4, + // wal_seg_size + 0x78, 0x56, 0x34, 0x12, + // proposer_uuid + 0xc4, 0x7a, 0x42, 0xa5, 0x0f, 0x44, 0xe5, 0x53, 0xe9, 0xa5, 0x2a, 0x42, 0x66, 0xed, 0x2d, 0x11, + // commit_lsn + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + // truncate_lsn + 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, 0x00, 0x00, + // wal_start_lsn + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV1::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v2() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV2 { + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfoV2 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, + 0x34, 0x12, 0xcf, 0x04, 0x80, 0x92, 0x97, 0x07, 0xee, 0x75, 0x37, 0x23, 0x37, 0xef, + 0xaa, 0x5e, 0xcf, 0x96, 0x11, 0x2d, 0xed, 0x66, 0x42, 0x2a, 0xa5, 0xe9, 0x53, 0xe5, + 0x44, 0x0f, 0xa5, 0x42, 0x7a, 0xc4, 0x78, 0x56, 0x34, 0x12, 0xc4, 0x7a, 0x42, 0xa5, + 0x0f, 0x44, 0xe5, 0x53, 0xe9, 0xa5, 0x2a, 0x42, 0x66, 0xed, 0x2d, 0x11, 0x78, 0x02, + 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV2::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v3() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV3 { + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfoV3 { + pg_version: 14, + system_id: 0x1234567887654321, + tenant_id, + timeline_id, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + commit_lsn: Lsn(1234567800), + truncate_lsn: Lsn(123456780), + wal_start_lsn: Lsn(1234567800 - 8), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x66, 0x30, 0x34, + 0x38, 0x30, 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, + 0x32, 0x33, 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, + 0x36, 0x36, 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, + 0x34, 0x34, 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x34, 0x37, 0x61, + 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, + 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xcd, 0x5b, 0x07, 0x00, 0x00, + 0x00, 0x00, 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV3::des(&ser).unwrap(); + + assert_eq!(state, deser); + } + + #[test] + fn roundtrip_v4() { + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperStateV4 { + tenant_id, + timeline_id, + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfo { + pg_version: 14, + system_id: 0x1234567887654321, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + peers: PersistedPeers(vec![( + NodeId(1), + PersistedPeerInfo { + backup_lsn: Lsn(1234567000), + term: 42, + flush_lsn: Lsn(1234567800 - 8), + commit_lsn: Lsn(1234567600), + }, + )]), + commit_lsn: Lsn(1234567800), + s3_wal_lsn: Lsn(1234567300), + peer_horizon_lsn: Lsn(9999999), + remote_consistent_lsn: Lsn(1234560000), + }; + + let ser = state.ser().unwrap(); + let expected = [ + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x66, 0x30, 0x34, 0x38, 0x30, + 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, 0x32, 0x33, + 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, 0x20, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, 0x36, 0x36, + 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, 0x34, 0x34, + 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, 0x2a, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, 0x78, 0x56, + 0x34, 0x12, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x34, 0x37, 0x61, + 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, + 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, 0x84, 0x00, 0x96, 0x49, 0x00, 0x00, + 0x00, 0x00, 0x7f, 0x96, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x95, 0x49, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0xff, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, + 0x00, 0x00, 0xb0, 0x01, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperStateV4::des(&ser).unwrap(); + + assert_eq!(state, deser); + } +} diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index ee9d7118c6..daf9255ecb 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -5,6 +5,7 @@ use std::fs::DirEntry; use std::io::BufReader; use std::io::Read; use std::path::PathBuf; +use std::sync::Arc; use anyhow::Result; use camino::Utf8Path; @@ -13,7 +14,6 @@ use postgres_ffi::XLogSegNo; use serde::Deserialize; use serde::Serialize; -use serde_with::{serde_as, DisplayFromStr}; use utils::id::NodeId; use utils::id::TenantTimelineId; use utils::id::{TenantId, TimelineId}; @@ -28,7 +28,7 @@ use crate::send_wal::WalSenderState; use crate::GlobalTimelines; /// Various filters that influence the resulting JSON output. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct Args { /// Dump all available safekeeper state. False by default. pub dump_all: bool, @@ -53,15 +53,76 @@ pub struct Args { } /// Response for debug dump request. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize)] pub struct Response { pub start_time: DateTime, pub finish_time: DateTime, - pub timelines: Vec, + pub timelines: Vec, pub timelines_count: usize, pub config: Config, } +pub struct TimelineDumpSer { + pub tli: Arc, + pub args: Args, + pub runtime: Arc, +} + +impl std::fmt::Debug for TimelineDumpSer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TimelineDumpSer") + .field("tli", &self.tli.ttid) + .field("args", &self.args) + .finish() + } +} + +impl Serialize for TimelineDumpSer { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let dump = self + .runtime + .block_on(build_from_tli_dump(self.tli.clone(), self.args.clone())); + dump.serialize(serializer) + } +} + +async fn build_from_tli_dump(timeline: Arc, args: Args) -> Timeline { + let control_file = if args.dump_control_file { + let mut state = timeline.get_state().await.1; + if !args.dump_term_history { + state.acceptor_state.term_history = TermHistory(vec![]); + } + Some(state) + } else { + None + }; + + let memory = if args.dump_memory { + Some(timeline.memory_dump().await) + } else { + None + }; + + let disk_content = if args.dump_disk_content { + // build_disk_content can fail, but we don't want to fail the whole + // request because of that. + build_disk_content(&timeline.timeline_dir).ok() + } else { + None + }; + + Timeline { + tenant_id: timeline.ttid.tenant_id, + timeline_id: timeline.ttid.timeline_id, + control_file, + memory, + disk_content, + } +} + /// Safekeeper configuration. #[derive(Debug, Serialize, Deserialize)] pub struct Config { @@ -74,12 +135,9 @@ pub struct Config { pub wal_backup_enabled: bool, } -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct Timeline { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub control_file: Option, pub memory: Option, @@ -140,8 +198,12 @@ pub async fn build(args: Args) -> Result { GlobalTimelines::get_all() }; - // TODO: return Stream instead of Vec let mut timelines = Vec::new(); + let runtime = Arc::new( + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(), + ); for tli in ptrs_snapshot { let ttid = tli.ttid; if let Some(tenant_id) = args.tenant_id { @@ -155,38 +217,11 @@ pub async fn build(args: Args) -> Result { } } - let control_file = if args.dump_control_file { - let mut state = tli.get_state().await.1; - if !args.dump_term_history { - state.acceptor_state.term_history = TermHistory(vec![]); - } - Some(state) - } else { - None - }; - - let memory = if args.dump_memory { - Some(tli.memory_dump().await) - } else { - None - }; - - let disk_content = if args.dump_disk_content { - // build_disk_content can fail, but we don't want to fail the whole - // request because of that. - build_disk_content(&tli.timeline_dir).ok() - } else { - None - }; - - let timeline = Timeline { - tenant_id: ttid.tenant_id, - timeline_id: ttid.timeline_id, - control_file, - memory, - disk_content, - }; - timelines.push(timeline); + timelines.push(TimelineDumpSer { + tli, + args: args.clone(), + runtime: runtime.clone(), + }); } let config = GlobalTimelines::get_global_config(); diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index 6d0ed8650d..d5333abae6 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use std::str::{self}; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::{info, info_span, Instrument}; +use tracing::{debug, info, info_span, Instrument}; use crate::auth::check_permission; use crate::json_ctrl::{handle_json_ctrl, AppendLogicalMessage}; @@ -165,26 +165,27 @@ impl postgres_backend::Handler .auth .as_ref() .expect("auth_type is configured but .auth of handler is missing"); - let data = - auth.decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?)?; + let data = auth + .decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?) + .map_err(|e| QueryError::Unauthorized(e.0))?; // The handler might be configured to allow only tenant scope tokens. if matches!(allowed_auth_scope, Scope::Tenant) && !matches!(data.claims.scope, Scope::Tenant) { - return Err(QueryError::Other(anyhow::anyhow!( - "passed JWT token is for full access, but only tenant scope is allowed" - ))); + return Err(QueryError::Unauthorized( + "passed JWT token is for full access, but only tenant scope is allowed".into(), + )); } if matches!(data.claims.scope, Scope::Tenant) && data.claims.tenant_id.is_none() { - return Err(QueryError::Other(anyhow::anyhow!( - "jwt token scope is Tenant, but tenant id is missing" - ))); + return Err(QueryError::Unauthorized( + "jwt token scope is Tenant, but tenant id is missing".into(), + )); } - info!( - "jwt auth succeeded for scope: {:#?} by tenant id: {:?}", + debug!( + "jwt scope check succeeded for scope: {:#?} by tenant id: {:?}", data.claims.scope, data.claims.tenant_id, ); @@ -263,7 +264,7 @@ impl SafekeeperPostgresHandler { // when accessing management api supply None as an argument // when using to authorize tenant pass corresponding tenant id - fn check_permission(&self, tenant_id: Option) -> anyhow::Result<()> { + fn check_permission(&self, tenant_id: Option) -> Result<(), QueryError> { if self.auth.is_none() { // auth is set to Trust, nothing to check so just return ok return Ok(()); @@ -275,7 +276,7 @@ impl SafekeeperPostgresHandler { .claims .as_ref() .expect("claims presence already checked"); - check_permission(claims, tenant_id) + check_permission(claims, tenant_id).map_err(|e| QueryError::Unauthorized(e.0)) } async fn handle_timeline_status( diff --git a/safekeeper/src/http/openapi_spec.yaml b/safekeeper/src/http/openapi_spec.yaml index 51ce7589a0..a617e0310c 100644 --- a/safekeeper/src/http/openapi_spec.yaml +++ b/safekeeper/src/http/openapi_spec.yaml @@ -86,6 +86,41 @@ paths: default: $ref: "#/components/responses/GenericError" + /v1/tenant/{tenant_id}/timeline/{source_timeline_id}/copy: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex + - name: source_timeline_id + in: path + required: true + schema: + type: string + format: hex + + post: + tags: + - "Timeline" + summary: Register new timeline as copy of existing timeline + description: "" + operationId: v1CopyTenantTimeline + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/TimelineCopyRequest" + responses: + "201": + description: Timeline created + # TODO: return timeline info? + "403": + $ref: "#/components/responses/ForbiddenError" + default: + $ref: "#/components/responses/GenericError" + /v1/tenant/{tenant_id}/timeline/{timeline_id}: parameters: @@ -210,6 +245,18 @@ components: type: integer minimum: 0 + TimelineCopyRequest: + type: object + required: + - target_timeline_id + - until_lsn + properties: + target_timeline_id: + type: string + format: hex + until_lsn: + type: string + SkTimelineInfo: type: object required: diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 940ac82df6..c48b5330b3 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -4,7 +4,6 @@ use once_cell::sync::Lazy; use postgres_ffi::WAL_SEGMENT_SIZE; use safekeeper_api::models::SkTimelineInfo; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use std::collections::{HashMap, HashSet}; use std::fmt; use std::str::FromStr; @@ -13,7 +12,12 @@ use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use tokio::fs::File; use tokio::io::AsyncReadExt; -use utils::http::endpoint::request_span; + +use std::io::Write as _; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tracing::info_span; +use utils::http::endpoint::{request_span, ChannelWriter}; use crate::receive_wal::WalReceiverState; use crate::safekeeper::Term; @@ -26,7 +30,7 @@ use crate::timelines_global_map::TimelineDeleteForceResult; use crate::GlobalTimelines; use crate::SafeKeeperConf; use utils::{ - auth::JwtAuth, + auth::SwappableJwtAuth, http::{ endpoint::{self, auth_middleware, check_permission_with}, error::ApiError, @@ -62,11 +66,9 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { /// Same as TermLsn, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. -#[serde_as] #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct TermSwitchApiEntry { pub term: Term, - #[serde_as(as = "DisplayFromStr")] pub lsn: Lsn, } @@ -88,28 +90,18 @@ pub struct AcceptorStateStatus { } /// Info about timeline on safekeeper ready for reporting. -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct TimelineStatus { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub acceptor_state: AcceptorStateStatus, pub pg_info: ServerInfo, - #[serde_as(as = "DisplayFromStr")] pub flush_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub timeline_start_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub local_start_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub backup_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub peer_horizon_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, pub peers: Vec, pub walsenders: Vec, @@ -373,8 +365,52 @@ async fn dump_debug_handler(mut request: Request) -> Result .await .map_err(ApiError::InternalServerError)?; - // TODO: use streaming response - json_response(StatusCode::OK, resp) + let started_at = std::time::Instant::now(); + + let (tx, rx) = mpsc::channel(1); + + let body = Body::wrap_stream(ReceiverStream::new(rx)); + + let mut writer = ChannelWriter::new(128 * 1024, tx); + + let response = Response::builder() + .status(200) + .header(hyper::header::CONTENT_TYPE, "application/octet-stream") + .body(body) + .unwrap(); + + let span = info_span!("blocking"); + tokio::task::spawn_blocking(move || { + let _span = span.entered(); + + let res = serde_json::to_writer(&mut writer, &resp) + .map_err(std::io::Error::from) + .and_then(|_| writer.flush()); + + match res { + Ok(()) => { + tracing::info!( + bytes = writer.flushed_bytes(), + elapsed_ms = started_at.elapsed().as_millis(), + "responded /v1/debug_dump" + ); + } + Err(e) => { + tracing::warn!("failed to write out /v1/debug_dump response: {e:#}"); + // semantics of this error are quite... unclear. we want to error the stream out to + // abort the response to somehow notify the client that we failed. + // + // though, most likely the reason for failure is that the receiver is already gone. + drop( + writer + .tx + .blocking_send(Err(std::io::ErrorKind::BrokenPipe.into())), + ); + } + } + }); + + Ok(response) } /// Safekeeper http router. @@ -392,8 +428,11 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder if ALLOWLIST_ROUTES.contains(request.uri()) { None } else { - // Option> is always provided as data below, hence unwrap(). - request.data::>>().unwrap().as_deref() + // Option> is always provided as data below, hence unwrap(). + request + .data::>>() + .unwrap() + .as_deref() } })) } diff --git a/safekeeper/src/json_ctrl.rs b/safekeeper/src/json_ctrl.rs index 2ad2ca7706..303bdd67fe 100644 --- a/safekeeper/src/json_ctrl.rs +++ b/safekeeper/src/json_ctrl.rs @@ -44,8 +44,11 @@ pub struct AppendLogicalMessage { // fields from AppendRequestHeader pub term: Term, + #[serde(with = "utils::lsn::serde_as_u64")] pub epoch_start_lsn: Lsn, + #[serde(with = "utils::lsn::serde_as_u64")] pub begin_lsn: Lsn, + #[serde(with = "utils::lsn::serde_as_u64")] pub truncate_lsn: Lsn, pub pg_version: u32, } diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index aa70cee591..3a086f1f54 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -1,3 +1,4 @@ +#![deny(clippy::undocumented_unsafe_blocks)] use camino::Utf8PathBuf; use once_cell::sync::Lazy; use remote_storage::RemoteStorageConfig; @@ -6,7 +7,10 @@ use tokio::runtime::Runtime; use std::time::Duration; use storage_broker::Uri; -use utils::id::{NodeId, TenantId, TenantTimelineId}; +use utils::{ + auth::SwappableJwtAuth, + id::{NodeId, TenantId, TenantTimelineId}, +}; mod auth; pub mod broker; @@ -69,7 +73,7 @@ pub struct SafeKeeperConf { pub wal_backup_enabled: bool, pub pg_auth: Option>, pub pg_tenant_only_auth: Option>, - pub http_auth: Option>, + pub http_auth: Option>, pub current_thread_runtime: bool, } diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 1343bba5cc..ad3a18a536 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -1,3 +1,4 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use anyhow::{bail, Context, Result}; @@ -5,8 +6,6 @@ use tokio::io::AsyncWriteExt; use tracing::info; use utils::id::{TenantId, TenantTimelineId, TimelineId}; -use serde_with::{serde_as, DisplayFromStr}; - use crate::{ control_file, debug_dump, http::routes::TimelineStatus, @@ -15,12 +14,9 @@ use crate::{ }; /// Info about timeline on safekeeper ready for reporting. -#[serde_as] #[derive(Debug, Serialize, Deserialize)] pub struct Request { - #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - #[serde_as(as = "DisplayFromStr")] pub timeline_id: TimelineId, pub http_hosts: Vec, } @@ -32,6 +28,16 @@ pub struct Response { // TODO: add more fields? } +/// Response for debug dump request. +#[derive(Debug, Serialize, Deserialize)] +pub struct DebugDumpResponse { + pub start_time: DateTime, + pub finish_time: DateTime, + pub timelines: Vec, + pub timelines_count: usize, + pub config: debug_dump::Config, +} + /// Find the most advanced safekeeper and pull timeline from it. pub async fn handle_request(request: Request) -> Result { let existing_tli = GlobalTimelines::get(TenantTimelineId::new( @@ -103,7 +109,7 @@ async fn pull_timeline(status: TimelineStatus, host: String) -> Result // Implementing our own scp over HTTP. // At first, we need to fetch list of files from safekeeper. - let dump: debug_dump::Response = client + let dump: DebugDumpResponse = client .get(format!( "{}/v1/debug_dump?dump_all=true&tenant_id={}&timeline_id={}", host, status.tenant_id, status.timeline_id diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index 934dda10b2..9ce9b049ba 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -111,7 +111,7 @@ impl WalReceivers { .count() } - /// Unregister walsender. + /// Unregister walreceiver. fn unregister(self: &Arc, id: WalReceiverId) { let mut shared = self.mutex.lock(); shared.slots[id] = None; @@ -138,8 +138,8 @@ pub enum WalReceiverStatus { Streaming, } -/// Scope guard to access slot in WalSenders registry and unregister from it in -/// Drop. +/// Scope guard to access slot in WalReceivers registry and unregister from +/// it in Drop. pub struct WalReceiverGuard { id: WalReceiverId, walreceivers: Arc, diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 1437bdb50e..47a624281d 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -52,7 +52,7 @@ impl From<(Term, Lsn)> for TermLsn { } } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq)] pub struct TermHistory(pub Vec); impl TermHistory { @@ -178,7 +178,7 @@ impl fmt::Debug for TermHistory { pub type PgUuid = [u8; 16]; /// Persistent consensus state of the acceptor. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct AcceptorState { /// acceptor's last term it voted for (advanced in 1 phase) pub term: Term, @@ -209,16 +209,16 @@ pub struct ServerInfo { pub wal_seg_size: u32, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct PersistedPeerInfo { /// LSN up to which safekeeper offloaded WAL to s3. - backup_lsn: Lsn, + pub backup_lsn: Lsn, /// Term of the last entry. - term: Term, + pub term: Term, /// LSN of the last record. - flush_lsn: Lsn, + pub flush_lsn: Lsn, /// Up to which LSN safekeeper regards its WAL as committed. - commit_lsn: Lsn, + pub commit_lsn: Lsn, } impl PersistedPeerInfo { @@ -232,12 +232,12 @@ impl PersistedPeerInfo { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct PersistedPeers(pub Vec<(NodeId, PersistedPeerInfo)>); /// Persistent information stored on safekeeper node /// On disk data is prefixed by magic and format version and followed by checksum. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SafeKeeperState { #[serde(with = "hex")] pub tenant_id: TenantId, @@ -1096,7 +1096,7 @@ mod tests { use super::*; use crate::wal_storage::Storage; - use std::{ops::Deref, time::Instant}; + use std::{ops::Deref, str::FromStr, time::Instant}; // fake storage for tests struct InMemoryState { @@ -1314,4 +1314,98 @@ mod tests { }) ); } + + #[test] + fn test_sk_state_bincode_serde_roundtrip() { + use utils::Hex; + let tenant_id = TenantId::from_str("cf0480929707ee75372337efaa5ecf96").unwrap(); + let timeline_id = TimelineId::from_str("112ded66422aa5e953e5440fa5427ac4").unwrap(); + let state = SafeKeeperState { + tenant_id, + timeline_id, + acceptor_state: AcceptorState { + term: 42, + term_history: TermHistory(vec![TermLsn { + lsn: Lsn(0x1), + term: 41, + }]), + }, + server: ServerInfo { + pg_version: 14, + system_id: 0x1234567887654321, + wal_seg_size: 0x12345678, + }, + proposer_uuid: { + let mut arr = timeline_id.as_arr(); + arr.reverse(); + arr + }, + timeline_start_lsn: Lsn(0x12345600), + local_start_lsn: Lsn(0x12), + commit_lsn: Lsn(1234567800), + backup_lsn: Lsn(1234567300), + peer_horizon_lsn: Lsn(9999999), + remote_consistent_lsn: Lsn(1234560000), + peers: PersistedPeers(vec![( + NodeId(1), + PersistedPeerInfo { + backup_lsn: Lsn(1234567000), + term: 42, + flush_lsn: Lsn(1234567800 - 8), + commit_lsn: Lsn(1234567600), + }, + )]), + }; + + let ser = state.ser().unwrap(); + + #[rustfmt::skip] + let expected = [ + // tenant_id as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x66, 0x30, 0x34, 0x38, 0x30, 0x39, 0x32, 0x39, 0x37, 0x30, 0x37, 0x65, 0x65, 0x37, 0x35, 0x33, 0x37, 0x32, 0x33, 0x33, 0x37, 0x65, 0x66, 0x61, 0x61, 0x35, 0x65, 0x63, 0x66, 0x39, 0x36, + // timeline_id as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x31, 0x31, 0x32, 0x64, 0x65, 0x64, 0x36, 0x36, 0x34, 0x32, 0x32, 0x61, 0x61, 0x35, 0x65, 0x39, 0x35, 0x33, 0x65, 0x35, 0x34, 0x34, 0x30, 0x66, 0x61, 0x35, 0x34, 0x32, 0x37, 0x61, 0x63, 0x34, + // term + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // length prefix + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // unsure why this order is swapped + 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // pg_version + 0x0e, 0x00, 0x00, 0x00, + // systemid + 0x21, 0x43, 0x65, 0x87, 0x78, 0x56, 0x34, 0x12, + // wal_seg_size + 0x78, 0x56, 0x34, 0x12, + // pguuid as length prefixed hex + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x34, 0x37, 0x61, 0x34, 0x32, 0x61, 0x35, 0x30, 0x66, 0x34, 0x34, 0x65, 0x35, 0x35, 0x33, 0x65, 0x39, 0x61, 0x35, 0x32, 0x61, 0x34, 0x32, 0x36, 0x36, 0x65, 0x64, 0x32, 0x64, 0x31, 0x31, + + // timeline_start_lsn + 0x00, 0x56, 0x34, 0x12, 0x00, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x00, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0x96, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe4, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + // length prefix for persistentpeers + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // nodeid + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // backuplsn + 0x58, 0xff, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + 0xb0, 0x01, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, + ]; + + assert_eq!(Hex(&ser), Hex(&expected)); + + let deser = SafeKeeperState::des(&ser).unwrap(); + + assert_eq!(deser, state); + } } diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index cd765dcbce..44f14f8c7e 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -16,7 +16,6 @@ use postgres_ffi::get_current_timestamp; use postgres_ffi::{TimestampTz, MAX_SEND_SIZE}; use pq_proto::{BeMessage, WalSndKeepAlive, XLogDataBody}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, DisplayFromStr}; use tokio::io::{AsyncRead, AsyncWrite}; use utils::id::TenantTimelineId; use utils::lsn::AtomicLsn; @@ -313,10 +312,8 @@ impl WalSendersShared { } // Serialized is used only for pretty printing in json. -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WalSenderState { - #[serde_as(as = "DisplayFromStr")] ttid: TenantTimelineId, addr: SocketAddr, conn_id: ConnectionId, diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 96d844fa7a..2ba871207e 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -5,10 +5,8 @@ use anyhow::{anyhow, bail, Result}; use camino::Utf8PathBuf; use postgres_ffi::XLogSegNo; use serde::{Deserialize, Serialize}; -use serde_with::serde_as; use tokio::fs; -use serde_with::DisplayFromStr; use std::cmp::max; use std::sync::Arc; use std::time::Duration; @@ -42,7 +40,6 @@ use crate::SafeKeeperConf; use crate::{debug_dump, wal_storage}; /// Things safekeeper should know about timeline state on peers. -#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerInfo { pub sk_id: NodeId, @@ -50,13 +47,10 @@ pub struct PeerInfo { /// Term of the last entry. pub last_log_term: Term, /// LSN of the last record. - #[serde_as(as = "DisplayFromStr")] pub flush_lsn: Lsn, - #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, /// Since which LSN safekeeper has WAL. TODO: remove this once we fill new /// sk since backup_lsn. - #[serde_as(as = "DisplayFromStr")] pub local_start_lsn: Lsn, /// When info was received. Serde annotations are not very useful but make /// the code compile -- we don't rely on this field externally. diff --git a/scripts/export_import_between_pageservers.py b/scripts/export_import_between_pageservers.py index fca645078a..77e4310eac 100755 --- a/scripts/export_import_between_pageservers.py +++ b/scripts/export_import_between_pageservers.py @@ -18,7 +18,7 @@ # 6. We wait for the new pageserver's remote_consistent_lsn to catch up # # For more context on how to use this, see: -# https://github.com/neondatabase/cloud/wiki/Storage-format-migration +# https://www.notion.so/neondatabase/Storage-format-migration-9a8eba33ccf8417ea8cf50e6a0c542cf import argparse import os diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 81a7b0750d..6737ca5fe3 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -361,7 +361,6 @@ class PgProtocol: @dataclass class AuthKeys: - pub: str priv: str def generate_token(self, *, scope: str, **token_data: str) -> str: @@ -626,6 +625,8 @@ class NeonEnvBuilder: sk.stop(immediate=True) for pageserver in self.env.pageservers: + pageserver.assert_no_metric_errors() + pageserver.stop(immediate=True) if self.env.attachment_service is not None: @@ -875,9 +876,31 @@ class NeonEnv: @cached_property def auth_keys(self) -> AuthKeys: - pub = (Path(self.repo_dir) / "auth_public_key.pem").read_text() priv = (Path(self.repo_dir) / "auth_private_key.pem").read_text() - return AuthKeys(pub=pub, priv=priv) + return AuthKeys(priv=priv) + + def regenerate_keys_at(self, privkey_path: Path, pubkey_path: Path): + # compare generate_auth_keys() in local_env.rs + subprocess.run( + ["openssl", "genpkey", "-algorithm", "ed25519", "-out", privkey_path], + cwd=self.repo_dir, + check=True, + ) + + subprocess.run( + [ + "openssl", + "pkey", + "-in", + privkey_path, + "-pubout", + "-out", + pubkey_path, + ], + cwd=self.repo_dir, + check=True, + ) + del self.auth_keys def generate_endpoint_id(self) -> str: """ @@ -1784,6 +1807,21 @@ class NeonPageserver(PgProtocol): assert not errors + def assert_no_metric_errors(self): + """ + Certain metrics should _always_ be zero: they track conditions that indicate a bug. + """ + if not self.running: + log.info(f"Skipping metrics check on pageserver {self.id}, it is not running") + return + + for metric in [ + "pageserver_tenant_manager_unexpected_errors_total", + "pageserver_deletion_queue_unexpected_errors_total", + ]: + value = self.http_client().get_metric_value(metric) + assert value == 0, f"Nonzero {metric} == {value}" + def log_contains(self, pattern: str) -> Optional[str]: """Check that the pageserver log contains a line that matches the given regex""" logfile = open(os.path.join(self.workdir, "pageserver.log"), "r") @@ -1833,6 +1871,8 @@ def append_pageserver_param_overrides( params_to_update.append( f"--pageserver-config-override=remote_storage={remote_storage_toml_table}" ) + else: + params_to_update.append('--pageserver-config-override=remote_storage=""') env_overrides = os.getenv("NEON_PAGESERVER_OVERRIDES") if env_overrides is not None: @@ -2139,6 +2179,29 @@ class NeonProxy(PgProtocol): *["--allow-self-signed-compute", "true"], ] + class Console(AuthBackend): + def __init__(self, endpoint: str, fixed_rate_limit: Optional[int] = None): + self.endpoint = endpoint + self.fixed_rate_limit = fixed_rate_limit + + def extra_args(self) -> list[str]: + args = [ + # Console auth backend params + *["--auth-backend", "console"], + *["--auth-endpoint", self.endpoint], + ] + if self.fixed_rate_limit is not None: + args += [ + *["--disable-dynamic-rate-limiter", "false"], + *["--rate-limit-algorithm", "aimd"], + *["--initial-limit", str(1)], + *["--rate-limiter-timeout", "1s"], + *["--aimd-min-limit", "0"], + *["--aimd-increase-by", "1"], + *["--wake-compute-cache", "size=0"], # Disable cache to test rate limiter. + ] + return args + @dataclass(frozen=True) class Postgres(AuthBackend): pg_conn_url: str @@ -2868,7 +2931,7 @@ class SafekeeperHttpClient(requests.Session): params = params or {} res = self.get(f"http://localhost:{self.port}/v1/debug_dump", params=params) res.raise_for_status() - res_json = res.json() + res_json = json.loads(res.text) assert isinstance(res_json, dict) return res_json @@ -2968,24 +3031,33 @@ class S3Scrubber: self.env = env self.log_dir = log_dir - def scrubber_cli(self, args, timeout): + def scrubber_cli(self, args: list[str], timeout) -> str: assert isinstance(self.env.pageserver_remote_storage, S3Storage) s3_storage = self.env.pageserver_remote_storage env = { "REGION": s3_storage.bucket_region, "BUCKET": s3_storage.bucket_name, + "BUCKET_PREFIX": s3_storage.prefix_in_bucket, + "RUST_LOG": "DEBUG", } env.update(s3_storage.access_env_vars()) if s3_storage.endpoint is not None: env.update({"AWS_ENDPOINT_URL": s3_storage.endpoint}) - base_args = [self.env.neon_binpath / "s3_scrubber"] + base_args = [str(self.env.neon_binpath / "s3_scrubber")] args = base_args + args - (output_path, _, status_code) = subprocess_capture( - self.log_dir, args, echo_stderr=True, echo_stdout=True, env=env, check=False + (output_path, stdout, status_code) = subprocess_capture( + self.log_dir, + args, + echo_stderr=True, + echo_stdout=True, + env=env, + check=False, + capture_stdout=True, + timeout=timeout, ) if status_code: log.warning(f"Scrub command {args} failed") @@ -2994,8 +3066,18 @@ class S3Scrubber: raise RuntimeError("Remote storage scrub failed") - def scan_metadata(self): - self.scrubber_cli(["scan-metadata"], timeout=30) + assert stdout is not None + return stdout + + def scan_metadata(self) -> Any: + stdout = self.scrubber_cli(["scan-metadata", "--json"], timeout=30) + + try: + return json.loads(stdout) + except: + log.error("Failed to decode JSON output from `scan-metadata`. Dumping stdout:") + log.error(stdout) + raise def get_test_output_dir(request: FixtureRequest, top_output_dir: Path) -> Path: diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index d2c3715c52..aff7959aa7 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -189,6 +189,10 @@ class PageserverHttpClient(requests.Session): assert res_json is None return res_json + def reload_auth_validation_keys(self): + res = self.post(f"http://localhost:{self.port}/v1/reload_auth_validation_keys") + self.verbose_error(res) + def tenant_list(self) -> List[Dict[Any, Any]]: res = self.get(f"http://localhost:{self.port}/v1/tenant") self.verbose_error(res) diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index e54b82dfb4..ba8d70d5a9 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -35,6 +35,7 @@ def subprocess_capture( echo_stderr=False, echo_stdout=False, capture_stdout=False, + timeout=None, **kwargs: Any, ) -> Tuple[str, Optional[str], int]: """Run a process and bifurcate its output to files and the `log` logger @@ -104,7 +105,7 @@ def subprocess_capture( stderr_handler = OutputHandler(p.stderr, stderr_f, echo=echo_stderr, capture=False) stderr_handler.start() - r = p.wait() + r = p.wait(timeout=timeout) stdout_handler.join() stderr_handler.join() diff --git a/test_runner/regress/test_auth.py b/test_runner/regress/test_auth.py index 76b75c1caf..f729bdee98 100644 --- a/test_runner/regress/test_auth.py +++ b/test_runner/regress/test_auth.py @@ -1,12 +1,35 @@ +import os from contextlib import closing +from pathlib import Path import psycopg2 import pytest -from fixtures.neon_fixtures import NeonEnvBuilder, PgProtocol -from fixtures.pageserver.http import PageserverApiException +from fixtures.neon_fixtures import ( + NeonEnv, + NeonEnvBuilder, + PgProtocol, +) +from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.types import TenantId, TimelineId +def assert_client_authorized(env: NeonEnv, http_client: PageserverHttpClient): + http_client.timeline_create( + pg_version=env.pg_version, + tenant_id=env.initial_tenant, + new_timeline_id=TimelineId.generate(), + ancestor_timeline_id=env.initial_timeline, + ) + + +def assert_client_not_authorized(env: NeonEnv, http_client: PageserverHttpClient): + with pytest.raises( + PageserverApiException, + match="Forbidden: JWT authentication error", + ): + assert_client_authorized(env, http_client) + + def test_pageserver_auth(neon_env_builder: NeonEnvBuilder): neon_env_builder.auth_enabled = True env = neon_env_builder.init_start() @@ -27,30 +50,14 @@ def test_pageserver_auth(neon_env_builder: NeonEnvBuilder): ps.safe_psql("set FOO", password=pageserver_token) # tenant can create branches - tenant_http_client.timeline_create( - pg_version=env.pg_version, - tenant_id=env.initial_tenant, - new_timeline_id=TimelineId.generate(), - ancestor_timeline_id=env.initial_timeline, - ) + assert_client_authorized(env, tenant_http_client) + # console can create branches for tenant - pageserver_http_client.timeline_create( - pg_version=env.pg_version, - tenant_id=env.initial_tenant, - new_timeline_id=TimelineId.generate(), - ancestor_timeline_id=env.initial_timeline, - ) + assert_client_authorized(env, pageserver_http_client) # fail to create branch using token with different tenant_id - with pytest.raises( - PageserverApiException, match="Forbidden: Tenant id mismatch. Permission denied" - ): - invalid_tenant_http_client.timeline_create( - pg_version=env.pg_version, - tenant_id=env.initial_tenant, - new_timeline_id=TimelineId.generate(), - ancestor_timeline_id=env.initial_timeline, - ) + with pytest.raises(PageserverApiException, match="Forbidden: JWT authentication error"): + assert_client_authorized(env, invalid_tenant_http_client) # create tenant using management token pageserver_http_client.tenant_create(TenantId.generate()) @@ -58,7 +65,7 @@ def test_pageserver_auth(neon_env_builder: NeonEnvBuilder): # fail to create tenant using tenant token with pytest.raises( PageserverApiException, - match="Forbidden: Attempt to access management api with tenant scope. Permission denied", + match="Forbidden: JWT authentication error", ): tenant_http_client.tenant_create(TenantId.generate()) @@ -82,6 +89,96 @@ def test_compute_auth_to_pageserver(neon_env_builder: NeonEnvBuilder): assert cur.fetchone() == (5000050000,) +def test_pageserver_multiple_keys(neon_env_builder: NeonEnvBuilder): + neon_env_builder.auth_enabled = True + env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*Authentication error: InvalidSignature.*") + env.pageserver.allowed_errors.append(".*Unauthorized: malformed jwt token.*") + + pageserver_token_old = env.auth_keys.generate_pageserver_token() + pageserver_http_client_old = env.pageserver.http_client(pageserver_token_old) + + pageserver_http_client_old.reload_auth_validation_keys() + + # This test is to ensure that the pageserver supports multiple keys. + # The neon_local tool generates one key pair at a hardcoded path by default. + # As a preparation for our test, move the public key of the key pair into a + # directory at the same location as the hardcoded path by: + # 1. moving the the file at `configured_pub_key_path` to a temporary location + # 2. creating a new directory at `configured_pub_key_path` + # 3. moving the file from the temporary location into the newly created directory + configured_pub_key_path = Path(env.repo_dir) / "auth_public_key.pem" + os.rename(configured_pub_key_path, Path(env.repo_dir) / "auth_public_key.pem.file") + os.mkdir(configured_pub_key_path) + os.rename( + Path(env.repo_dir) / "auth_public_key.pem.file", + configured_pub_key_path / "auth_public_key_old.pem", + ) + + # Add a new key pair + # This invalidates env.auth_keys and makes them be regenerated + env.regenerate_keys_at( + Path("auth_private_key.pem"), Path("auth_public_key.pem/auth_public_key_new.pem") + ) + + # Reload the keys on the pageserver side + pageserver_http_client_old.reload_auth_validation_keys() + + # We can continue doing things using the old token + assert_client_authorized(env, pageserver_http_client_old) + + pageserver_token_new = env.auth_keys.generate_pageserver_token() + pageserver_http_client_new = env.pageserver.http_client(pageserver_token_new) + + # The new token also works + assert_client_authorized(env, pageserver_http_client_new) + + # Remove the old token and reload + os.remove(Path(env.repo_dir) / "auth_public_key.pem" / "auth_public_key_old.pem") + pageserver_http_client_old.reload_auth_validation_keys() + + # Reloading fails now with the old token, but the new token still works + assert_client_not_authorized(env, pageserver_http_client_old) + assert_client_authorized(env, pageserver_http_client_new) + + +def test_pageserver_key_reload(neon_env_builder: NeonEnvBuilder): + neon_env_builder.auth_enabled = True + env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*Authentication error: InvalidSignature.*") + env.pageserver.allowed_errors.append(".*Unauthorized: malformed jwt token.*") + + pageserver_token_old = env.auth_keys.generate_pageserver_token() + pageserver_http_client_old = env.pageserver.http_client(pageserver_token_old) + + pageserver_http_client_old.reload_auth_validation_keys() + + # Regenerate the keys + env.regenerate_keys_at(Path("auth_private_key.pem"), Path("auth_public_key.pem")) + + # Reload the keys on the pageserver side + pageserver_http_client_old.reload_auth_validation_keys() + + # Next attempt fails as we use the old auth token + with pytest.raises( + PageserverApiException, + match="Forbidden: JWT authentication error", + ): + pageserver_http_client_old.reload_auth_validation_keys() + + # same goes for attempts trying to create a timeline + assert_client_not_authorized(env, pageserver_http_client_old) + + pageserver_token_new = env.auth_keys.generate_pageserver_token() + pageserver_http_client_new = env.pageserver.http_client(pageserver_token_new) + + # timeline creation works with the new token + assert_client_authorized(env, pageserver_http_client_new) + + # reloading also works with the new token + pageserver_http_client_new.reload_auth_validation_keys() + + @pytest.mark.parametrize("auth_enabled", [False, True]) def test_auth_failures(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): neon_env_builder.auth_enabled = auth_enabled diff --git a/test_runner/regress/test_bad_connection.py b/test_runner/regress/test_bad_connection.py new file mode 100644 index 0000000000..ba0624c730 --- /dev/null +++ b/test_runner/regress/test_bad_connection.py @@ -0,0 +1,60 @@ +import random +import time + +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnvBuilder + + +def test_compute_pageserver_connection_stress(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + env.pageserver.allowed_errors.append(".*simulated connection error.*") + + pageserver_http = env.pageserver.http_client() + env.neon_cli.create_branch("test_compute_pageserver_connection_stress") + endpoint = env.endpoints.create_start("test_compute_pageserver_connection_stress") + + # Enable failpoint after starting everything else up so that loading initial + # basebackup doesn't fail + pageserver_http.configure_failpoints(("simulated-bad-compute-connection", "50%return(15)")) + + pg_conn = endpoint.connect() + cur = pg_conn.cursor() + + # Create table, and insert some rows. Make it big enough that it doesn't fit in + # shared_buffers, otherwise the SELECT after restart will just return answer + # from shared_buffers without hitting the page server, which defeats the point + # of this test. + cur.execute("CREATE TABLE foo (t text)") + cur.execute( + """ + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 100000) g + """ + ) + + # Verify that the table is larger than shared_buffers + cur.execute( + """ + select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_size + from pg_settings where name = 'shared_buffers' + """ + ) + row = cur.fetchone() + assert row is not None + log.info(f"shared_buffers is {row[0]}, table size {row[1]}") + assert int(row[0]) < int(row[1]) + + cur.execute("SELECT count(*) FROM foo") + assert cur.fetchone() == (100000,) + + end_time = time.time() + 30 + times_executed = 0 + while time.time() < end_time: + if random.random() < 0.5: + cur.execute("INSERT INTO foo VALUES ('stas'), ('heikki')") + else: + cur.execute("SELECT t FROM foo ORDER BY RANDOM() LIMIT 10") + cur.fetchall() + times_executed += 1 + log.info(f"Workload executed {times_executed} times") diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index b1b47b3f2c..23839a4dd1 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -26,6 +26,7 @@ def test_local_corruption(neon_env_builder: NeonEnvBuilder): ".*will not become active. Current state: Broken.*", ".*failed to load metadata.*", ".*load failed.*load local timeline.*", + ".*layer loading failed permanently: load layer: .*", ] ) diff --git a/test_runner/regress/test_change_pageserver.py b/test_runner/regress/test_change_pageserver.py index 31092b70b9..410bf03c2b 100644 --- a/test_runner/regress/test_change_pageserver.py +++ b/test_runner/regress/test_change_pageserver.py @@ -1,9 +1,13 @@ +import asyncio + from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder from fixtures.remote_storage import RemoteStorageKind def test_change_pageserver(neon_env_builder: NeonEnvBuilder): + num_connections = 3 + neon_env_builder.num_pageservers = 2 neon_env_builder.enable_pageserver_remote_storage( remote_storage_kind=RemoteStorageKind.MOCK_S3, @@ -16,15 +20,24 @@ def test_change_pageserver(neon_env_builder: NeonEnvBuilder): alt_pageserver_id = env.pageservers[1].id env.pageservers[1].tenant_attach(env.initial_tenant) - pg_conn = endpoint.connect() - cur = pg_conn.cursor() + pg_conns = [endpoint.connect() for i in range(num_connections)] + curs = [pg_conn.cursor() for pg_conn in pg_conns] + + def execute(statement: str): + for cur in curs: + cur.execute(statement) + + def fetchone(): + results = [cur.fetchone() for cur in curs] + assert all(result == results[0] for result in results) + return results[0] # Create table, and insert some rows. Make it big enough that it doesn't fit in # shared_buffers, otherwise the SELECT after restart will just return answer # from shared_buffers without hitting the page server, which defeats the point # of this test. - cur.execute("CREATE TABLE foo (t text)") - cur.execute( + curs[0].execute("CREATE TABLE foo (t text)") + curs[0].execute( """ INSERT INTO foo SELECT 'long string to consume some space' || g @@ -33,25 +46,25 @@ def test_change_pageserver(neon_env_builder: NeonEnvBuilder): ) # Verify that the table is larger than shared_buffers - cur.execute( + curs[0].execute( """ select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_size from pg_settings where name = 'shared_buffers' """ ) - row = cur.fetchone() + row = curs[0].fetchone() assert row is not None log.info(f"shared_buffers is {row[0]}, table size {row[1]}") assert int(row[0]) < int(row[1]) - cur.execute("SELECT count(*) FROM foo") - assert cur.fetchone() == (100000,) + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) endpoint.reconfigure(pageserver_id=alt_pageserver_id) # Verify that the neon.pageserver_connstring GUC is set to the correct thing - cur.execute("SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'") - connstring = cur.fetchone() + execute("SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'") + connstring = fetchone() assert connstring is not None expected_connstring = f"postgresql://no_user:@localhost:{env.pageservers[1].service_port.pg}" assert expected_connstring == expected_connstring @@ -60,5 +73,45 @@ def test_change_pageserver(neon_env_builder: NeonEnvBuilder): 0 ].stop() # Stop the old pageserver just to make sure we're reading from the new one - cur.execute("SELECT count(*) FROM foo") - assert cur.fetchone() == (100000,) + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) + + # Try failing back, and this time we will stop the current pageserver before reconfiguring + # the endpoint. Whereas the previous reconfiguration was like a healthy migration, this + # is more like what happens in an unexpected pageserver failure. + env.pageservers[0].start() + env.pageservers[1].stop() + + endpoint.reconfigure(pageserver_id=env.pageservers[0].id) + + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) + + env.pageservers[0].stop() + env.pageservers[1].start() + + # Test a (former) bug where a child process spins without updating its connection string + # by executing a query separately. This query will hang until we issue the reconfigure. + async def reconfigure_async(): + await asyncio.sleep( + 1 + ) # Sleep for 1 second just to make sure we actually started our count(*) query + endpoint.reconfigure(pageserver_id=env.pageservers[1].id) + + def execute_count(): + execute("SELECT count(*) FROM FOO") + + async def execute_and_reconfigure(): + task_exec = asyncio.to_thread(execute_count) + task_reconfig = asyncio.create_task(reconfigure_async()) + await asyncio.gather( + task_exec, + task_reconfig, + ) + + asyncio.run(execute_and_reconfigure()) + assert fetchone() == (100000,) + + # One final check that nothing hangs + execute("SELECT count(*) FROM foo") + assert fetchone() == (100000,) diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 161662bc99..98f6677c00 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -449,7 +449,7 @@ def check_neon_works( ) # Check that project can be recovered from WAL - # loosely based on https://github.com/neondatabase/cloud/wiki/Recovery-from-WAL + # loosely based on https://www.notion.so/neondatabase/Storage-Recovery-from-WAL-d92c0aac0ebf40df892b938045d7d720 tenant_id = snapshot_config["default_tenant_id"] timeline_id = dict(snapshot_config["branch_name_mappings"]["main"])[tenant_id] pageserver_port = snapshot_config["pageservers"][0]["listen_http_addr"].split(":")[-1] diff --git a/test_runner/regress/test_ddl_forwarding.py b/test_runner/regress/test_ddl_forwarding.py index 6bd09c7030..07ee8b405e 100644 --- a/test_runner/regress/test_ddl_forwarding.py +++ b/test_runner/regress/test_ddl_forwarding.py @@ -72,7 +72,7 @@ class DdlForwardingContext: self.dbs: Dict[str, str] = {} self.roles: Dict[str, str] = {} self.fail = False - endpoint = "/management/api/v2/roles_and_databases" + endpoint = "/test/roles_and_databases" ddl_url = f"http://{host}:{port}{endpoint}" self.pg.configure( [ diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index 726e5e5def..d2d8d71e3f 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -1,11 +1,14 @@ import time +import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnv, logical_replication_sync, wait_for_last_flush_lsn, ) +from fixtures.types import Lsn +from fixtures.utils import query_scalar def test_logical_replication(neon_simple_env: NeonEnv, vanilla_pg): @@ -147,3 +150,89 @@ COMMIT; endpoint.start() # it must be gone (but walproposer slot still exists, hence 1) assert endpoint.safe_psql("select count(*) from pg_replication_slots")[0][0] == 1 + + +# Test compute start at LSN page of which starts with contrecord +# https://github.com/neondatabase/neon/issues/5749 +def test_wal_page_boundary_start(neon_simple_env: NeonEnv, vanilla_pg): + env = neon_simple_env + + env.neon_cli.create_branch("init") + endpoint = env.endpoints.create_start("init") + tenant_id = endpoint.safe_psql("show neon.tenant_id")[0][0] + timeline_id = endpoint.safe_psql("show neon.timeline_id")[0][0] + + cur = endpoint.connect().cursor() + cur.execute("create table t(key int, value text)") + cur.execute("CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int);") + cur.execute("insert into replication_example values (1, 2)") + cur.execute("create publication pub1 for table replication_example") + + # now start subscriber + vanilla_pg.start() + vanilla_pg.safe_psql("create table t(pk integer primary key, value text)") + vanilla_pg.safe_psql("CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int);") + + log.info(f"ep connstr is {endpoint.connstr()}, subscriber connstr {vanilla_pg.connstr()}") + connstr = endpoint.connstr().replace("'", "''") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + logical_replication_sync(vanilla_pg, endpoint) + vanilla_pg.stop() + + with endpoint.cursor() as cur: + # measure how much space logical message takes. Sometimes first attempt + # creates huge message and then it stabilizes, have no idea why. + for _ in range(3): + lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"current_lsn={lsn_before}") + # Non-transactional logical message doesn't write WAL, only XLogInsert's + # it, so use transactional. Which is a bit problematic as transactional + # necessitates commit record. Alternatively we can do smth like + # select neon_xlogflush(pg_current_wal_insert_lsn()); + # but isn't much better + that particular call complains on 'xlog flush + # request 0/282C018 is not satisfied' as pg_current_wal_insert_lsn skips + # page headers. + payload = "blahblah" + cur.execute(f"select pg_logical_emit_message(true, 'pref', '{payload}')") + lsn_after_by_curr_wal_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + lsn_diff = lsn_after_by_curr_wal_lsn - lsn_before + logical_message_base = lsn_after_by_curr_wal_lsn - lsn_before - len(payload) + log.info( + f"before {lsn_before}, after {lsn_after_by_curr_wal_lsn}, lsn diff is {lsn_diff}, base {logical_message_base}" + ) + + # and write logical message spanning exactly as we want + lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"current_lsn={lsn_before}") + curr_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + offs = int(curr_lsn) % 8192 + till_page = 8192 - offs + payload_len = ( + till_page - logical_message_base - 8 + ) # not sure why 8 is here, it is deduced from experiments + log.info(f"current_lsn={curr_lsn}, offs {offs}, till_page {till_page}") + + # payload_len above would go exactly till the page boundary; but we want contrecord, so make it slightly longer + payload_len += 8 + + cur.execute(f"select pg_logical_emit_message(true, 'pref', 'f{'a' * payload_len}')") + supposedly_contrecord_end = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) + log.info(f"supposedly_page_boundary={supposedly_contrecord_end}") + # The calculations to hit the page boundary are very fuzzy, so just + # ignore test if we fail to reach it. + if not (int(supposedly_contrecord_end) % 8192 == 32): + pytest.skip("missed page boundary, bad luck") + + cur.execute("insert into replication_example values (2, 3)") + + wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop().start() + + cur = endpoint.connect().cursor() + # this should flush current wal page + cur.execute("insert into replication_example values (3, 4)") + vanilla_pg.start() + logical_replication_sync(vanilla_pg, endpoint) + assert vanilla_pg.safe_psql( + "select sum(somedata) from replication_example" + ) == endpoint.safe_psql("select sum(somedata) from replication_example") diff --git a/test_runner/regress/test_lsn_mapping.py b/test_runner/regress/test_lsn_mapping.py index 726bfa5f29..f79c1c347c 100644 --- a/test_runner/regress/test_lsn_mapping.py +++ b/test_runner/regress/test_lsn_mapping.py @@ -79,13 +79,32 @@ def test_lsn_mapping_old(neon_env_builder: NeonEnvBuilder): def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() - new_timeline_id = env.neon_cli.create_branch("test_lsn_mapping") - endpoint_main = env.endpoints.create_start("test_lsn_mapping") - log.info("postgres is running on 'test_lsn_mapping' branch") + tenant_id, _ = env.neon_cli.create_tenant( + conf={ + # disable default GC and compaction + "gc_period": "1000 m", + "compaction_period": "0 s", + "gc_horizon": f"{1024 ** 2}", + "checkpoint_distance": f"{1024 ** 2}", + "compaction_target_size": f"{1024 ** 2}", + } + ) + + timeline_id = env.neon_cli.create_branch("test_lsn_mapping", tenant_id=tenant_id) + endpoint_main = env.endpoints.create_start("test_lsn_mapping", tenant_id=tenant_id) + timeline_id = endpoint_main.safe_psql("show neon.timeline_id")[0][0] + log.info("postgres is running on 'main' branch") cur = endpoint_main.connect().cursor() + + # Obtain an lsn before all write operations on this branch + start_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_lsn()")) + # Create table, and insert rows, each in a separate transaction # Disable synchronous_commit to make this initialization go faster. + # Disable `synchronous_commit` to make this initialization go faster. + # XXX: on my laptop this test takes 7s, and setting `synchronous_commit=off` + # doesn't change anything. # # Each row contains current insert LSN and the current timestamp, when # the row was inserted. @@ -104,40 +123,63 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): cur.execute("INSERT INTO foo VALUES (-1)") # Wait until WAL is received by pageserver - wait_for_last_flush_lsn(env, endpoint_main, env.initial_tenant, new_timeline_id) + last_flush_lsn = wait_for_last_flush_lsn(env, endpoint_main, tenant_id, timeline_id) with env.pageserver.http_client() as client: - # Check edge cases: timestamp in the future + # Check edge cases + # Timestamp is in the future probe_timestamp = tbl[-1][1] + timedelta(hours=1) result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) assert result["kind"] == "future" + # make sure that we return a well advanced lsn here + assert Lsn(result["lsn"]) > start_lsn - # timestamp too the far history + # Timestamp is in the unreachable past probe_timestamp = tbl[0][1] - timedelta(hours=10) result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) assert result["kind"] == "past" + # make sure that we return the minimum lsn here at the start of the range + assert Lsn(result["lsn"]) < start_lsn # Probe a bunch of timestamps in the valid range for i in range(1, len(tbl), 100): probe_timestamp = tbl[i][1] result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 + tenant_id, timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) + assert result["kind"] not in ["past", "nodata"] lsn = result["lsn"] # Call get_lsn_by_timestamp to get the LSN # Launch a new read-only node at that LSN, and check that only the rows # that were supposed to be committed at that point in time are visible. endpoint_here = env.endpoints.create_start( - branch_name="test_lsn_mapping", endpoint_id="ep-lsn_mapping_read", lsn=lsn + branch_name="test_lsn_mapping", + endpoint_id="ep-lsn_mapping_read", + lsn=lsn, + tenant_id=tenant_id, ) assert endpoint_here.safe_psql("SELECT max(x) FROM foo")[0][0] == i endpoint_here.stop_and_destroy() + # Do the "past" check again at a new branch to ensure that we don't return something before the branch cutoff + timeline_id_child = env.neon_cli.create_branch( + "test_lsn_mapping_child", tenant_id=tenant_id, ancestor_branch_name="test_lsn_mapping" + ) + + # Timestamp is in the unreachable past + probe_timestamp = tbl[0][1] - timedelta(hours=10) + result = client.timeline_get_lsn_by_timestamp( + tenant_id, timeline_id_child, f"{probe_timestamp.isoformat()}Z", 2 + ) + assert result["kind"] == "past" + # make sure that we return the minimum lsn here at the start of the range + assert Lsn(result["lsn"]) >= last_flush_lsn + # Test pageserver get_timestamp_of_lsn API def test_ts_of_lsn_api(neon_env_builder: NeonEnvBuilder): diff --git a/test_runner/regress/test_neon_cli.py b/test_runner/regress/test_neon_cli.py index 1b3984583a..de18ea0e6b 100644 --- a/test_runner/regress/test_neon_cli.py +++ b/test_runner/regress/test_neon_cli.py @@ -134,6 +134,9 @@ def test_cli_start_stop(neon_env_builder: NeonEnvBuilder): env.neon_cli.pageserver_stop(env.pageserver.id) env.neon_cli.safekeeper_stop() + # Keep NeonEnv state up to date, it usually owns starting/stopping services + env.pageserver.running = False + # Default start res = env.neon_cli.raw_cli(["start"]) res.check_returncode() @@ -155,6 +158,10 @@ def test_cli_start_stop_multi(neon_env_builder: NeonEnvBuilder): env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID) env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 1) + # Keep NeonEnv state up to date, it usually owns starting/stopping services + env.pageservers[0].running = False + env.pageservers[1].running = False + # Addressing a nonexistent ID throws with pytest.raises(RuntimeError): env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 100) diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 78ef6b9165..c3f4ad476f 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -21,6 +21,7 @@ from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, PgBin, + S3Scrubber, last_flush_lsn_upload, wait_for_last_flush_lsn, ) @@ -234,8 +235,22 @@ def test_generations_upgrade(neon_env_builder: NeonEnvBuilder): assert len(suffixed_objects) > 0 assert len(legacy_objects) > 0 + # Flush through deletions to get a clean state for scrub: we are implicitly validating + # that our generations-enabled pageserver was able to do deletions of layers + # from earlier which don't have a generation. + env.pageserver.http_client().deletion_queue_flush(execute=True) + assert get_deletion_queue_unexpected_errors(env.pageserver.http_client()) == 0 + # Having written a mixture of generation-aware and legacy index_part.json, + # ensure the scrubber handles the situation as expected. + metadata_summary = S3Scrubber( + neon_env_builder.test_output_dir, neon_env_builder + ).scan_metadata() + assert metadata_summary["count"] == 1 # Scrubber should have seen our timeline + assert not metadata_summary["with_errors"] + assert not metadata_summary["with_warnings"] + def test_deferred_deletion(neon_env_builder: NeonEnvBuilder): neon_env_builder.enable_generations = True @@ -351,11 +366,17 @@ def test_deletion_queue_recovery( assert get_deletion_queue_dropped_lsn_updates(ps_http) == 0 if validate_before == ValidateBefore.VALIDATE: + # At this point, one or more DeletionLists have been written. We have set a failpoint + # to prevent them successfully executing, but we want to see them get validated. + # + # We await _some_ validations instead of _all_ validations, because our execution failpoint + # will prevent validation proceeding for any but the first DeletionList. Usually the workload + # just generates one, but if it generates two due to timing, then we must not expect that the + # second one will be validated. + def assert_some_validations(): + assert get_deletion_queue_validated(ps_http) > 0 - def assert_validation_complete(): - assert get_deletion_queue_submitted(ps_http) == get_deletion_queue_validated(ps_http) - - wait_until(20, 1, assert_validation_complete) + wait_until(20, 1, assert_some_validations) # The validatated keys statistic advances before the header is written, so we # also wait to see the header hit the disk: this seems paranoid but the race @@ -365,6 +386,11 @@ def test_deletion_queue_recovery( wait_until(20, 1, assert_header_written) + # If we will lose attachment, then our expectation on restart is that only the ones + # we already validated will execute. Act like only those were present in the queue. + if keep_attachment == KeepAttachment.LOSE: + before_restart_depth = get_deletion_queue_validated(ps_http) + log.info(f"Restarting pageserver with {before_restart_depth} deletions enqueued") env.pageserver.stop(immediate=True) @@ -387,11 +413,13 @@ def test_deletion_queue_recovery( ps_http.deletion_queue_flush(execute=True) wait_until(10, 1, lambda: assert_deletion_queue(ps_http, lambda n: n == 0)) - if keep_attachment == KeepAttachment.KEEP or validate_before == ValidateBefore.VALIDATE: + if keep_attachment == KeepAttachment.KEEP: # - If we kept the attachment, then our pre-restart deletions should execute # because on re-attach they were from the immediately preceding generation - # - If we validated before restart, then the deletions should execute because the - # deletion queue header records a validated deletion list sequence number. + assert get_deletion_queue_executed(ps_http) == before_restart_depth + elif validate_before == ValidateBefore.VALIDATE: + # - If we validated before restart, then we should execute however many keys were + # validated before restart. assert get_deletion_queue_executed(ps_http) == before_restart_depth else: env.pageserver.allowed_errors.extend([".*Dropping stale deletions.*"]) diff --git a/test_runner/regress/test_pageserver_restart.py b/test_runner/regress/test_pageserver_restart.py index 4c51155236..fa1b131537 100644 --- a/test_runner/regress/test_pageserver_restart.py +++ b/test_runner/regress/test_pageserver_restart.py @@ -20,6 +20,8 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool) endpoint = env.endpoints.create_start("main") pageserver_http = env.pageserver.http_client() + assert pageserver_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + pg_conn = endpoint.connect() cur = pg_conn.cursor() @@ -52,6 +54,9 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool) env.pageserver.stop() env.pageserver.start() + # We reloaded our tenant + assert pageserver_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + cur.execute("SELECT count(*) FROM foo") assert cur.fetchone() == (100000,) diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index c93cdf637a..0f2cd9768f 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -1,3 +1,4 @@ +import asyncio import json import subprocess import time @@ -11,6 +12,29 @@ from fixtures.neon_fixtures import PSQL, NeonProxy, VanillaPostgres GET_CONNECTION_PID_QUERY = "SELECT pid FROM pg_stat_activity WHERE state = 'active'" +@pytest.mark.asyncio +async def test_http_pool_begin_1(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth with password 'http' superuser") + + def query(*args) -> Any: + static_proxy.http_query( + "SELECT pg_sleep(10);", + args, + user="http_auth", + password="http", + expected_code=200, + ) + + query() + loop = asyncio.get_running_loop() + tasks = [loop.run_in_executor(None, query) for _ in range(10)] + # Wait for all the tasks to complete + completed, pending = await asyncio.wait(tasks) + # Get the results + results = [task.result() for task in completed] + print(results) + + def test_proxy_select_1(static_proxy: NeonProxy): """ A simplest smoke test: check proxy against a local postgres instance. diff --git a/test_runner/regress/test_proxy_rate_limiter.py b/test_runner/regress/test_proxy_rate_limiter.py new file mode 100644 index 0000000000..f39f0cad07 --- /dev/null +++ b/test_runner/regress/test_proxy_rate_limiter.py @@ -0,0 +1,84 @@ +import asyncio +import time +from pathlib import Path +from typing import Iterator + +import pytest +from fixtures.neon_fixtures import ( + PSQL, + NeonProxy, +) +from fixtures.port_distributor import PortDistributor +from pytest_httpserver import HTTPServer +from werkzeug.wrappers.response import Response + + +def waiting_handler(status_code: int) -> Response: + # wait more than timeout to make sure that both (two) connections are open. + # It would be better to use a barrier here, but I don't know how to do that together with pytest-httpserver. + time.sleep(2) + return Response(status=status_code) + + +@pytest.fixture(scope="function") +def proxy_with_rate_limit( + port_distributor: PortDistributor, + neon_binpath: Path, + httpserver_listen_address, + test_output_dir: Path, +) -> Iterator[NeonProxy]: + """Neon proxy that routes directly to vanilla postgres.""" + + proxy_port = port_distributor.get_port() + mgmt_port = port_distributor.get_port() + http_port = port_distributor.get_port() + external_http_port = port_distributor.get_port() + (host, port) = httpserver_listen_address + endpoint = f"http://{host}:{port}/billing/api/v1/usage_events" + + with NeonProxy( + neon_binpath=neon_binpath, + test_output_dir=test_output_dir, + proxy_port=proxy_port, + http_port=http_port, + mgmt_port=mgmt_port, + external_http_port=external_http_port, + auth_backend=NeonProxy.Console(endpoint, fixed_rate_limit=5), + ) as proxy: + proxy.start() + yield proxy + + +@pytest.mark.asyncio +async def test_proxy_rate_limit( + httpserver: HTTPServer, + proxy_with_rate_limit: NeonProxy, +): + uri = "/billing/api/v1/usage_events/proxy_get_role_secret" + # mock control plane service + httpserver.expect_ordered_request(uri, method="GET").respond_with_handler( + lambda _: Response(status=200) + ) + httpserver.expect_ordered_request(uri, method="GET").respond_with_handler( + lambda _: waiting_handler(429) + ) + httpserver.expect_ordered_request(uri, method="GET").respond_with_handler( + lambda _: waiting_handler(500) + ) + + psql = PSQL(host=proxy_with_rate_limit.host, port=proxy_with_rate_limit.proxy_port) + f = await psql.run("select 42;") + await proxy_with_rate_limit.find_auth_link(uri, f) + # Limit should be 2. + + # Run two queries in parallel. + f1, f2 = await asyncio.gather(psql.run("select 42;"), psql.run("select 42;")) + await proxy_with_rate_limit.find_auth_link(uri, f1) + await proxy_with_rate_limit.find_auth_link(uri, f2) + + # Now limit should be 0. + f = await psql.run("select 42;") + await proxy_with_rate_limit.find_auth_link(uri, f) + + # There last query shouldn't reach the http-server. + assert httpserver.assertions == [] diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index ae77197088..6e35890922 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -63,6 +63,9 @@ def test_tenant_delete_smoke( conf=MANY_SMALL_LAYERS_TENANT_CONFIG, ) + # Default tenant and the one we created + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 2 + # create two timelines one being the parent of another parent = None for timeline in ["first", "second"]: @@ -88,7 +91,9 @@ def test_tenant_delete_smoke( iterations = poll_for_remote_storage_iterations(remote_storage_kind) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 2 tenant_delete_wait_completed(ps_http, tenant_id, iterations) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 1 tenant_path = env.pageserver.tenant_dir(tenant_id) assert not tenant_path.exists() @@ -104,6 +109,9 @@ def test_tenant_delete_smoke( ), ) + # Deletion updates the tenant count: the one default tenant remains + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + class Check(enum.Enum): RETRY_WITHOUT_RESTART = enum.auto() diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 11e8a80e1d..03e78dda2b 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -26,6 +26,16 @@ from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until from prometheus_client.samples import Sample +# In tests that overlap endpoint activity with tenant attach/detach, there are +# a variety of warnings that the page service may emit when it cannot acquire +# an active tenant to serve a request +PERMIT_PAGE_SERVICE_ERRORS = [ + ".*page_service.*Tenant .* not found", + ".*page_service.*Tenant .* is not active", + ".*page_service.*cancelled", + ".*page_service.*will not become active.*", +] + def do_gc_target( pageserver_http: PageserverHttpClient, tenant_id: TenantId, timeline_id: TimelineId @@ -60,12 +70,7 @@ def test_tenant_reattach( # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: with endpoint.cursor() as cur: @@ -235,10 +240,7 @@ def test_tenant_reattach_while_busy( # Attempts to connect from compute to pageserver while the tenant is # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) endpoint = env.endpoints.create_start("main", tenant_id=tenant_id) @@ -259,7 +261,7 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() - env.pageserver.allowed_errors.append(".*NotFound: Tenant .*") + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # first check for non existing tenant tenant_id = TenantId.generate() @@ -271,19 +273,9 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): assert excinfo.value.status_code == 404 - # the error will be printed to the log too - env.pageserver.allowed_errors.append(".*NotFound: tenant *") - # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) - # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -345,12 +337,7 @@ def test_tenant_detach_ignored_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -401,12 +388,7 @@ def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # assert tenant exists on disk assert env.pageserver.tenant_dir(tenant_id).exists() @@ -453,12 +435,7 @@ def test_detach_while_attaching( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) # Create table, and insert some rows. Make it big enough that it doesn't fit in # shared_buffers, otherwise the SELECT after restart will just return answer @@ -593,12 +570,7 @@ def test_ignored_tenant_download_missing_layers(neon_env_builder: NeonEnvBuilder tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) data_id = 1 data_secret = "very secret secret" @@ -649,12 +621,7 @@ def test_load_attach_negatives(neon_env_builder: NeonEnvBuilder): tenant_id = env.initial_tenant - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) env.pageserver.allowed_errors.append(".*tenant .*? already exists, state:.*") with pytest.raises( @@ -693,12 +660,7 @@ def test_ignore_while_attaching( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" - ) + env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) data_id = 1 data_secret = "very secret secret" diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 6669a672ee..dd067cf656 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 6669a672ee14ab2c09d44c4552f9a13fad3afc10 +Subproject commit dd067cf656f6810a25aca6025633d32d02c5085a diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index ab67ab9635..bc88f53931 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit ab67ab96355d61e9d0218630be4aa7db53bf83e7 +Subproject commit bc88f539312fcc4bb292ce94ae9db09ab6656e8a diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 550ffa6495..763000f1d0 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 550ffa6495a5dc62fccc3a8b449386633758680b +Subproject commit 763000f1d0873b827829c41f2f6f799ffc0de55c diff --git a/vendor/revisions.json b/vendor/revisions.json index 012fb14035..377357e131 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "postgres-v16": "550ffa6495a5dc62fccc3a8b449386633758680b", - "postgres-v15": "ab67ab96355d61e9d0218630be4aa7db53bf83e7", - "postgres-v14": "6669a672ee14ab2c09d44c4552f9a13fad3afc10" + "postgres-v16": "763000f1d0873b827829c41f2f6f799ffc0de55c", + "postgres-v15": "bc88f539312fcc4bb292ce94ae9db09ab6656e8a", + "postgres-v14": "dd067cf656f6810a25aca6025633d32d02c5085a" } diff --git a/vm-image-spec.yaml b/vm-image-spec.yaml new file mode 100644 index 0000000000..2aa935fac6 --- /dev/null +++ b/vm-image-spec.yaml @@ -0,0 +1,126 @@ +# Supplemental file for neondatabase/autoscaling's vm-builder, for producing the VM compute image. +--- +commands: + - name: cgconfigparser + user: root + sysvInitAction: sysinit + shell: 'cgconfigparser -l /etc/cgconfig.conf -s 1664' + - name: pgbouncer + user: nobody + sysvInitAction: respawn + shell: '/usr/local/bin/pgbouncer /etc/pgbouncer.ini' + - name: postgres-exporter + user: nobody + sysvInitAction: respawn + shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres" /bin/postgres_exporter' +shutdownHook: | + su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10' +files: + - filename: pgbouncer.ini + content: | + [databases] + *=host=localhost port=5432 auth_user=cloud_admin + [pgbouncer] + listen_port=6432 + listen_addr=0.0.0.0 + auth_type=scram-sha-256 + auth_user=cloud_admin + auth_dbname=postgres + client_tls_sslmode=disable + server_tls_sslmode=disable + pool_mode=transaction + max_client_conn=10000 + default_pool_size=16 + max_prepared_statements=0 + - filename: cgconfig.conf + content: | + # Configuration for cgroups in VM compute nodes + group neon-postgres { + perm { + admin { + uid = postgres; + } + task { + gid = users; + } + } + memory {} + } +build: | + # Build cgroup-tools + # + # At time of writing (2023-03-14), debian bullseye has a version of cgroup-tools (technically + # libcgroup) that doesn't support cgroup v2 (version 0.41-11). Unfortunately, the vm-monitor + # requires cgroup v2, so we'll build cgroup-tools ourselves. + FROM debian:bullseye-slim as libcgroup-builder + ENV LIBCGROUP_VERSION v2.0.3 + + RUN set -exu \ + && apt update \ + && apt install --no-install-recommends -y \ + git \ + ca-certificates \ + automake \ + cmake \ + make \ + gcc \ + byacc \ + flex \ + libtool \ + libpam0g-dev \ + && git clone --depth 1 -b $LIBCGROUP_VERSION https://github.com/libcgroup/libcgroup \ + && INSTALL_DIR="/libcgroup-install" \ + && mkdir -p "$INSTALL_DIR/bin" "$INSTALL_DIR/include" \ + && cd libcgroup \ + # extracted from bootstrap.sh, with modified flags: + && (test -d m4 || mkdir m4) \ + && autoreconf -fi \ + && rm -rf autom4te.cache \ + && CFLAGS="-O3" ./configure --prefix="$INSTALL_DIR" --sysconfdir=/etc --localstatedir=/var --enable-opaque-hierarchy="name=systemd" \ + # actually build the thing... + && make install + + FROM quay.io/prometheuscommunity/postgres-exporter:v0.12.0 AS postgres-exporter + + # Build pgbouncer + # + FROM debian:bullseye-slim AS pgbouncer + RUN set -e \ + && apt-get update \ + && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libevent-dev \ + libssl-dev + + ENV PGBOUNCER_VERSION 1.21.0 + ENV PGBOUNCER_GITPATH 1_21_0 + RUN set -e \ + && curl -sfSL https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_${PGBOUNCER_GITPATH}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz -o pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ + && tar xzvf pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ + && cd pgbouncer-${PGBOUNCER_VERSION} \ + && LDFLAGS=-static ./configure --prefix=/usr/local/pgbouncer --without-openssl \ + && make -j $(nproc) \ + && make install +merge: | + # tweak nofile limits + RUN set -e \ + && echo 'fs.file-max = 1048576' >>/etc/sysctl.conf \ + && test ! -e /etc/security || ( \ + echo '* - nofile 1048576' >>/etc/security/limits.conf \ + && echo 'root - nofile 1048576' >>/etc/security/limits.conf \ + ) + + COPY cgconfig.conf /etc/cgconfig.conf + COPY pgbouncer.ini /etc/pgbouncer.ini + RUN set -e \ + && chown postgres:postgres /etc/pgbouncer.ini \ + && chmod 0644 /etc/pgbouncer.ini \ + && chmod 0644 /etc/cgconfig.conf + + COPY --from=libcgroup-builder /libcgroup-install/bin/* /usr/bin/ + COPY --from=libcgroup-builder /libcgroup-install/lib/* /usr/lib/ + COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/ + COPY --from=postgres-exporter /bin/postgres_exporter /bin/postgres_exporter + COPY --from=pgbouncer /usr/local/pgbouncer/bin/pgbouncer /usr/local/bin/pgbouncer diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index e2a65ad150..3535c2cf94 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -25,6 +25,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd clap = { version = "4", features = ["derive", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8" } +dashmap = { version = "5", default-features = false, features = ["raw-api"] } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures = { version = "0.3" } @@ -38,7 +39,7 @@ hex = { version = "0.4", features = ["serde"] } hyper = { version = "0.14", features = ["full"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] } +log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } nom = { version = "7" } num-bigint = { version = "0.4" } @@ -55,7 +56,6 @@ scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } smallvec = { version = "1", default-features = false, features = ["write"] } -standback = { version = "0.2", default-features = false, features = ["std"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-rustls = { version = "0.24" } @@ -76,14 +76,13 @@ cc = { version = "1", default-features = false, features = ["parallel"] } either = { version = "1" } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } -log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] } +log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } nom = { version = "7" } prost = { version = "0.11" } regex = { version = "1" } regex-syntax = { version = "0.7" } serde = { version = "1", features = ["alloc", "derive"] } -standback = { version = "0.2", default-features = false, features = ["std"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "full", "visit", "visit-mut"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] }