rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-25 00:50:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

R-Tree layer map (#2317)

Browse Source 
Replace the layer array and linear search with R-tree

So far, the in-memory layer map that holds information about layer
files that exist, has used a simple Vec, in no particular order, to
hold information about all the layers. That obviously doesn't scale
very well; with thousands of layer files the linear search was
consuming a lot of CPU. Replace it with a two-dimensional R-tree, with
Key and LSN ranges as the dimensions.

For the R-tree, use the 'rstar' crate. To be able to use that, we
convert the Keys and LSNs into 256-bit integers. 64 bits would be
enough to represent LSNs, and 128 bits would be enough to represent
Keys. However, we use 256 bits, because rstar internally performs
multiplication to calculate the area of rectangles, and the result of
multiplying two 128 bit integers doesn't necessarily fit in 128 bits,
causing integer overflow and, if overflow-checks are enabled,
panic. To avoid that, we use 256 bit integers.

Add a performance test that creates a lot of layer files, to
demonstrate the benefit.
This commit is contained in:
Konstantin Knizhnik
2022-09-22 08:35:06 +03:00
committed by GitHub
parent e9a103c09f
commit f3073a4db9
8 changed files with 548 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
Cargo.lock generated
View File

@@ -37,6 +37,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "amplify_num"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f27d3d00d3d115395a7a8a4dc045feb7aa82b641e485f7e15f4e67ac16f4f56d"
[[package]]
name = "ansi_term"
version = "0.12.1"
@@ -135,6 +141,15 @@ dependencies = [
"syn",
]
[[package]]
name = "atomic-polyfill"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c041a8d9751a520ee19656232a18971f18946a7900f1520ee4400002244dd89"
dependencies = [
"critical-section",
]
[[package]]
name = "atty"
version = "0.2.14"
@@ -212,6 +227,21 @@ dependencies = [
"rustc-demangle",
]
[[package]]
name = "bare-metal"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5deb64efa5bd81e31fcd1938615a6d98c82eafcbcd787162b6f63b91d6bac5b3"
dependencies = [
"rustc_version 0.2.3",
]
[[package]]
name = "bare-metal"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603"
[[package]]
name = "base64"
version = "0.13.0"
@@ -250,6 +280,18 @@ dependencies = [
"which",
]
[[package]]
name = "bit_field"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
[[package]]
name = "bitfield"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719"
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -528,6 +570,18 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "cortex-m"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70858629a458fdfd39f9675c4dc309411f2a3f83bede76988d81bf1a0ecee9e0"
dependencies = [
"bare-metal 0.2.5",
"bitfield",
"embedded-hal",
"volatile-register",
]
[[package]]
name = "cpp_demangle"
version = "0.3.5"
@@ -552,7 +606,7 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e"
dependencies = [
"rustc_version",
"rustc_version 0.4.0",
]
[[package]]
@@ -600,6 +654,18 @@ dependencies = [
"itertools",
]
[[package]]
name = "critical-section"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95da181745b56d4bd339530ec393508910c909c784e8962d15d722bacf0bcbcd"
dependencies = [
"bare-metal 1.0.0",
"cfg-if",
"cortex-m",
"riscv",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
@@ -844,6 +910,16 @@ version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be"
[[package]]
name = "embedded-hal"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff"
dependencies = [
"nb 0.1.3",
"void",
]
[[package]]
name = "encoding_rs"
version = "0.8.31"
@@ -1165,6 +1241,15 @@ version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hash32"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"
dependencies = [
"byteorder",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -1174,6 +1259,19 @@ dependencies = [
"ahash",
]
[[package]]
name = "heapless"
version = "0.7.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743"
dependencies = [
"atomic-polyfill",
"hash32",
"rustc_version 0.4.0",
"spin 0.9.4",
"stable_deref_trait",
]
[[package]]
name = "heck"
version = "0.3.3"
@@ -1491,6 +1589,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "libm"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565"
[[package]]
name = "lock_api"
version = "0.4.7"
@@ -1649,6 +1753,21 @@ dependencies = [
"tempfile",
]
[[package]]
name = "nb"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f"
dependencies = [
"nb 1.0.0",
]
[[package]]
name = "nb"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae"
[[package]]
name = "nix"
version = "0.23.1"
@@ -1716,6 +1835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
"libm",
]
[[package]]
@@ -1828,6 +1948,7 @@ checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4"
name = "pageserver"
version = "0.1.0"
dependencies = [
"amplify_num",
"anyhow",
"async-stream",
"async-trait",
@@ -1852,6 +1973,7 @@ dependencies = [
"itertools",
"metrics",
"nix",
"num-traits",
"once_cell",
"postgres",
"postgres-protocol",
@@ -1861,6 +1983,7 @@ dependencies = [
"rand",
"regex",
"remote_storage",
"rstar",
"scopeguard",
"serde",
"serde_json",
@@ -2515,12 +2638,33 @@ dependencies = [
"cc",
"libc",
"once_cell",
"spin",
"spin 0.5.2",
"untrusted",
"web-sys",
"winapi",
]
[[package]]
name = "riscv"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6907ccdd7a31012b70faf2af85cd9e5ba97657cc3987c4f13f8e4d2c2a088aba"
dependencies = [
"bare-metal 1.0.0",
"bit_field",
"riscv-target",
]
[[package]]
name = "riscv-target"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88aa938cda42a0cf62a20cfe8d139ff1af20c2e681212b5b34adb5a58333f222"
dependencies = [
"lazy_static",
"regex",
]
[[package]]
name = "routerify"
version = "3.0.0"
@@ -2534,6 +2678,17 @@ dependencies = [
"regex",
]
[[package]]
name = "rstar"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b40f1bfe5acdab44bc63e6699c28b74f75ec43afb59f3eda01e145aff86a25fa"
dependencies = [
"heapless",
"num-traits",
"smallvec",
]
[[package]]
name = "rstest"
version = "0.12.0"
@@ -2543,7 +2698,7 @@ dependencies = [
"cfg-if",
"proc-macro2",
"quote",
"rustc_version",
"rustc_version 0.4.0",
"syn",
]
@@ -2565,7 +2720,7 @@ dependencies = [
"log",
"rusoto_credential",
"rusoto_signature",
"rustc_version",
"rustc_version 0.4.0",
"serde",
"serde_json",
"tokio",
@@ -2623,7 +2778,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"rusoto_credential",
"rustc_version",
"rustc_version 0.4.0",
"serde",
"sha2 0.9.9",
"tokio",
@@ -2641,13 +2796,22 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver 0.9.0",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver",
"semver 1.0.13",
]
[[package]]
@@ -2800,12 +2964,27 @@ dependencies = [
"libc",
]
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser",
]
[[package]]
name = "semver"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711"
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.142"
@@ -2999,6 +3178,15 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spin"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09"
dependencies = [
"lock_api",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@@ -3675,6 +3863,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "vcell"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002"
[[package]]
name = "vcpkg"
version = "0.2.15"
@@ -3687,6 +3881,21 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]]
name = "volatile-register"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ee8f19f9d74293faf70901bc20ad067dc1ad390d2cbf1e3f75f721ffee908b6"
dependencies = [
"vcell",
]
[[package]]
name = "wal_craft"
version = "0.1.0"
@@ -3952,6 +4161,7 @@ dependencies = [
"regex-syntax",
"scopeguard",
"serde",
"stable_deref_trait",
"syn",
"time 0.3.12",
"tokio",