mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
26b47b5beb
## Problem Neon's storage system currently has hard-coded 512-byte block size for Direct IO operations, which causes I/O errors on systems with disks that have 4096-byte block sizes. This results in errors like "vec read failed" and "Invalid argument (os error 22)" on certain hardware configurations. See issue #12623 for details. ## Summary of changes Make Direct IO alignment configurable at build time to support both 512-byte and 4096-byte block sizes: - Add `io-align-512` and `io-align-4k` cargo features (default: 512-byte for backward compatibility) - Make `DEFAULT_IO_BUFFER_ALIGNMENT` configurable via cargo features in `pageserver_api` - Update `DIO_CHUNK_SIZE` in vectored_dio_read to use the configured alignment value dynamically - Add `IO_ALIGNMENT` build argument to Dockerfile to allow building images with different alignment settings - Add startup logging to display the configured IO buffer alignment for operational visibility - Fix validation logic in `virtual_file.rs` to use the configured alignment instead of hard-coded 512 This change allows Neon to run on systems with different disk block sizes by building with the appropriate feature flag, addressing the compatibility issues described in the RFC on Direct IO implementation ## Performance Note Benchmarks show 512-byte alignment performs significantly better than 4k: - Write: 512-byte is 21-71% faster across percentiles (p99: 71% faster) - Read: 512-byte is slightly faster (5-21% improvement) This is why 512-byte remains the default. However, some storage systems require 4k alignment and will fail with EINVAL otherwise. This change adds build-time configuration to support both environments.
151 lines
4.2 KiB
TOML
151 lines
4.2 KiB
TOML
[package]
|
|
name = "pageserver"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints", "pageserver_api/testing", "wal_decoder/testing", "pageserver_client/testing"]
|
|
|
|
# Direct IO alignment options (propagated to pageserver_api)
|
|
io-align-512 = ["pageserver_api/io-align-512"]
|
|
io-align-4k = ["pageserver_api/io-align-4k"]
|
|
|
|
fuzz-read-path = ["testing"]
|
|
|
|
# Enables benchmarking only APIs
|
|
benchmarking = []
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
arc-swap.workspace = true
|
|
async-compression.workspace = true
|
|
async-stream.workspace = true
|
|
bincode.workspace = true
|
|
bit_field.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino-tempfile.workspace = true
|
|
camino.workspace = true
|
|
chrono = { workspace = true, features = ["serde"] }
|
|
clap = { workspace = true, features = ["string"] }
|
|
consumption_metrics.workspace = true
|
|
crc32c.workspace = true
|
|
either.workspace = true
|
|
enum-map.workspace = true
|
|
enumset = { workspace = true, features = ["serde"]}
|
|
fail.workspace = true
|
|
futures.workspace = true
|
|
hashlink.workspace = true
|
|
hex.workspace = true
|
|
http.workspace = true
|
|
http-utils.workspace = true
|
|
humantime-serde.workspace = true
|
|
humantime.workspace = true
|
|
hyper0.workspace = true
|
|
itertools.workspace = true
|
|
jsonwebtoken.workspace = true
|
|
md5.workspace = true
|
|
metrics.workspace = true
|
|
nix.workspace = true
|
|
num_cpus.workspace = true # hack to get the number of worker threads tokio uses
|
|
num-traits.workspace = true
|
|
once_cell.workspace = true
|
|
pageserver_api.workspace = true
|
|
pageserver_client.workspace = true # for ResponseErrorMessageExt TOOD refactor that
|
|
pageserver_compaction.workspace = true
|
|
pageserver_page_api.workspace = true
|
|
pem.workspace = true
|
|
pin-project-lite.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres_connection.workspace = true
|
|
postgres_ffi.workspace = true
|
|
postgres_ffi_types.workspace = true
|
|
postgres_initdb.workspace = true
|
|
postgres-protocol.workspace = true
|
|
postgres-types.workspace = true
|
|
posthog_client_lite.workspace = true
|
|
pprof.workspace = true
|
|
pq_proto.workspace = true
|
|
rand.workspace = true
|
|
range-set-blaze = { version = "0.1.16", features = ["alloc"] }
|
|
regex.workspace = true
|
|
remote_storage.workspace = true
|
|
reqwest.workspace = true
|
|
rpds.workspace = true
|
|
rustls.workspace = true
|
|
scopeguard.workspace = true
|
|
send-future.workspace = true
|
|
serde_json = { workspace = true, features = ["raw_value"] }
|
|
serde_path_to_error.workspace = true
|
|
serde_with.workspace = true
|
|
serde.workspace = true
|
|
smallvec.workspace = true
|
|
storage_broker.workspace = true
|
|
strum_macros.workspace = true
|
|
strum.workspace = true
|
|
sysinfo.workspace = true
|
|
tenant_size_model.workspace = true
|
|
thiserror.workspace = true
|
|
tikv-jemallocator.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
|
|
tokio-epoll-uring.workspace = true
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-rustls.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-tar.workspace = true
|
|
tokio-util.workspace = true
|
|
toml_edit = { workspace = true, features = [ "serde" ] }
|
|
tonic.workspace = true
|
|
tonic-reflection.workspace = true
|
|
tower.workspace = true
|
|
tracing.workspace = true
|
|
tracing-utils.workspace = true
|
|
url.workspace = true
|
|
utils.workspace = true
|
|
wal_decoder.workspace = true
|
|
walkdir.workspace = true
|
|
workspace_hack.workspace = true
|
|
twox-hash.workspace = true
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
procfs.workspace = true
|
|
|
|
[dev-dependencies]
|
|
base64.workspace = true
|
|
criterion.workspace = true
|
|
hex-literal.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
|
|
indoc.workspace = true
|
|
uuid.workspace = true
|
|
rstest.workspace = true
|
|
|
|
[[bench]]
|
|
name = "bench_layer_map"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_walredo"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_ingest"
|
|
harness = false
|
|
required-features = ["benchmarking"]
|
|
|
|
[[bench]]
|
|
name = "upload_queue"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_metrics"
|
|
harness = false
|
|
|
|
[[bin]]
|
|
name = "test_helper_slow_client_reads"
|
|
required-features = [ "testing" ]
|