mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-21 20:45:57 +00:00
Bruno RamirezandClaude Sonnet 5
3fca33fcb5
Short-lived Python processes using this client can occasionally crash with SIGABRT during interpreter shutdown, even after every operation they ran completed successfully. The cause is the shared Tokio runtime backing every async call: it's never told to shut down at normal process exit, only reset (and deliberately leaked) on `fork()`. Its worker threads keep running, uncoordinated with the interpreter, until the process actually ends, and if one is mid-task exactly as `Py_Finalize` starts tearing down interpreter state, it can panic on state that's already gone. That panic happens on a background thread with no PyO3-wrapped call frame to catch it, so Rust aborts the whole process instead of just failing that one call. This PR gives the runtime a coordinated, bounded shutdown by registering a Python `atexit` callback that runs while the interpreter is still fully valid. Getting the exit lifecycle right took a few rounds of review. Earlier versions freed the runtime as soon as `Arc::strong_count` looked low, but that's the wrong signal — it reflects who currently holds a reference, not who's logically still in flight. That mistake showed up three ways: a caller could dereference memory already freed out from under it; an install already in progress could finish invisibly after `shutdown()` had already decided there was nothing to do; and a spawned task could end up as the final owner of the `Runtime`, so completing it dropped the runtime from inside one of its own worker threads, which Tokio itself forbids and panics on (this reproduced unprompted in this branch's own test suite). Fixing all three meant replacing reference-count-based tracking with an explicit counter of in-flight top-level calls that `shutdown()` waits on directly. This was accomplished with the following changes: - The runtime lives in an `ArcSwapOption<Tagged>`, where `Tagged` pairs the `Runtime` with the fork generation it was built in. - An `OUTSTANDING` counter, incremented before a top-level `spawn`/`spawn_blocking`/`block_on` call does anything else and decremented only once it has truly finished (via an `OutstandingGuard` token that carries no reference to the runtime), is what `shutdown()` waits on — not `Arc::strong_count` or whether the slot looks empty. This closes the install-race and makes it impossible for a task's own completion to be the runtime's final drop. - Once `shutdown()`'s bound elapses, it stops waiting and attempts retirement anyway, rather than returning with the runtime and its workers left fully alive. - `spawn`/`spawn_blocking` use `Handle::try_current()` to pin any nested spawn (`future_into_py` spawns a task that itself spawns a second one for the real work) to whichever runtime is already executing it, so a reclaim landing between the two calls can't split one logical operation across two different runtime instances. - The fork-child handler now only bumps a bare `GENERATION` counter — no `ArcSwapOption` call of any kind from that context, since `swap`/`compare_and_swap` do real reader-reconciliation work (thread-local state, potentially an allocation) that isn't safe in a forked child. `get_runtime()` compares its installed runtime's generation against the live counter from ordinary context and rebuilds on a mismatch. - Registered `shutdown_runtime` as a Python `atexit` callback in the `_lancedb` module init, running with the GIL released (`Python::detach`) since the bounded wait could otherwise deadlock against any in-flight task that itself needs the GIL. ### Testing - Unit tests in `runtime.rs` cover: shutdown with no runtime created, shutdown after use and lazy rebuild afterward, calling shutdown twice in a row, a concurrent stress test racing many threads against shutdown, a nested-spawn test reproducing `future_into_py`'s own spawn-within-a-spawn shape under concurrent shutdown, a test confirming a top-level task in flight survives a concurrent shutdown reclaim, and a test forcing the install-vs-shutdown race directly. - Built the wheel and ran a concurrent reproducer (many threads hammering the client while `atexit` fires) over 100 times with no hangs or crashes, plus a 30-second-join variant and repeated runs of a short-lived process confirming clean exits with no added latency. --------- Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
54 lines
1.5 KiB
TOML
54 lines
1.5 KiB
TOML
[package]
|
|
name = "lancedb-python"
|
|
version = "0.39.0-beta.8"
|
|
publish = false
|
|
edition.workspace = true
|
|
description = "Python bindings for LanceDB"
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
keywords.workspace = true
|
|
categories.workspace = true
|
|
rust-version = "1.91.0"
|
|
|
|
[lib]
|
|
name = "_lancedb"
|
|
crate-type = ["cdylib"]
|
|
|
|
[dependencies]
|
|
arc-swap = "1.9"
|
|
arrow = { workspace = true, features = ["pyarrow"] }
|
|
async-trait.workspace = true
|
|
bytes.workspace = true
|
|
lancedb.workspace = true
|
|
datafusion-common.workspace = true
|
|
lance-core.workspace = true
|
|
lance-namespace.workspace = true
|
|
lance-namespace-impls.workspace = true
|
|
lance-io.workspace = true
|
|
env_logger.workspace = true
|
|
log.workspace = true
|
|
# Maturin enables extension-module mode for Python builds. Keeping it out of
|
|
# Cargo features lets Rust unit tests link against libpython.
|
|
pyo3 = { version = "0.28", features = ["abi3-py310", "chrono", "uuid"] }
|
|
chrono.workspace = true
|
|
pyo3-async-runtimes = { version = "0.28", features = [
|
|
"attributes",
|
|
"tokio-runtime",
|
|
] }
|
|
pin-project.workspace = true
|
|
futures.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
snafu.workspace = true
|
|
tokio.workspace = true
|
|
uuid.workspace = true
|
|
libc = "0.2"
|
|
|
|
[build-dependencies]
|
|
pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
|
|
|
|
[features]
|
|
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
|
|
fp16kernels = ["lancedb/fp16kernels"]
|
|
remote = ["lancedb/remote"]
|