mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-29 09:28:27 +00:00
test(python): cover concurrent S3 table opens (#3833)
## Summary - add regression coverage for the reported synchronous Python workload with 32 simultaneous `open_table` calls - verify every independently opened S3-backed table handle can read through the connection's shared session and object-store client ## Root cause In Python v0.13.0, each synchronous table handle lazily constructed its own Lance dataset. Opening many handles in parallel therefore triggered independent S3 client construction and bucket-region resolution, which failed under thread pressure. The current Rust-backed connection path owns a shared Lance session and retains its object-store handle, so table opens reuse the existing S3 client; these tests lock in that behavior through the public Python API and a causal Session-registry invariant. ## Validation - `uvx --from 'ruff==0.15.20' ruff format --check python/tests/test_s3.py` - `uvx --from 'ruff==0.15.20' ruff check .` - `cargo fmt --all` - `cargo test --quiet --features remote -p lancedb test_concurrent_open_table_reuses_connection_object_store` - `cargo check --quiet --features remote --tests --examples` - equivalent 32-thread `open_table(...).count_rows()` workload against a local database - targeted S3 test collected successfully locally; execution requires the CI LocalStack service, which is unavailable in this runner Fixes #1786 <!-- lance-gatekeeper-fix:v1 agent=d311f3c7151f77ae22b4997702e7b7db generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
committed by
GitHub
parent
391cac9034
commit
2fbf6d6211
@@ -4,6 +4,7 @@
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import timedelta
|
||||
import threading
|
||||
|
||||
@@ -86,6 +87,25 @@ def test_s3_lifecycle(s3_bucket: str):
|
||||
asyncio.run(test())
|
||||
|
||||
|
||||
@pytest.mark.s3_test
|
||||
def test_concurrent_open_table(s3_bucket: str):
|
||||
uri = f"s3://{s3_bucket}/test_concurrent_open_table"
|
||||
db = lancedb.connect(uri, storage_options=copy.copy(CONFIG))
|
||||
db.create_table("test", pa.table({"x": [1, 2, 3]}))
|
||||
|
||||
num_workers = 32
|
||||
barrier = threading.Barrier(num_workers)
|
||||
|
||||
def open_and_count(_):
|
||||
barrier.wait()
|
||||
return db.open_table("test").count_rows()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
row_counts = list(pool.map(open_and_count, range(num_workers)))
|
||||
|
||||
assert row_counts == [3] * num_workers
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kms_key():
|
||||
kms = get_boto3_client("kms", endpoint_url=CONFIG["aws_endpoint"])
|
||||
|
||||
Reference in New Issue
Block a user